Spark读取Hbase数据 以你之姓@ 2022-04-12 12:27 284阅读 0赞 \[size=medium\] 直接上代码:可以上传jar包,给远程的spark集群,这样能在idea中windows环境上运行代码: \[/size\] package com.tools.hbase import org.apache.hadoop.hbase.HBaseConfiguration import org.apache.hadoop.hbase.client.Result import org.apache.hadoop.hbase.io.ImmutableBytesWritable import org.apache.hadoop.hbase.mapreduce.TableInputFormat import org.apache.spark.{SparkContext, SparkConf} /** * Created by qindongliang on 2016/1/12. */ object HbaseTest { def main(args: Array[String]) { System.setProperty("user.name", "webmaster"); System.setProperty("HADOOP_USER_NAME", "webmaster"); val jarPath="target\\scala-2.11\\scala-spark_2.11-1.0.jar"; val sparkConf=new SparkConf().setMaster("spark://192.168.1.187:7077").setAppName("read hbase"); val seq=Seq(jarPath) :+ "D:\\bigdata\\hbaselib\\hbase-protocol-0.98.12-hadoop2.jar" :+ "D:\\bigdata\\hbaselib\\hbase-common-0.98.12-hadoop2.jar" :+ "D:\\bigdata\\hbaselib\\htrace-core-2.04.jar" :+ "D:\\bigdata\\hbaselib\\hbase-client-0.98.12-hadoop2.jar" :+ "D:\\bigdata\\hbaselib\\hbase-server-0.98.12-hadoop2.jar" :+ "D:\\bigdata\\hbaselib\\guava-12.0.1.jar" // val seq=Seq(jarPath) // println("jar包路径:"+seq) sparkConf.setJars(seq) val sc=new SparkContext(sparkConf); val conf=HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", "192.168.1.187"); conf.set(TableInputFormat.INPUT_TABLE,"xxxx") //得到Hbase的Result转成RDD val rdd=sc.newAPIHadoopRDD(conf,classOf[TableInputFormat] ,classOf[ImmutableBytesWritable],classOf[Result]); val count=rdd.count(); println("数量:"+count) sc.stop(); } }
还没有评论,来说两句吧...