CDH集群中安装kylin：code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask

柔光的暖阳◎ 2022-02-24 06:23 163阅读 0赞

下载cdh版的kylin tar包： [http://mirrors.tuna.tsinghua.edu.cn/apache/kylin/apache-kylin-2.6.1/apache-kylin-2.6.1-bin-cdh60.tar.gz ][http_mirrors.tuna.tsinghua.edu.cn_apache_kylin_apache-kylin-2.6.1_apache-kylin-2.6.1-bin-cdh60.tar.gz]  
注： 此版本kylin依赖jdk8

# 1, 配置环境变量 #

export JAVA_HOME=/usr/java/jdk1.8
    export KYLIN_HOME=/opt/kylin
    export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark
    export PATH=$PATH:$JAVA_HOME/bin:$KYLIN_HOME/bin

# 2，启动服务 #

*  check-env.sh , 查看是否配置正确
 *  kylin.sh start

kylin启动报错：　找不到SPARK\_HOME (2.6.1有没有自带spark, 2.1.0有自带spark不会报此错误)

提示spark not found, set SPARK_HOME, or run bin/download-spark.sh
    function retrieveDependency() {
        #retrive $hive_dependency and $hbase_dependency
        source ${dir}/find-hive-dependency.sh
        source ${dir}/find-hbase-dependency.sh
        source ${dir}/find-hadoop-conf-dir.sh
        source ${dir}/find-kafka-dependency.sh
        source ${dir}/find-spark-dependency.sh
    	
    find-spark-dependency.sh==>定位报错点
    if [ ! -d "$spark_home/jars" ]
      then
        quit "spark not found, set SPARK_HOME, or run bin/download-spark.sh"
    fi
    ======》下载对应版本的spark： ./bin/download-spark.sh
    
    启动服务： kylin.sh start ==>webui:  http://host:7070/kylin ,ADMIN/KYLIN

# 3，测试使用：code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask #

测试使用： sample.sh /sample-streaming.sh， 重启服务kylin.sh stop && kylin.sh start
    cube-->action: build 报错日志
    FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
    排查错误：　设置kylin_hive_conf.xml文件, 由于内存过下，使得程序不能运行，所以要关闭map端join的优化，设置相关值为false
    <property>
    	<name>hive.auto.convert.join</name>
    	<value>true</value>
    </property>
    <property>
    	<name>hive.auto.convert.join.noconditionaltask</name>
    	<value>true</value>
    </property>
    <property>
    	<name>hive.auto.convert.join.noconditionaltask.size</name>
    	<value>100000000</value>
    </property>
    
    或者配置hive启动参数：
    SET hive.auto.convert.join=false;
    SET mapreduce.map.memory.mb = 16384; 
    SET mapreduce.map.java.opts='-Djava.net.preferIPv4Stack=true -Xmx13107M';
    SET mapreduce.reduce.memory.mb = 13107; 
    SET mapreduce.reduce.java.opts='-Djava.net.preferIPv4Stack=true -Xmx16384M';
    set hive.support.concurrency = false;

# 4,配置kylin.properties文件 #

<table> 
 <thead> 
  <tr> 
   <th align="left">配置项</th> 
   <th align="left">k1-v1</th> 
   <th align="left">k2-v2</th> 
   <th align="left">k3-v3</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td align="left">METADATA ,ENV</td> 
   <td align="left">kylin.metadata.url=kylin_metadata@hbase</td> 
   <td align="left">kylin.server.mode=all</td> 
   <td align="left">kylin.server.cluster-servers=node1:7070</td> 
  </tr> 
  <tr> 
   <td align="left">SOURCE</td> 
   <td align="left">kylin.source.hive.client=cli</td> 
   <td align="left">kylin.source.hive.enable-sparksql-for-table-ops=true</td> 
   <td align="left"></td> 
  </tr> 
  <tr> 
   <td align="left">JOB</td> 
   <td align="left">kylin.job.retry=4</td> 
   <td align="left">kylin.job.max-concurrent-jobs=10</td> 
   <td align="left"></td> 
  </tr> 
  <tr> 
   <td align="left">ENGINE</td> 
   <td align="left">kylin.job.retry=4</td> 
   <td align="left">kylin.engine.mr.reduce-input-mb=200</td> 
   <td align="left">kylin.engine.mr.max-reducer-number=100</td> 
  </tr> 
  <tr> 
   <td align="left">SPARK ENGINE CONFIGS</td> 
   <td align="left">kylin.env.hadoop-conf-dir=/etc/hadoop/conf</td> 
   <td align="left">kylin.engine.spark-conf.spark.master=yarn</td> 
   <td align="left">kylin.engine.spark-conf.spark.yarn.archive=hdfs://xxx/spark-libs.jar</td> 
  </tr> 
 </tbody> 
</table>

# 5,配置kylin使用spark-engine #

*  配置kylin.properties

#### SOURCE ###
    kylin.source.hive.enable-sparksql-for-table-ops=false
    kylin.source.hive.sparksql-beeline-shell=/path/to/spark-client/bin/beeline
    hive.security.authorization.sqlstd.confwhitelist.append= -u jdbc:hive2://localhost:10000
    
    #### SPARK ENGINE CONFIGS ###
    kylin.env.hadoop-conf-dir=/etc/hadoop/conf
    kylin.engine.spark-conf.spark.master=yarn
    #拷贝hive-site.xml到指定的hadoop配置目录
    cp /etc/hive/conf/hive-site.xml    /etc/hadoop/conf

*  配置spark的在hdfs文件系统的jar包

jar cv0f spark-libs.jar -C $SPARK_HOME/jars/ .
    hadoop fs -mkdir -p /kylin/spark/
    hadoop fs -put spark-libs.jar /kylin/spark/
    
    # 配置kylin.properties
    kylin.engine.spark-conf.spark.yarn.archive=hdfs://node1:8020/kylin/spark/spark-libs.jar

# 6,设置开机启动 #

[hdfs@node1 kylin]$ tail /etc/rc.d/rc.local
    service mysqld start
    echo never > /sys/kernel/mm/transparent_hugepage/defrag
    echo never > /sys/kernel/mm/transparent_hugepage/enabled
    /root/sh/cdh.sh start
    sleep 100
    #ssh root@localhost 'source /etc/profile; su -s /bin/sh -c "/opt/kylin/bin/kylin.sh start" hdfs'
    /usr/local/bin/kylin start

[http_mirrors.tuna.tsinghua.edu.cn_apache_kylin_apache-kylin-2.6.1_apache-kylin-2.6.1-bin-cdh60.tar.gz]: http://mirrors.tuna.tsinghua.edu.cn/apache/kylin/apache-kylin-2.6.1/apache-kylin-2.6.1-bin-cdh60.tar.gz