Cloudera(CDH) 部署在centos7系统教程

比眉伴天荒 2022-04-16 04:54 319阅读 0赞

系统

[root@i-138dr3jn ~]# cat /etc/redhat-release 
    CentOS Linux release 7.2.1511 (Core)

主机节点

<table> 
 <thead> 
  <tr> 
   <th>主机</th> 
   <th>ip</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td><a href="http://cdh.master.aniu.so" rel="nofollow">cdh.master.aniu.so</a> master</td> 
   <td>10.39.47.63</td> 
  </tr> 
  <tr> 
   <td><a href="http://cdh.node1.aniu.so" rel="nofollow">cdh.node1.aniu.so</a> node1</td> 
   <td>10.39.47.64</td> 
  </tr> 
  <tr> 
   <td><a href="http://cdh.node2.aniu.so" rel="nofollow">cdh.node2.aniu.so</a> node2</td> 
   <td>10.39.47.65</td> 
  </tr> 
  <tr> 
   <td><a href="http://cdh.node3.aniu.so" rel="nofollow">cdh.node3.aniu.so</a> node3</td> 
   <td>10.39.47.66</td> 
  </tr> 
 </tbody> 
</table>

设置主机名

[root@master ~]#   hostnamectl set-hostname master

主机更新

yum update -y

关闭防火墙

sudo chkconfig iptables off
    sudo service iptables stop
    systemctl stop firewalld
    systemctl disable firewalld

设置selinux

参考https://www.cloudera.com/documentation/enterprise/6/6.0/topics/install_cdh_disable_selinux.html
    
    sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config && setenforce 0

设置每台服务器之间设置免密认证

[root@node3 ~]# cat /etc/hosts
    10.39.47.63 cdh.master.aniu.so master
    10.39.47.64 cdh.node1.aniu.so node1
    10.39.47.65 cdh.node2.aniu.so node2
    10.39.47.66 cdh.node3.aniu.so node3
    
    # 设置swap参数
    echo never > /sys/kernel/mm/transparent_hugepage/defrag  #建议写到开启启动新里
    sysctl -w vm.swappiness=0  # 建议写进sysctl.conf

同步主机时间

参考 https://www.cloudera.com/documentation/enterprise/6/6.0/topics/install_cdh_enable_ntp.html
    yum install ntp -y
    
    启动服务：systemctl start ntpd.service 
    开机自启：systemctl enable ntpd.service
    
    hwclock --systohc

yum源

wget  https://archive.cloudera.com/cm6/6.0.0/redhat7/yum/cloudera-manager.repo  -P /etc/yum.repos.d/ 
    
    rpm --import https://archive.cloudera.com/cm6/6.0.0/redhat7/yum/RPM-GPG-KEY-cloudera

安装 oracle jdk

yum install oracle-j2sdk1.8 -y

安装 Cloudera Manager Packages

yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server -y

安装数据库并创建数据库以及安装数据库驱动  
参考  
[https://www.cloudera.com/documentation/enterprise/6/6.0/topics/cm\_ig\_mysql.html\#cmig\_topic\_5\_5][https_www.cloudera.com_documentation_enterprise_6_6.0_topics_cm_ig_mysql.html_cmig_topic_5_5]

create database cmf DEFAULT CHARACTER SET utf8;
    GRANT ALL PRIVILEGES ON `cmf`.* TO 'cmf'@'%' IDENTIFIED BY 'enN#$12345';
    flush privileges;
    
    [root@master ~]# wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
    --2018-11-17 11:30:21--  https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
    Resolving dev.mysql.com (dev.mysql.com)... 137.254.60.11
    Connecting to dev.mysql.com (dev.mysql.com)|137.254.60.11|:443... connected.
    HTTP request sent, awaiting response... 302 Found
    Location: https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz [following]
    --2018-11-17 11:30:24--  https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
    Resolving cdn.mysql.com (cdn.mysql.com)... 2.19.60.91
    Connecting to cdn.mysql.com (cdn.mysql.com)|2.19.60.91|:443... connected.
    HTTP request sent, awaiting response... 200 OK
    Length: 4434926 (4.2M) [application/x-tar-gz]
    Saving to: ‘mysql-connector-java-5.1.46.tar.gz’
    
    100%[====================================================================================================================>] 4,434,926   1.03MB/s   in 4.1s   
    
    2018-11-17 11:30:29 (1.03 MB/s) - ‘mysql-connector-java-5.1.46.tar.gz’ saved [4434926/4434926]
    
    [root@master ~]# tar zxvf mysql-connector-java-5.1.46.tar.gz
    [root@master mysql-connector-java-5.1.46]# cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
    [root@master mysql-connector-java-5.1.46]# ls -l /usr/share/java/mysql-connector-java.jar

数据库表

https://www.cloudera.com/documentation/enterprise/6/6.0/topics/cm_ig_mysql.html#cmig_topic_5_5

启动

systemctl start cloudera-scm-server
    systemctl status cloudera-scm-server

查看是否启动

[root@master ~]# netstat -tlnp 
    Active Internet connections (only servers)
    Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
    tcp        0      0 0.0.0.0:7180            0.0.0.0:*               LISTEN      26110/java          
    tcp        0      0 0.0.0.0:7182            0.0.0.0:*               LISTEN      26110/java          
    tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      16396/sshd          
    tcp6       0      0 :::3306                 :::*                    LISTEN      27191/mysqld        
    tcp6       0      0 :::22                   :::*                    LISTEN      16396/sshd

相关配置文件

[root@master ~]# cat /etc/cloudera-scm-server/db.properties 
    # Copyright (c) 2012 Cloudera, Inc. All rights reserved.
    #
    # This file describes the database connection.
    #
    
    # The database type
    # Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
    com.cloudera.cmf.db.type=mysql
    
    # The database host
    # If a non standard port is needed, use 'hostname:port'
    com.cloudera.cmf.db.host=master
    
    # The database name
    com.cloudera.cmf.db.name=cmf
    
    # The database user
    com.cloudera.cmf.db.user=cmf
    
    # The database user's password
    com.cloudera.cmf.db.password=enN#$12345
    
    # The db setup type
    # After fresh install it is set to INIT
    # and will be changed post config.
    # If scm-server uses Embedded DB then it is set to EMBEDDED
    # If scm-server uses External DB then it is set to EXTERNAL
    com.cloudera.cmf.db.setupType=EXTERNAL

[root@master ~]# cat /etc/default/cloudera-scm-server
    #
    # Specify any command line arguments for the Cloudera SCM Server here.
    #
    
    CMF_SERVER_ARGS=""
    
    #
    # Locate the JDBC driver jar file.
    #
    # The default value is the default system mysql driver on RHEL/CentOS/Ubuntu
    # and the standard, documented location for where to put the oracle jar in CM
    # deployments.
    #
    
    export CMF_JDBC_DRIVER_JAR="/usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar"
    
    #
    # Java Options.
    #
    # Default value sets Java maximum heap size to 2GB, and Java maximum permanent
    # generation size to 256MB.
    #
    
    export CMF_JAVA_OPTS="-Xmx2G -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp"

访问 [http://10.39.47.63:7180][http_10.39.47.63_7180]  
![90][]

![09][]

![111][]

![99][]

![99][99 1]

![111][111 1]

![122][]

![13][]  
参考：  
[官方网站][Link 1]  
[Cloudera(CDH) 简介和在线安装][Cloudera_CDH_]

[CentOS7离线安装Cloudera Manager 5.14.1][CentOS7_Cloudera Manager 5.14.1]

[部署Cloudera Management for centos 7][Cloudera Management for centos 7]

[https_www.cloudera.com_documentation_enterprise_6_6.0_topics_cm_ig_mysql.html_cmig_topic_5_5]: https://www.cloudera.com/documentation/enterprise/6/6.0/topics/cm_ig_mysql.html#cmig_topic_5_5
[http_10.39.47.63_7180]: http://10.39.47.63:7180
[90]: /images/20220416/fe10e267c4384530a6944e68451fadca.png
[09]: /images/20220416/6e85deacfc1e4e548680cfd192a4beb9.png
[111]: /images/20220416/5b7531fe24d14501a052c9229b669798.png
[99]: /images/20220416/44b2a7679e1a43afbd3945826bf83779.png
[99 1]: /images/20220416/f0a5a14e9e4a4225b51a6fa34ff7f777.png
[111 1]: /images/20220416/dd5f977497f448ab830876701effaefa.png
[122]: /images/20220416/96e278ae7c7a4a61a373150698a06f8d.png
[13]: /images/20220416/b8265d7cbed746b388e908a5c269fc82.png
[Link 1]: https://www.cloudera.com/documentation/enterprise/6/6.0/topics/configure_cm_repo.html#cm_repo
[Cloudera_CDH_]: https://blog.csdn.net/wh211212/article/details/78743191
[CentOS7_Cloudera Manager 5.14.1]: https://blog.csdn.net/chenhai201/article/details/78856007
[Cloudera Management for centos 7]: https://www.cnblogs.com/mr-ivan/p/6518800.html