安装BigTop编译的Hadoop3.1.1版本过程记录

桃扇骨 2021-09-28 16:18 968阅读 0赞

本文记录安装编译后的rpm包的过程，主要是在不同主机上分配好需要安装的服务，使用yum命令安装，然后修改对应的配置文件，最后启动服务即可。

# 1、前期准备 #

主要包括主机的设置方面，配置主机名、IP和主机名映射、关闭防火墙等

#关闭防火墙
    hostnamectl set-hostname hdp1
    
    #修改主机名
    172.16.25.139 hdp1
    172.16.25.140 hdp2
    172.16.25.141 hdp3
    172.16.25.142 hdp4
    172.16.25.143 hdp5
    
    #关闭防火墙
    systemctl stop firewalld
    systemctl disable firewalld

# 2、配置免密码登陆 #

分别在hdp1、hdp2配置免密登陆

在hdp1上
    cd ~
    ssh-keygen -t rsa
    cd .ssh
    cat id_rsa.pub >> authorized_keys
    scp ~/.ssh/authorized_keys hdp1:~/.ssh
    scp ~/.ssh/authorized_keys hdp2:~/.ssh
    scp ~/.ssh/authorized_keys hdp3:~/.ssh
    scp ~/.ssh/authorized_keys hdp4:~/.ssh
    
    在hdp2上
    cd ~/.ssh
    cat id_rsa.pub >> authorized_keys
    scp ~/.ssh/authorized_keys hdp1:~/.ssh
    scp ~/.ssh/authorized_keys hdp2:~/.ssh
    scp ~/.ssh/authorized_keys hdp3:~/.ssh
    scp ~/.ssh/authorized_keys hdp4:~/.ssh

# 3、配置JDK #

每台主机分别配置

解压安装包
    /opt/jdk1.8.0_131
    
    配置环境变量，在文件/etc/profile末尾添加如下内容：
    export JAVA_HOME=/opt/jdk1.8.0_131
    export PATH=$PATH:$JAVA_HOME/bin
    
    使环境变量生效
    source /etc/profile

# 4、yum源配置、清理、缓存 #

配置yum源

rpm包放到下面的目录中
    /var/www/html
    
    配置hdp的repo
    cat /etc/yum.repos.d/hdp.repo
    [hdp]
    name=hadoop
    baseurl=http://172.16.25.139
    enabled=1
    gpgcheck=0
    
    启动http服务
    service httpd start
    
    yum clean all
    yum makecache

# 5、所有关于hadoop的rpm包 #

yum list | grep hadoop
    hadoop.x86_64                              3.1.1-1.el7                 @hdp
    hadoop-hdfs.x86_64                         3.1.1-1.el7                 @hdp
    hadoop-hdfs-journalnode.x86_64             3.1.1-1.el7                 @hdp
    hadoop-hdfs-namenode.x86_64                3.1.1-1.el7                 @hdp
    hadoop-hdfs-zkfc.x86_64                    3.1.1-1.el7                 @hdp
    hadoop-hdfs-datanode.x86_64                3.1.1-1.el7                 hdp
    hadoop-hdfs-fuse.x86_64                    3.1.1-1.el7                 hdp
    hadoop-hdfs-secondarynamenode.x86_64       3.1.1-1.el7                 hdp
    hadoop-libhdfs.x86_64                      3.1.1-1.el7                 hdp
    hadoop-libhdfs-devel.x86_64                3.1.1-1.el7                 hdp
    
    
    hadoop-mapreduce.x86_64                    3.1.1-1.el7                 @hdp
    hadoop-mapreduce-historyserver.x86_64      3.1.1-1.el7                 @hdp
    
    hadoop-yarn.x86_64                         3.1.1-1.el7                 @hdp
    hadoop-yarn-nodemanager.x86_64             3.1.1-1.el7                 @hdp
    hadoop-yarn-resourcemanager.x86_64         3.1.1-1.el7                 @hdp
    hadoop-yarn-proxyserver.x86_64             3.1.1-1.el7                 hdp
    hadoop-yarn-timelineserver.x86_64          3.1.1-1.el7                 hdp
    
    hadoop-client.x86_64                       3.1.1-1.el7                 hdp
    hadoop-debuginfo.x86_64                    3.1.1-1.el7                 hdp
    hadoop-doc.x86_64                          3.1.1-1.el7                 hdp
    
    hadoop-conf-pseudo.x86_64                  3.1.1-1.el7                 hdp

# 6、服务分配 #

<table> 
 <thead> 
  <tr> 
   <th>hostname</th> 
   <th>ip</th> 
   <th>说明</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>hdp1</td> 
   <td>172.16.25.139</td> 
   <td>name node</td> 
  </tr> 
  <tr> 
   <td>hdp2</td> 
   <td>172.16.25.140</td> 
   <td>name node</td> 
  </tr> 
  <tr> 
   <td>hdp3</td> 
   <td>172.16.25.141</td> 
   <td>data node</td> 
  </tr> 
  <tr> 
   <td>hdp4</td> 
   <td>172.16.25.142</td> 
   <td>data node</td> 
  </tr> 
  <tr> 
   <td>hdp5</td> 
   <td>172.16.25.143</td> 
   <td>data node</td> 
  </tr> 
 </tbody> 
</table>

<table> 
 <thead> 
  <tr> 
   <th>&nbsp;</th> 
   <th>hdp1</th> 
   <th>hdp2</th> 
   <th>hdp3</th> 
   <th>hdp4</th> 
   <th>hdp5</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>NameNode</td> 
   <td>√</td> 
   <td>√</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
  </tr> 
  <tr> 
   <td>DataNode</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
  </tr> 
  <tr> 
   <td>ResourceManager</td> 
   <td>√</td> 
   <td>√</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
  </tr> 
  <tr> 
   <td>NodeManager</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
  </tr> 
  <tr> 
   <td>Zookeeper</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
  </tr> 
  <tr> 
   <td>journalnode</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
   <td>√</td> 
  </tr> 
  <tr> 
   <td>zkfc</td> 
   <td>√</td> 
   <td>√</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
   <td>&nbsp;</td> 
  </tr> 
 </tbody> 
</table>

# 7、安装服务 #

## hdp1 ##

yum install zookeeper  -y
    yum install hadoop-hdfs-namenode -y
    yum install hadoop-yarn-resourcemanager -y
    yum install hadoop-hdfs-zkfc -y
    yum install hadoop-yarn-nodemanager -y 
    yum install hadoop-hdfs-journalnode -y
    yum install hadoop-mapreduce-historyserver -y
    yum install hadoop-yarn-proxyserver -y    
    yum install hadoop-yarn-timelineserver -y
    yum install hadoop-hdfs-secondarynamenode -y

## hdp2 ##

yum install zookeeper  -y
    yum install hadoop-hdfs-namenode -y
    yum install hadoop-yarn-resourcemanager -y
    yum install hadoop-hdfs-zkfc -y 
    yum install hadoop-yarn-nodemanager -y 
    yum install hadoop-hdfs-journalnode -y
    yum install hadoop-mapreduce-historyserver -y
    yum install hadoop-hdfs-secondarynamenode -y

## hdp3 ##

yum install zookeeper  -y
    yum install hadoop-hdfs-datanode -y
    yum install hadoop-yarn-nodemanager -y
    yum install hadoop-hdfs-journalnode -y
    yum install hadoop-mapreduce-historyserver -y

## hdp4和hdp5 ##

yum install hadoop-hdfs-datanode -y
    yum install hadoop-yarn-nodemanager -y
    yum install hadoop-hdfs-journalnode -y
    yum install hadoop-mapreduce-historyserver -y

# 8、配置服务 #

## zookeeper ##

zoo.cfg添加
    
    dataLogDir=/var/lib/zookeeper/log
    
    server.1=hdp1:2888:3888
    server.2=hdp2:2888:3888
    server.3=hdp3:2888:3888
    在data目录下添加myid文件，内容为对应每台主机的serverid，分别为1，2，3
    
    mkdir /var/lib/zookeeper/log

## hadoop ##

### hadoop-env.sh ###

## 在文件开头加上，根据自己服务器配置设置jvm内存大小
    export JAVA_HOME=/opt/jdk1.8.0_131
    #export PATH=$PATH:$JAVA_HOME/bin
    #export HADOOP_NAMENODE_OPTS=" -Xms1024m -Xmx1024m -XX:+UseParallelGC"
    #export HADOOP_DATANODE_OPTS=" -Xms512m -Xmx512m"
    export HADOOP_LOG_DIR=/opt/data/logs/hadoop

### core-site.xml ###

<configuration>
        
        <property>
            <name>fs.defaultFS</name>
            <value>hdfs://mycluster</value>
        </property>
    
        <property>
            <name>hadoop.tmp.dir</name>
            <value>/opt/data/hadoop/tmp</value>
        </property>
    
        
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>hdp1:2181,hdp2:2181,hdp3:2181</value>
    </property>
    
    
         <property>
             <name>ha.zookeeper.session-timeout.ms</name>
             <value>30000</value>
             <description>ms</description>
         </property>
    
        <property>
            <name>fs.trash.interval</name>
            <value>1440</value>
        </property>
    
    </configuration>

### hdfs-site.xml ###

<configuration>
    
    
    <property>
        <name>dfs.qjournal.start-segment.timeout.ms</name>
        <value>60000</value>
    </property>
        
        <property>
            <name>dfs.nameservices</name>
            <value>mycluster</value>
        </property>
            
        <property>
            <name>dfs.ha.namenodes.mycluster</name>
            <value>nn1,nn1</value>
        </property>
        
        <property>
            <name>dfs.namenode.rpc-address.mycluster.nn1</name>
            <value>hdp1:8020</value>
        </property>
        
        <property>
            <name>dfs.namenode.rpc-address.mycluster.nn2</name>
            <value>hdp2:8020</value>
        </property>
         
        <property>
            <name>dfs.namenode.http-address.mycluster.nn1</name>
            <value>hdp1:50070</value>
        </property>
        
        <property>
            <name>dfs.namenode.http-address.mycluster.nn2</name>
            <value>hdp2:50070</value>
        </property>
        
        <property>
            <name>dfs.namenode.shared.edits.dir</name>
           <value>qjournal://hdp1:8485;hdp2:8485;hdp3:8485;hdp4:8485;hdp5:8485/mycluster</value>
        </property>
        
        <property>
            <name>dfs.client.failover.proxy.provider.mycluster</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
        
        <property>
            <name>dfs.ha.fencing.methods</name>
            <value>
                sshfence
    	    shell(/bin/true)
            </value>
        </property>
      <property>
         <name>dfs.permissions.enabled</name>
         <value>false</value>
      </property>
        <property>
            <name>dfs.support.append</name>
            <value>true</value>
        </property>
        
        <property>
            <name>dfs.ha.fencing.ssh.private-key-files</name>
            <value>/root/.ssh/id_rsa</value>
        </property>
        
        <property>
            <name>dfs.replication</name>
            <value>2</value>
        </property>
        <property>
            <name>dfs.namenode.name.dir</name>
            <value>/opt/data/hadoop/hdfs/nn</value>
        </property>
        <property>
            <name>dfs.datanode.data.dir</name>
            <value>/opt/data/hadoop/hdfs/dn</value>
        </property>
        
        <property>
            <name>dfs.journalnode.edits.dir</name>
            <value>/opt/data/hadoop/hdfs/jn</value>
        </property>
        
        <property>
            <name>dfs.ha.automatic-failover.enabled</name>
            <value>true</value>
        </property>
        
        <property>
            <name>dfs.webhdfs.enabled</name>
            <value>true</value>
        </property>
        
        <property>
            <name>dfs.ha.fencing.ssh.connect-timeout</name>
            <value>30000</value>
        </property>
        <property>
            <name>ha.failover-controller.cli-check.rpc-timeout.ms</name>
            <value>60000</value>
        </property>
    
    </configuration>

### mapred-site.xml ###

<configuration>
        
        <property>
            <name>mapreduce.framework.name</name>
            <value>yarn</value>
        </property>
    
        
        <property>
            <name>mapreduce.jobhistory.address</name>
            <value>hdp1:10020</value>
        </property>
        
        <property>
            <name>mapreduce.jobhistory.webapp.address</name>
            <value>hdp1:19888</value>
        </property>
        <property>
          <name>mapreduce.application.classpath</name>
          <value>
              /usr/lib/hadoop/etc/hadoop,
              /usr/lib/hadoop/*,
              /usr/lib/hadoop/lib/*,
              /usr/lib/hadoop-hdfs/*,
              /usr/lib/hadoop-hdfs/lib/*,
            /usr/lib/mapreduce/*,
             /usr/lib/mapreduce/lib/*,
              /usr/lib/hadoop-yarn/*,
              /usr/lib/hadoop-yarn/lib/*
          </value>
        </property>
    </configuration>

### yarn-site.xml ###

<configuration>
    
    
        
        <property>
            <name>yarn.resourcemanager.ha.enabled</name>
            <value>true</value>
        </property>
        
        <property>
            <name>yarn.resourcemanager.cluster-id</name>
            <value>yrc</value>
        </property>
        
        <property>
            <name>yarn.resourcemanager.ha.rm-ids</name>
            <value>rm1,rm2</value>
        </property>
        
        <property>
            <name>yarn.resourcemanager.hostname.rm1</name>
            <value>hdp1</value>
        </property>
        <property>
            <name>yarn.resourcemanager.hostname.rm2</name>
            <value>hdp2</value>
        </property>
        
        <property>
            <name>yarn.resourcemanager.zk-address</name>
            <value>hdp1:2181,hdp2:2181,hdp3:2181</value>
        </property>
        <property>
            <name>yarn.nodemanager.aux-services</name>
            <value>mapreduce_shuffle</value>
        </property>
        <property>
            <name>yarn.log-aggregation-enable</name>
            <value>true</value>
        </property>
        <property>
            <name>yarn.log-aggregation.retain-seconds</name>
            <value>86400</value>
        </property>
      <property>
      <name>yarn.resourcemanager.webapp.address.rm1</name>
      <value>hdp1:8088</value>
    </property>
    <property>
      <name>yarn.resourcemanager.webapp.address.rm2</name>
      <value>hdp2:8088</value>
    </property>
        
        <property>
            <name>yarn.resourcemanager.recovery.enabled</name>
            <value>true</value>
        </property>
        
        <property>
            <name>yarn.resourcemanager.store.class</name>
            <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
        </property>
    </configuration>

### workers ###

hdp3
    hdp4
    hdp5

# 9、创建目录 #

hdp1-hdp5

mkdir -p /opt/data/logs/hadoop
    mkdir -p /opt/data/hadoop/hdfs/nn
    mkdir -p /opt/data/hadoop/hdfs/dn
    mkdir -p /opt/data/hadoop/hdfs/jn
    mkdir -p /opt/data/hadoop/tmp
    mkdir -p /opt/data/hadoop/tmp/yarn/timeline/
     
    chown -R hdfs:hdfs /opt/data
    chown -R yarn:yarn /opt/data/hadoop/tmp/yarn
    
    export BIGTOP_DEFAULTS_DIR=/usr/lib
    export HADOOP_PREFIX=/usr/lib/hadoop
    export YARN_CONF_DIR=/usr/lib/hadoop-yarn

# 10、启动服务 #

## zookeeper ##

hdp1-hdp3启动

## hadoop ##

#启动journalnode
    hdp1,hdp2,hdp3,hdp4,hdp5
    /etc/init.d/hadoop-hdfs-journalnode start
    
    #格式化namenode
    hdp1
    hadoop namenode -format
    
    
    #格式化zkfc
    hdp1
    hdfs zkfc -formatZK
    ./hadoop-hdfs-zkfc start
    
    
    #启动HDFS,zookeeper会选举哪个是active，哪个是standby
    hdp1&hdp2
    /etc/init.d/hadoop-hdfs-namenode start
    
    
    #启动YARN
    #在主备 resourcemanager 中随便选择一台进行启动
    start-yarn.sh
    
    
    #启动 mapreduce 任务历史服务器
    mr-jobhistory-daemon.sh start historyserver