5台hadoop ha集群搭建

喜欢ヅ旅行 2024-04-17 14:13 72阅读 0赞

[https://blog.csdn.net/javaChengXuY/article/details/101525263][https_blog.csdn.net_javaChengXuY_article_details_101525263]

### hadoop ha集群搭建 ###

*   *   *   *  一、hosts文件配置：
             *  二、配置集群服务器之间免密码登陆
             *  三、集群时间同步
             *  四、环境变量配置：
             *   *  1、zookeeper集群配置(将修改配置文件拷贝到各个节点)
                 *   *  1.1 zoo.cfg配置
                     *  1.2 设置myid
                     *  1.3 启动集群
                 *  2、hadoop集群配置(将修改配置文件拷贝到各个节点)
                 *   *  2.1进入hadoop 配置目录
                     *  2.2 vi hadoop-env.sh
                     *  2.3 vi yarn-env.sh
                     *  2.5 vi core-site.xml
                     *  2.5 vi yarn-site.xml
                     *  2.5 vi mapred-site.xml
                     *  2.5 vi slaves
                     *  2.6将修改后的hadoop配置文件拷贝到各个节点
                 *  2.7 hadoop集群启动
                 *  2.7.1在ubuntu-14，ubuntu-15，ubuntu-16分别执行
                 *  2.7.2在ubuntu-18上执行格式化
                 *  2.7.3将执行成功后生成的dir拷贝到ubuntu-19
                 *  2.7.4在ubuntu-18上执启动hadoop 集群
                 *  2.7.5在ubuntu-19上执启动hadoop 集群
             *  五、查看namenode ：
             *  六、手动切换namenode active：

#### 一、hosts文件配置： ####

10.211.55.10 ubuntu-19
    10.211.55.9 ubuntu-18
    10.211.55.12 ubuntu-14
    10.211.55.17 ubuntu-15
    10.211.55.18 ubuntu-16

#### 二、配置集群服务器之间免密码登陆 ####

for ip in 14 15 16 18 19;do ssh-copy-id -i ~/.ssh/id_rsa.pub ubuntu@ubuntu-${ip} ;done

#### 三、集群时间同步 ####

ntpdate ntp1.aliyun.com

#### 四、环境变量配置： ####

export JAVA_HOME=/usr/src/jdk1.8.0_211               
    export JRE_HOME=${JAVA_HOME}/jre
    export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
    export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
    
    export MAVEN_HOME=/usr/src/apache-maven-3.6.1
    export MAVEN_OPTS="-Xms256m -Xmx512m"
    export PATH=$PATH:$MAVEN_HOME/bin
    
    export HADOOP_HOME=/usr/src/hadoop-2.7.7
    export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
    
    export ZOOKEEPER_HOME=/usr/src/apache-zookeeper-3.5.5-bin
    export PATH=$PATH:$ZOOKEEPER_HOME/bin

##### 1、zookeeper集群配置(将修改配置文件拷贝到各个节点) #####

###### 1.1 zoo.cfg配置 ######

tickTime=2000
    initLimit=10
    syncLimit=5
    dataDir=/usr/src/apache-zookeeper-3.5.5-bin/tmp
    clientPort=2181
    server.1=ubuntu-18:2888:3888
    server.2=ubuntu-14:2888:3888
    server.3=ubuntu-15:2888:3888
    server.4=ubuntu-16:2888:3888
    server.5=ubuntu-19:2888:3888:observer

###### 1.2 设置myid ######

[ubuntu@ubuntu-14]# echo "1" > /usr/src/apache-zookeeper-3.5.5-bin/tmp/myid 
    [ubuntu@ubuntu-15]# echo "2" > /usr/src/apache-zookeeper-3.5.5-bin/tmp/myid 
    [ubuntu@ubuntu-16]# echo "3" > /usr/src/apache-zookeeper-3.5.5-bin/tmp/myid 
    [ubuntu@ubuntu-18]# echo "4" > /usr/src/apache-zookeeper-3.5.5-bin/tmp/myid 
    ubuntu@ubuntu-19]# echo "5" >/usr/src/apache-zookeeper-3.5.5-bin/tmp/myid

###### 1.3 启动集群 ######

[ubuntu@ubuntu-14]# zkServer.sh start 
    [ubuntu@ubuntu-15]# zkServer.sh start 
    [ubuntu@ubuntu-16]# zkServer.sh start

##### 2、hadoop集群配置(将修改配置文件拷贝到各个节点) #####

###### 2.1进入hadoop 配置目录 ######

cd /usr/src/hadoop-2.7.7/etc/hadoop

###### 2.2 vi hadoop-env.sh ######

export JAVA_HOME=/usr/src/jdk1.8.0_211

###### 2.3 vi yarn-env.sh ######

export JAVA_HOME=/usr/src/jdk1.8.0_211 
    
    
    
      ###### 2.4 vi hdfs-site.xml
    <?xml version="1.0" encoding="UTF-8"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
         <property>
           <name>dfs.replication</name>
           <value>3</value>
         </property>
         <property>
           <name>dfs.permissions</name>
           <value>false</value>
        </property>
        <property>
          <name>dfs.nameservices</name>
          <value>ns1</value>
        </property>
        <property>
         <name>dfs.blocksize</name>
         <value>134217728</value>
        </property>
        <property>
          <name>dfs.ha.namenodes.ns1</name>
          <value>nn1,nn2</value>
        </property>
        <property>
         <name>dfs.namenode.rpc-address.ns1.nn1</name>
         <value>ubuntu-18:8020</value>
        </property>
        <property>
          <name>dfs.namenode.http-address.ns1.nn1</name>
          <value>ubuntu-18:50070</value>
        </property>
        
        <property>
           <name>dfs.namenode.rpc-address.ns1.nn2</name>
           <value>ubuntu-19:8020</value>
        </property>
        <property>
          <name>dfs.namenode.http-address.ns1.nn2</name>
          <value>ubuntu-19:50070</value>
        </property>
        <property>
         <name>dfs.namenode.shared.edits.dir</name>
         <value>qjournal://ubuntu-14:8485;ubuntu-15:8485;ubuntu-16:8485/ns1</value>
        </property>
        <property>
         <name>dfs.journalnode.edits.dir</name>
         <value>/usr/src/hadoop-2.7.7/data/journalnode</value>
        </property>
        <property>
          <name>dfs.client.failover.proxy.provider.ns1</name>
          <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
        <property>
         <name>dfs.ha.fencing.methods</name>
         <value>sshfence</value>
        </property>
        <property>
          <name>dfs.ha.automatic-failover.enabled</name>
          <value>true</value>
        </property>
    </configuration>

###### 2.5 vi core-site.xml ######

<?xml version="1.0" encoding="UTF-8"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
         <property>
           <name>fs.defaultFS</name>
           <value>hdfs://ns1</value>
         </property>
         <property>
           <name>hadoop.tmp.dir</name>
           <value>/usr/src/hadoop-2.7.7/data/tmp</value>
         </property>
          <property>
            <name>fs.trash.interval</name>
            <value>1440</value>
         </property>
        <property>
          <name>ha.zookeeper.quorum</name>
          <value>ubuntu-14:2181,ubuntu-15:2181,ubuntu-16:2181</value>
        </property>
    </configuration>

###### 2.5 vi yarn-site.xml ######

<?xml version="1.0"?>
    <configuration>
            <property>
                     <name>mapreduce.framework.name</name>
                     <value>yarn</value>
                     <description>Execution framework set to Hadoop YARN.</description>
            </property>
    </configuration>

###### 2.5 vi mapred-site.xml ######

<?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
            <property>
                    <name>mapreduce.framework.name</name>
                    <value>yarn</value>
            </property>
            <property>
                    <name>mapreduce.jobhistory.address</name>
                    <value>ubuntu-18:10020</value>
            </property>
             <property>
                     <name>mapreduce.jobhistory.webapp.address</name>
                     <value>ubuntu-18:19888</value>
            </property>
    </configuration>

###### 2.5 vi slaves ######

ubuntu-14
    ubuntu-15
    ubuntu-16

###### 2.6将修改后的hadoop配置文件拷贝到各个节点 ######

for ip in 14 15 16 19;do scp -r /usr/src/hadoop-2.7.7/etc/hadoop ubuntu@ubuntu-${ip}:/usr/src/hadoop-2.7.7/etc/;done

##### 2.7 hadoop集群启动 #####

##### 2.7.1在ubuntu-14，ubuntu-15，ubuntu-16分别执行 #####

hadoop-daemon.sh start journalnode

##### 2.7.2在ubuntu-18上执行格式化 #####

hdfs namenode -format
    hdfs zkfc -formatZK

##### 2.7.3将执行成功后生成的dir拷贝到ubuntu-19 #####

scp -r  /usr/src/hadoop-2.7.7/data/tmp ubuntu@ubuntu-19:/usr/src/hadoop-2.7.7/data/

##### 2.7.4在ubuntu-18上执启动hadoop 集群 #####