高可用集群搭建系列文章之三：Hadoop集群搭建

﹏ヽ暗。殇╰゛Y 2024-04-18 15:09 68阅读 0赞

本篇文章主要介绍Hadoop HA的搭建

# 1、下载安装包，并解压 #

[下载地址][Link 1]  
解压命令如下：

tar -zxvf hadoop-2.6.5.tar.gz

# 2、配置/etc/profile #

export HADOOP_HOME=/home/hadoop/app/hadoop/hadoop-2.6.5
    export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

# 3、配置hadoop-env.sh，添加jdk路径 #

export JAVA_HOME=/opt/java/jdk1.8.0_221

# 4、配置core-site.xml #

<property>
                    <name>fs.defaultFS</name>
                    <value>hdfs://mycluster</value>
            </property>
            
            <property>
                    
                    <name>fs.trash.checkpoint.interval</name>
                    <value>0</value>
            </property>
            <property>
                    
                    <name>fs.trash.interval</name>
                    <value>1440</value>
            </property>
             
            <property>   
                    <name>hadoop.tmp.dir</name>
                    <value>/home/hadoop/app/hadoop/hadoop-2.6.5/tmp</value>
            </property>
             
            <property>
                    <name>ha.zookeeper.quorum</name>
                    <value>slave01:2181,slave02:2181,slave03:2181,master01:2181,master02:2181</value>
            </property>
             
            <property>
                    <name>ha.zookeeper.session-timeout.ms</name>
                    <value>2000</value>
            </property>
           
            <property>
               <name>hadoop.proxyuser.hadoop.hosts</name>
               <value>*</value> 
            </property> 
           
            <property> 
                <name>hadoop.proxyuser.hadoop.groups</name> 
                <value>*</value> 
           </property>

# 5、配置hdfs-site.xml，其中dfs.nameservices的值要和fs.defaultFS的值一致 #

<property>
            <name>dfs.permissions.superusergroup</name>
    	    <value>hadoop</value>
        </property>
    
    	
    	<property>
    		<name>dfs.webhdfs.enabled</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>dfs.namenode.name.dir</name>
    		<value>/home/hadoop/app/hadoop/hadoop-2.6.5/data/dfs/name</value>
    		<description> namenode 存放name table(fsimage)本地目录（需要修改）</description>
    	</property>
    	<property>
    		<name>dfs.datanode.data.dir</name>
    		<value>/home/hadoop/app/hadoop/hadoop-2.6.5/data/dfs/data</value>
    		<description>datanode存放block本地目录（需要修改）</description>
    	</property>
    	
    	<property>
    		<name>dfs.replication</name>
    		<value>3</value>
    	</property>
    	
    	<property>
    		<name>dfs.blocksize</name>
    		<value>268435456</value>
    	</property>
    	
        
    	
    	<property>
    		<name>dfs.nameservices</name>
    		<value>mycluster</value>
    	</property>
    	<property>
    		
    		<name>dfs.ha.namenodes.mycluster</name>
    		<value>nn1,nn2</value>
    	</property>
    
    	
    	<property>
    		<name>dfs.namenode.rpc-address.mycluster.nn1</name>
    		<value>master01:8020</value>
    	</property>
    	
    	<property>
    		<name>dfs.namenode.rpc-address.mycluster.nn2</name>
    		<value>master02:8020</value>
    	</property>
    
        
    	<property>
    		<name>dfs.namenode.http-address.mycluster.nn1</name>
    		<value>master01:50070</value>
    	</property>
    	
    	<property>
    		<name>dfs.namenode.http-address.mycluster.nn2</name>
    		<value>master02:50070</value>
    	</property>
    
    	
    	
    	<property>
    		<name>dfs.journalnode.http-address</name>
    		<value>0.0.0.0:8480</value>
    	</property>
    	<property>
    		<name>dfs.journalnode.rpc-address</name>
    		<value>0.0.0.0:8485</value>
    	</property>
    	<property>
            
    		<name>dfs.namenode.shared.edits.dir</name>
    		<value>qjournal://master01:8485;master02:8485;slave01:8485/mycluster</value>
    	</property>
    
    	<property>
            
    		<name>dfs.journalnode.edits.dir</name>
    		<value>/home/hadoop/app/hadoop/hadoop-2.6.5/data/dfs/jn</value>
    	</property>
    	
    	<property>
            
    		<name>dfs.client.failover.proxy.provider.mycluster</name>
    		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    	</property>
        
    	<property>
    		<name>dfs.ha.fencing.methods</name>
    		<value>sshfence</value>
    	</property>
    	 
    	<property>
    		<name>dfs.ha.fencing.ssh.private-key-files</name>
    		<value>/home/hadoop/.ssh/id_rsa</value>
    	</property>
    	<property>
            
    		<name>dfs.ha.fencing.ssh.connect-timeout</name>
    		<value>30000</value>
    	</property>
    
    	
    	
    	<property>
    		<name>dfs.ha.automatic-failover.enabled</name>
    		<value>true</value>
    	</property>
    	
    	 <property>
    	   <name>dfs.hosts</name>
    	   <value>/home/hadoop/app/hadoop/hadoop-2.6.5/etc/hadoop/slaves</value>
    	 </property>

# 6、配置yarn-site.xml #

<property>
    		<name>yarn.nodemanager.aux-services</name>
    		<value>mapreduce_shuffle</value>
    	</property>
    	<property>
    		<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    		<value>org.apache.hadoop.mapred.ShuffleHandler</value>
    	</property>
    	<property>
    		<name>yarn.nodemanager.localizer.address</name>
    		<value>0.0.0.0:23344</value>
    		<description>Address where the localizer IPC is.</description>
    	</property>
    	<property>
    		<name>yarn.nodemanager.webapp.address</name>
    		<value>0.0.0.0:23999</value>
    		<description>NM Webapp address.</description>
    	</property>
    
    	
    	
    	<property>
    		<name>yarn.resourcemanager.connect.retry-interval.ms</name>
    		<value>2000</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.ha.enabled</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.cluster-id</name>
    		<value>yarn-cluster</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.ha.rm-ids</name>
    		<value>rm01,rm02</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.hostname.rm01</name>
    		<value>master01</value>
    	</property>	
        
    	<property>
    		<name>yarn.resourcemanager.hostname.rm02</name>
    		<value>master02</value>
    	</property>
    
        
    
    	<property>
    		<name>yarn.resourcemanager.scheduler.class</name>
    		<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.recovery.enabled</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
    		<value>5000</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.store.class</name>
    		<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
    	</property>
    	 
    	<property>
    		<name>yarn.resourcemanager.zk-address</name>
    		<value>slave01:2181,slave02:2181,slave03:2181,master01:2181,master02:2181</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.zk.state-store.address</name>
    		<value>slave01:2181,slave02:2181,slave03:2181,master01:2181,master02:2181</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.address.rm01</name>
    		<value>master01:23140</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.address.rm02</name>
    		<value>master02:23140</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.scheduler.address.rm01</name>
    		<value>master01:23130</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.scheduler.address.rm02</name>
    		<value>master02:23130</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.admin.address.rm01</name>
    		<value>master01:23141</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.admin.address.rm02</name>
    		<value>master02:23141</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.resource-tracker.address.rm01</name>
    		<value>master01:23125</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.resource-tracker.address.rm02</name>
    		<value>master02:23125</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.webapp.address.rm01</name>
    		<value>master01:8088</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.webapp.address.rm02</name>
    		<value>master02:8088</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.webapp.https.address.rm01</name>
    		<value>master01:23189</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.webapp.https.address.rm02</name>
    		<value>master02:23189</value>
    	</property>
    
        
    	<property>
    	   <name>yarn.log-aggregation-enable</name>
    	   <value>true</value>
    	</property>
    	<property>
    		 <name>yarn.log.server.url</name>
    		 <value>http://master01:19888/jobhistory/logs</value>
    		 <description>每台机器上配置不一样</description>
    	</property>
    
    
    	<property>
    		<name>yarn.nodemanager.resource.memory-mb</name>
    		<value>2048</value>
    	</property>
    	<property>
    		<name>yarn.scheduler.minimum-allocation-mb</name>
    		<value>1024</value>
    		<discription>单个任务可申请最少内存，默认1024MB</discription>
    	 </property>
    
      
      <property>
    	<name>yarn.scheduler.maximum-allocation-mb</name>
    	<value>2048</value>
    	<discription>单个任务可申请最大内存，默认8192MB</discription>
      </property>
    
       <property>
           <name>yarn.nodemanager.resource.cpu-vcores</name>
           <value>2</value>
      </property>

# 7、配置mapred-site.xml #

<property>
    		<name>mapreduce.framework.name</name>
    		<value>yarn</value>
    	</property>
    	
    	
    	<property>
    		<name>mapreduce.jobhistory.address</name>
    		<value>master01:10020</value>
    		<description>每台机器上配置不一样</description>
    	</property>
    	
    	<property>
    		<name>mapreduce.jobhistory.webapp.address</name>
    		<value>master01:19888</value>
    		<description>每台机器上配置不一样</description>
    	</property>

# 8、配置slaves，配置DataNode #

slave01
    slave02
    slave03

# 9、拷贝hadoop到其他机器 #

scp -r hadoop hadoop@master02:/home/hadoop/app/
    scp -r hadoop hadoop@slave01:/home/hadoop/app/
    scp -r hadoop hadoop@slave02:/home/hadoop/app/
    scp -r hadoop hadoop@slave03:/home/hadoop/app/

# 9、格式化启动集群 #

### 9.1 在所有节点上，清空data目录下数据 ###

rm -rf /home/hadoop/app/hadoop/hadoop-2.6.5/data/*

### 9.2 在mster01、master02、slave01三个节点上启动journalnode ###

hadoop-daemon.sh start journalnode

### 9.3 格式化namenode节点（任一matser节点） ###

hadoop namenode -format

### 9.4 复制data目录到另一个matser节点 ###

scp -r dfs hadoop@master02:/home/hadoop/app/hadoop/hadoop-2.6.5/data/

### 9.5 初始化zkfc（任一matser节点） ###

hdfs zkfc -formatZK

### 9.6 启动集群 ###

start-dfs.sh
    start-yarn.sh

### 9.7 启动备用节点（master02）的resourcemanager ###

yarn-daemon.sh start resourcemanager

### 9.8 在每个节点上启动日志 ###

mr-jobhistory-daemon.sh start historyserver

# 10、正常启动集群 #

### 10.1 启动集群 ###

start-dfs.sh
    start-yarn.sh

### 10.2 启动备用节点（master02）的resourcemanager ###

yarn-daemon.sh start resourcemanager

### 10.3 在每个节点上启动日志 ###

mr-jobhistory-daemon.sh start historyserver

# 11、关闭集群 #

### 11.1 关闭集群 ###

stop-dfs.sh
    stop-yarn.sh

### 11.2 关闭备用节点（master02）的resourcemanager ###

yarn-daemon.sh stop resourcemanager

### 11.3 在每个节点上关闭日志 ###

mr-jobhistory-daemon.sh stop historyserver

# 12、验证 #

### 12.1 调用jps命令查看（这里自己编写了一个脚本，可以分别去各个节点上查看）![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70] ###

### 12.2 浏览器输入http://192.168.133.16:50070查看，此节点是备用主节点 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 1]

### 12.3 浏览器输入http://192.168.133.17:50070查看，此节点是主节点 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 2]

### 12.4 浏览器输入http://192.168.133.16:8088查看 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 3]

### 12.5 浏览器输入http://192.168.133.17:8088查看 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 4]

[Link 1]: http://archive.apache.org/dist/hadoop/core/
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/18/a8303177c4f44ad29901213e46ca7659.jpg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 1]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/18/0a8f1131a97544a38aae0cc2eda2254c.jpg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 2]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/18/84f5349d7ea242c29149af15f88adcab.jpg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 3]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/18/5deeeeb6bf70456aaa5c77343a27e9b7.jpg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZlbmdodWliaWFu_size_16_color_FFFFFF_t_70 4]: https://image.dandelioncloud.cn/pgy_files/images/2024/04/18/16d2c51dff0042b093496004a4068794.jpg