Hadoop 集群配置

爱被打了一巴掌 2022-02-25 01:26 247阅读 0赞

### 文章目录 ###

*   *  集群部署规划
     *  配置
     *   *  核心配置文件
         *  hdfs 配置文件
         *   *  hadoop-env.sh
             *  hdfs-site.xml
         *  yarn 配置文件
         *   *  yarn-env.sh
             *  yarn-site.xml
         *  mapreduce 配置文件
         *   *  mapred-env.sh
             *  mapred-site.xml
     *  分发配置

## 集群部署规划 ##

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>hadoop102</th> 
   <th>hadoop103</th> 
   <th>hadoop104</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>HDFS</td> 
   <td>NameNode <br>DataNode</td> 
   <td>DataNode</td> 
   <td>SecondaryNameNode<br> DataNode</td> 
  </tr> 
  <tr> 
   <td>YARN</td> 
   <td>NodeManager</td> 
   <td>ResourceManager<br> NodeManager</td> 
   <td>NodeManager</td> 
  </tr> 
 </tbody> 
</table>

## 配置 ##

### 核心配置文件 ###

core-site.xml

[root@hadoop102 hadoop]# pwd
    /opt/module/hadoop-2.7.2/etc/hadoop
    [root@hadoop102 hadoop]# vim core-site.xml

<property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop102:9000</value>
    </property>
    
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop-2.7.2/data/tmp</value>
    </property>

### hdfs 配置文件 ###

#### [hadoop-env.sh][] ####

[root@hadoop102 hadoop]# vim hadoop-env.sh

export JAVA_HOME=/opt/module/jdk1.8.0_144

#### hdfs-site.xml ####

[root@hadoop102 hadoop]# vim hdfs-site.xml

<property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    
    <property> 
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop104:50090</value>
    </property>

### yarn 配置文件 ###

#### [yarn-env.sh][] ####

[root@hadoop102 hadoop]# vim yarn-env.sh

export JAVA_HOME=/opt/module/jdk1.8.0_144

#### yarn-site.xml ####

[root@hadoop102 hadoop]# vim yarn-site.xml

<property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>hadoop103</value>
    </property>
    
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>

### mapreduce 配置文件 ###

#### [mapred-env.sh][] ####

[root@hadoop102 hadoop]# vim mapred-env.sh

export JAVA_HOME=/opt/module/jdk1.8.0_144

#### mapred-site.xml ####

[root@hadoop102 hadoop]# vim mapred-site.xml

<property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>hadoop102:10020</value>
    </property>
    
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>hadoop102:19888</value>
    </property>

## 分发配置 ##

[root@hadoop102 hadoop]$ xsync /opt/module/hadoop-2.7.2/

[hadoop-env.sh]: http://hadoop-env.sh
[yarn-env.sh]: http://yarn-env.sh
[mapred-env.sh]: http://mapred-env.sh