hadoop常用信息（脚本、端口号、配置文件）

骑猪看日落 2022-11-08 14:21 169阅读 0赞

### 文章目录 ###

*   *  脚本
     *   *  节点之间同步文件
         *  启动集群的hdfs 、yarn、历史记录服务器
         *  看所有节点上的java进程
     *  端口号
     *  配置文件

## 脚本 ##

### 节点之间同步文件 ###

#!/bin/bash
    
    
    
    if [ $# -lt 1 ]
    then 
        echo not enough argument!
        exit;
    fi
    
    for host in hadoop1 hadoop2 hadoop3
    do 
        echo  =======================$host=====================
        
        for file in $@
        do
           if [ -e $file ]
              then
    	   	pdir=$(cd -P $(dirname $file); pwd)
    		fname=$(basename $file)
    		ssh $host "mkdir -p $pdir"
    		rsync -av $pdir/$fname $host:$pdir
    	  else
    		echo $file does not exist!
           fi
        done
    done

### 启动集群的hdfs 、yarn、历史记录服务器 ###

#!/bin/bash
    
    if [ $# -lt 1 ]
        then echo "not enought arguments"
        exit;
    fi
    
    
    case $1 in
    "start")
    	echo "=====================start hadoop================="
    	echo "=====================start HDFS==================="
    	ssh hadoop1 "/opt/module/hadoop-3.1.4/sbin/start-dfs.sh"
    	echo "=====================start yarn==================="
    	ssh hadoop2 "/opt/module/hadoop-3.1.4/sbin/start-yarn.sh"
    	echo "=====================start history server=============="
    	ssh hadoop2 "/opt/module/hadoop-3.1.4/bin/mapred --daemon start historyserver"
    ;;
    
    "stop")
    	echo "====================stop hadoop====================="
    	echo "====================stop history server================="
    	ssh hadoop2 "/opt/module/hadoop-3.1.4/bin/mapred --daemon stop historyserver"
    	echo "====================stop yarn==========================="
    	ssh hadoop2 "/opt/module/hadoop-3.1.4/sbin/stop-yarn.sh"
    	echo "====================stop HDFS============================"
    	ssh hadoop1 "/opt/module/hadoop-3.1.4/sbin/stop-dfs.sh"
    ;;
    *)
    	echo "wrong input arguments"
    ;;
    esac

### 看所有节点上的java进程 ###

#!/bin/bash
    
    for host in hadoop1 hadoop2 hadoop3
    do
    	echo "================$host=================="
    	ssh $host "jps"
    done

--------------------

## 端口号 ##

1.  hadoop3：  
    HDFS内部通信常用端口：8020/9000/9820  
    HDFS对用户的web端口：9870  
    Yarn查看任务的web端口：8088  
    历史服务器的web端口：19888

1.  hadoop2：  
    HDFS内部通信常用端口：8020/9000  
    HDFS对用户的web端口：50070  
    Yarn查看任务的web端口：8088  
    历史服务器的web端口：19888

--------------------

## 配置文件 ##

<table> 
 <thead> 
  <tr> 
   <th>版本</th> 
   <th>常用配置文件</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>hadoop3</td> 
   <td>core-site.xml <br> hdfs-site.xml<br> yarn-site.xml<br> mapred-site.xml<br> workers</td> 
  </tr> 
  <tr> 
   <td>hadoop2</td> 
   <td>core-site.xml <br> hdfs-site.xml<br> yarn-site.xml<br> mapred-site.xml<br> slaves</td> 
  </tr> 
 </tbody> 
</table>

--------------------