Docker stack一键搭建Hadoop HDFS集群

刺骨的言语ヽ痛彻心扉 2023-03-03 05:54 2阅读 0赞

### 文章目录 ###

*   *  一、创建网络、节点自定义标签
     *   *  创建swarm集群网络sg-hadoop
         *  为节点自定义标签
     *  二、编写yml与环境变量env
     *   *  编写yml文件
         *  编写hadoop.env环境变量
     *  三、部署启动与查看
     *   *  docker stack部署启动
         *  查看服务状态，一秒一次，启动之后可通过IP：端口访问界面
         *  查看服务所在node节点
         *  停止删除hadoop服务

## 一、创建网络、节点自定义标签 ##

### 创建swarm集群网络sg-hadoop ###

docker network create --driver overlay --attachable --subnet 10.11.0.0/24 sg-hadoop

### 为节点自定义标签 ###

docker node update --label-add 标签key=标签value 节点名称

docker node update --label-add hadoop-datanode=datanode sangang

## 二、编写yml与环境变量env ##

### 编写yml文件 ###

version: '3'
    
    services:
      namenode:
        image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
        networks:
          - hbase
        ports:
          - 9870:9870
          - 9000:9000
        volumes:
          - namenode:/hadoop/dfs/name
        environment:
          - CLUSTER_NAME=test
        env_file:
          - ./hadoop.env
        deploy:
          mode: replicated
          replicas: 1
          restart_policy:
            condition: on-failure
          placement:
            constraints:
              - node.hostname == sangang
          labels:
            traefik.docker.network: hbase
            traefik.port: 9870
    
      datanode:
        image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
        networks:
          - hbase
        ports:
          - 9864:9864
        volumes:
          - datanode:/hadoop/dfs/data
        env_file:
          - ./hadoop.env
        environment:
          SERVICE_PRECONDITION: "namenode:9870"
        deploy:
          mode: global
          restart_policy:
            condition: on-failure
          placement:
            constraints:
              - node.labels.hadoop-datanode==datanode
          labels:
            traefik.docker.network: hbase
            traefik.port: 9864
    
      resourcemanager:
        image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
        networks:
          - hbase
        ports:
          - 8088:8088
        environment:
          SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
        env_file:
          - ./hadoop.env
        deploy:
          mode: replicated
          replicas: 1
          restart_policy:
            condition: on-failure
          placement:
            constraints:
              - node.hostname == sangang
          labels:
            traefik.docker.network: hbase
            traefik.port: 8088
        healthcheck:
          disable: true
    
      nodemanager:
        image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
        networks:
          - hbase
        environment:
          SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        env_file:
          - ./hadoop.env
        deploy:
          mode: global
          restart_policy:
            condition: on-failure
            max_attempts: 3
          placement:
            constraints:
              - node.labels.hadoop-datanode==datanode
          labels:
            traefik.docker.network: hbase
            traefik.port: 8042
            
      historyserver:
        image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
        networks:
          - hbase
        ports:
          - 8188:8188
        volumes:
          - historyserver:/hadoop/yarn/timeline
        environment:
          SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        env_file:
          - ./hadoop.env
        deploy:
          mode: replicated
          replicas: 1
          placement:
            constraints:
              - node.hostname == sangang
          labels:
            traefik.docker.network: hbase
            traefik.port: 8188
    
    volumes:
      datanode:
      namenode:
      historyserver:
    
    networks:
      hbase:
        external:
          name: sg-hadoop

### 编写hadoop.env环境变量 ###

CORE_CONF_fs_defaultFS=hdfs://namenode:9000
    CORE_CONF_hadoop_http_staticuser_user=root
    CORE_CONF_hadoop_proxyuser_hue_hosts=*
    CORE_CONF_hadoop_proxyuser_hue_groups=*
    CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
    
    HDFS_CONF_dfs_webhdfs_enabled=true
    HDFS_CONF_dfs_permissions_enabled=false
    HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
    
    YARN_CONF_yarn_log___aggregation___enable=true
    YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
    YARN_CONF_yarn_resourcemanager_recovery_enabled=true
    YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
    YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
    YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192
    YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4
    YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
    YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
    YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
    YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
    YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
    YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
    YARN_CONF_yarn_timeline___service_enabled=true
    YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
    YARN_CONF_yarn_timeline___service_hostname=historyserver
    YARN_CONF_mapreduce_map_output_compress=true
    YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec
    YARN_CONF_yarn_nodemanager_resource_memory___mb=16384
    YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8
    YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5
    YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
    YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle
    
    MAPRED_CONF_mapreduce_framework_name=yarn
    MAPRED_CONF_mapred_child_java_opts=-Xmx4096m
    MAPRED_CONF_mapreduce_map_memory_mb=4096
    MAPRED_CONF_mapreduce_reduce_memory_mb=8192
    MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
    MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
    MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
    MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
    MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/

## 三、部署启动与查看 ##

### docker stack部署启动 ###

docker stack deploy -c docker-compose.yml hadoop

### 查看服务状态，一秒一次，启动之后可通过IP：端口访问界面 ###

watch -n 1 docker stack services hadoop

### 查看服务所在node节点 ###

docker stack ps hadoop

### 停止删除hadoop服务 ###

docker stack rm hadoop