修炼k8s+flink+hdfs+dlink（一：安装hdfs）

小灰灰 2024-02-29 07:08 20阅读 0赞

## 一：安装jdk，并配置环境变量。 ##

在对应的所有的节点上进行安装。

mkdir /opt/app/java
    cd /opt/app/java
    wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%
    2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz"
    tar -zxvf jdk-8u141-linux-x64.tar.gz
    mv jdk1.8.0_141/ jdk
    cd jdk/bin
    ./java -version

![在这里插入图片描述][9701c7a036714fcc89c65cc82252e262.png]

vim /etc/profile
    export JAVA_HOME=/opt/app/java/jdk
    export CLASSPATH=.:$JAVA_HOME/lib
    export PATH=.:$JAVA_HOME/bin:$JAVA_HOME/lib:$PATH
    source /etc/profile

## 二：安装zookeeper。 ##

https://blog.csdn.net/weixin\_43446246/article/details/123327143

## 三：安装hadoop。 ##

下载安装包

mkdir /opt/app/hadoop
    cd /opt/app/hadoop
    wget https://archive.apache.org/dist/hadoop/common/hadoop-3.0.0/hadoop-3.1.2.tar.gz
    tar -zxvf hadoop-3.1.2.tar.gz

所有创建文件夹

mkdir /data/src/hadoop-3.1.2/ect/hadoop

修改配置文件

cd /opt/app/hadoop/hadoop-3.1.2/etc/hadoop/

vim hadoop-env.sh

export  JAVA_HOME=/opt/app/java/jdk
    export  HADOOP_CONF=/data/src/hadoop-3.1.2/ect/hadoop

![在这里插入图片描述][1ec4859228cb466fb6d7b3479fa05248.png]

vim core-site.xml

<property>
      <name>hadoop.tmp.dir</name>
      <value>/data/hadoop-data/hdfs/tmp</value>
    </property>
    <property>
      <name>fs.defaultFS</name>
      <value>hdfs://cluster</value>
    </property>
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>node01:2181,node02:2181,node03:2181</value>
     </property>

vim hdfs-site.xml

<property>
       <name>dfs.nameservices</name>
       <value>cluster</value>
    </property>
    
    
    <property>
        <name>dfs.ha.namenodes.cluster</name>
        <value>nn1,nn2</value>
    </property>
    
    <property>
       <name>dfs.namenode.rpc-address.cluster.nn1</name>
       <value>node01:8020</value>
    </property>
    
    <property>
       <name>dfs.namenode.rpc-address.cluster.nn2</name>
       <value>node02:8020</value>
    </property>
    
    <property>
       <name>dfs.namenode.http-address.cluster.nn1</name>
       <value>node01:9870</value>
    </property>
    
    <property>
       <name>dfs.namenode.http-address.cluster.nn2</name>
       <value>node02:9870</value>
    </property>
    
    <property>
      <name>dfs.namenode.shared.edits.dir</name>
    <value>qjournal://node01:8485;node02:8485;node03:8485/cluster</value>
    </property>
    
    <property>
       <name>dfs.ha.fencing.methods</name>
       <value>sshfence
              shell(/bin/true)
       </value>
    </property>
    
    <property>
       <name>dfs.journalnode.edits.dir</name>
       <value>/data/hadoop-data/jn</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/dfs/nn</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/dfs/dn</value>
    </property>
    <property>
        <name>dfs.datanode.max.locked.memory</name>
        <value>65536</value>
    </property>
    <property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>node01:2181,node02:2181,node03:2181</value>
     </property>
    <property>
       <name>dfs.client.failover.proxy.provider.cluster</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <property>
        <name>dfs.ha.automatic-failover.enabled</name>
        <value>true</value>
    </property>

vim workers

node01
    node02
    node03

copy到其他节点。

scp -r /opt/app/hadoop/hadoop-3.1.2 root@node02:/opt/app/hadoop/hadoop-3.1.2
    scp -r /opt/app/hadoop/hadoop-3.1.2 root@node03:/opt/app/hadoop/hadoop-3.1.2

所有机器全部配置环境变量

vim /etc/profile
    #hadoop
    export HADOOP_HOME=/opt/app/hadoop/hadoop-3.1.2
    export PATH=${PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin
    
    source /etc/profile

## 四：启动hadoop ##

1.  启动JournalNode 所有机器执行命令。

hdfs --daemon start journalnode

![在这里插入图片描述][9178660696a84750be56aa8767b2c87b.png]

1.  格式化NameNode

node01>> hdfs namenode -format
    node01>> hdfs --daemon start namenode
    node02>> hdfs namenode -bootstrapStandby
    node01>> hdfs zkfc -formatZK
    node01>> start-dfs.sh

访问地址  
http://node01:9870  
http://node02:9870  
![在这里插入图片描述][e1f299b3cffe4f47af1630075f4f0a02.png]

[9701c7a036714fcc89c65cc82252e262.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/29/a9348d5dcc46477f83ccc0aebf587e61.png
[1ec4859228cb466fb6d7b3479fa05248.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/29/92b7379a1d7140c0a5daccee99f1a387.png
[9178660696a84750be56aa8767b2c87b.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/29/b2d1e164de84479fbdc74ecf0ca8f925.png
[e1f299b3cffe4f47af1630075f4f0a02.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/29/1943a072df734e5b9b53526bf25d87e9.png