[kubernetes]-二进制安装k8s1.18

旧城等待， 2022-12-31 01:25 210阅读 0赞

导语：第一次安装记录一下大致步骤 方便自己后面操作。

### 服务器信息 ###

<table> 
 <thead> 
  <tr> 
   <th>角色</th> 
   <th>IP</th> 
   <th>组件</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>master</td> 
   <td>192.168.1.120</td> 
   <td>kube-apiserver，kube-controller-manager，kube-scheduler，docker，etcd</td> 
  </tr> 
  <tr> 
   <td>node</td> 
   <td>192.168.1.121</td> 
   <td>kube-apiserver，kube-controller-manager，kube-scheduler，docker，etcd</td> 
  </tr> 
  <tr> 
   <td>node</td> 
   <td>192.168.1.122</td> 
   <td>kubelet，kube-proxy，docker，etcd</td> 
  </tr> 
 </tbody> 
</table>

##### 需要的部分安装包 #####

链接: https://pan.baidu.com/s/1VwU1eJl1qfvFMFRTOPUNAg 密码: ubvw

### 初始化环境 ###

##### 初始化脚本init.sh #####

# 初始化
    init_security() {
    systemctl stop firewalld
    systemctl disable firewalld &>/dev/null
    setenforce 0
    sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
    sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
    sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
    systemctl enable sshd crond &> /dev/null
    rpm -e postfix --nodeps
    echo -e "\033[32m [安全配置] ==> OK \033[0m"
    }
    init_security
    
    init_yumsource() {
    if [ ! -d /etc/yum.repos.d/backup ];then
        mkdir /etc/yum.repos.d/backup
    fi
    mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
    if ! ping -c2 www.baidu.com &>/dev/null    
    then
        echo "您无法上外网，不能配置yum源"
        exit    
    fi
        curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
        curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
        yum clean all
        timedatectl set-timezone Asia/Shanghai
        echo "nameserver 114.114.114.114" > /etc/resolv.conf
        echo "nameserver 8.8.8.8" >> /etc/resolv.conf
        chattr +i /etc/resolv.conf
        yum -y install ntpdate
        ntpdate -b  ntp1.aliyun.com        # 对时很重要
        echo -e "\033[32m [YUM　Source] ==> OK \033[0m"
    }
    init_yumsource
    
    # 关掉swap分区
    swapoff -a
    # 如果想永久关掉swap分区，打开如下文件注释掉swap哪一行即可.
    sed -i '/ swap / s/^$.*$$/#\1/g' /etc/fstab #永久
    
    
    # 配置主机名解析
    cat  > /etc/hosts <<EOF
    127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
    ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
    192.168.1.120 op-jmx-m01
    192.168.1.121 op-jmx-n01
    192.168.1.122 op-jmx-n02
    EOF
    
    tail -3 /etc/hosts
    
    
    # 将桥接的IPv4流量传递到iptables的链
    cat > /etc/sysctl.d/k8s.conf << EOF
    net.bridge.bridge-nf-call-ip6tables = 1
    net.bridge.bridge-nf-call-iptables = 1
    EOF
    sysctl --system  # 生效
    
    
    # 升级内核(非必须,只是性能更好)
    wget https://cbs.centos.org/kojifiles/packages/kernel/4.9.220/37.el7/x86_64/kernel-4.9.220-37.el7.x86_64.rpm
    rpm -ivh kernel-4.9.220-37.el7.x86_64.rpm
    
    reboot

##### 运行初始化脚本 #####

sh init.sh

### 部署etcd集群 ###

> 为了节省机器，这里与K8s节点机器复用。也可以独立于k8s集群之外部署，只要apiserver能连接到就行。

##### 准备cfssl证书生成工具 #####

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
    wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
    wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
    chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
    mv cfssl_linux-amd64 /usr/local/bin/cfssl
    mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
    mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

##### 创建工作目录 #####

mkdir -p ~/TLS/{etcd,k8s}
    cd TLS/etcd

##### 自签CA #####

# 10年证书
    cat > ca-config.json << EOF
    {
      "signing": {
        "default": {
          "expiry": "87600h"
        },
        "profiles": {
          "www": {
             "expiry": "87600h",
             "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ]
          }
        }
      }
    }
    EOF
    
    cat > ca-csr.json << EOF
    {
        "CN": "etcd CA",
        "key": {
            "algo": "rsa",
            "size": 2048
        },
        "names": [
            {
                "C": "CN",
                "L": "Beijing",
                "ST": "Beijing"
            }
        ]
    }
    EOF

##### 生成证书 #####

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
    
    ls *pem

##### 使用自签CA签发etcd https证书 #####

cat > server-csr.json <<EOF
    {
        "CN": "kubernetes",
        "hosts": [
          "192.168.1.120",
          "192.168.1.121",
          "192.168.1.122"
        ],
        "key": {
            "algo": "rsa",
            "size": 2048
        },
        "names": [
            {
                "C": "CN",
                "L": "BeiJing",
                "ST": "BeiJing",
                "O": "k8s",
                "OU": "System"
            }
        ]
    }
    EOF

> 上述文件hosts字段中IP为所有etcd节点的集群内部通信IP，一个都不能少！为了方便后期扩容可以多写几个预留的IP。

##### 生成证书 #####

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
    
    ls server*pem

##### 下载etcd二进制文件 #####

wget https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz

##### 创建工作目录并解压二进制包 #####

mkdir /opt/etcd/{bin,cfg,ssl} -p
    tar zxvf etcd-v3.4.9-linux-amd64.tar.gz
    mv etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/
    
    # 配置etcd
    cat > /opt/etcd/cfg/etcd.conf << EOF
    #[Member]
    ETCD_NAME="etcd-1"
    ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
    ETCD_LISTEN_PEER_URLS="https://192.168.1.120:2380"
    ETCD_LISTEN_CLIENT_URLS="https://192.168.1.120:2379"
    #[Clustering]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.120:2380"
    ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.120:2379"
    ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.120:2380,etcd-2=https://192.168.1.121:2380,etcd-3=https://192.168.1.122:2380"
    ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
    ETCD_INITIAL_CLUSTER_STATE="new"
    EOF
    
    # ETCD_NAME：节点名称，集群中唯一
    # ETCD_DATA_DIR：数据目录
    # ETCD_LISTEN_PEER_URLS：集群通信监听地址
    # ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
    # ETCD_INITIAL_ADVERTISE_PEER_URLS：集群通告地址
    # ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址
    # ETCD_INITIAL_CLUSTER：集群节点地址
    # ETCD_INITIAL_CLUSTER_TOKEN：集群Token
    # ETCD_INITIAL_CLUSTER_STATE：加入集群的当前状态，new是新集群，existing表示加入已有集群

##### systemd管理etcd #####

cat > /usr/lib/systemd/system/etcd.service << EOF
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    [Service]
    Type=notify
    EnvironmentFile=/opt/etcd/cfg/etcd.conf
    ExecStart=/opt/etcd/bin/etcd \
    --cert-file=/opt/etcd/ssl/server.pem \
    --key-file=/opt/etcd/ssl/server-key.pem \
    --peer-cert-file=/opt/etcd/ssl/server.pem \
    --peer-key-file=/opt/etcd/ssl/server-key.pem \
    --trusted-ca-file=/opt/etcd/ssl/ca.pem \
    --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
    --logger=zap
    Restart=on-failure
    LimitNOFILE=65536
    [Install]
    WantedBy=multi-user.target
    EOF
    
    
    systemctl daemon-reload
    # 这里start可能会fail 启动其余2个etcd就可以了 如果报错timeout 估计配置出错了
    systemctl start etcd
    systemctl enable etcd

##### 拷贝上面生成的证书及文件到n01和n02 #####

cp ~/TLS/etcd/ca*pem ~/TLS/etcd/server*pem /opt/etcd/ssl/
    # 这里可以做下免密
    # ssh-copy-id op-jmx-n01;ssh-copy-id op-jmx-n02
    
    scp -r /opt/etcd/ op-jmx-n01:/opt/
    scp -r /opt/etcd/ op-jmx-n02:/opt/
    scp /usr/lib/systemd/system/etcd.service op-jmx-n01:/usr/lib/systemd/system/
    scp /usr/lib/systemd/system/etcd.service op-jmx-n02:/usr/lib/systemd/system/

##### 修改n01和n02的etcd.conf配置文件如下 #####

# n01
    #[Member]
    ETCD_NAME="etcd-2"
    ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
    ETCD_LISTEN_PEER_URLS="https://192.168.1.121:2380"
    ETCD_LISTEN_CLIENT_URLS="https://192.168.1.121:2379"
    #[Clustering]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.121:2380"
    ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.121:2379"
    ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.120:2380,etcd-2=https://192.168.1.121:2380,etcd-3=https://192.168.1.122:2380"
    ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
    ETCD_INITIAL_CLUSTER_STATE="new"
    
    # n02
    #[Member]
    ETCD_NAME="etcd-3"
    ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
    ETCD_LISTEN_PEER_URLS="https://192.168.1.122:2380"
    ETCD_LISTEN_CLIENT_URLS="https://192.168.1.122:2379"
    #[Clustering]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.122:2380"
    ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.122:2379"
    ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.120:2380,etcd-2=https://192.168.1.121:2380,etcd-3=https://192.168.1.122:2380"
    ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
    ETCD_INITIAL_CLUSTER_STATE="new"

# 启动服务并设置开机自启
    systemctl daemon-reload
    systemctl start etcd
    systemctl enable etcd

##### 验证etcd集群状态 #####

/opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.120:2379,https://192.168.1.121:2379,https://192.168.1.122:2379" endpoint health

![3a882e6f05d5f16ffcb10e1001f2c159.png][]

### 安装docker ###

yum -y install yum-utils
    yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
    yum -y install docker-ce-19.03.9-3.el7
    systemctl daemon-reload
    systemctl start docker
    systemctl enable docker

### 部署master node ###

生成kube-apiserver证书

cd /root/TLS/k8s
    
    cat > ca-config.json << EOF
    {
      "signing": {
        "default": {
          "expiry": "87600h"
        },
        "profiles": {
          "kubernetes": {
             "expiry": "87600h",
             "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ]
          }
        }
      }
    }
    EOF
    cat > ca-csr.json << EOF
    {
        "CN": "kubernetes",
        "key": {
            "algo": "rsa",
            "size": 2048
        },
        "names": [
            {
                "C": "CN",
                "L": "Beijing",
                "ST": "Beijing",
                "O": "k8s",
                "OU": "System"
            }
        ]
    }
    EOF
    
    # 生成证书
    cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
    
    ls *pem

##### 使用自签CA签发kube-apiserver https证书 #####

cat > /root/TLS/k8s/server-csr.json  <<EOF
    {
        "CN": "kubernetes",
        "hosts": [
          "10.0.0.1",
          "127.0.0.1",
          "192.168.1.120",
          "192.168.1.121",
          "192.168.1.122",
          "kubernetes",
          "kubernetes.default",
          "kubernetes.default.svc",
          "kubernetes.default.svc.cluster",
          "kubernetes.default.svc.cluster.local"
        ],
        "key": {
            "algo": "rsa",
            "size": 2048
        },
        "names": [
            {
                "C": "CN",
                "L": "BeiJing",
                "ST": "BeiJing",
                "O": "k8s",
                "OU": "System"
            }
        ]
    }
    EOF
    
    # 上述文件hosts字段中IP为所有Master/LB/VIP IP，一个都不能少！为了方便后期扩容可以多写几个预留的IP。
    
    # 生成证书
    cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
    
    ls server*pem

##### 下载解压二进制包 #####

wget https://dl.k8s.io/v1.18.4/kubernetes-server-linux-amd64.tar.gz
    mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs} 
    tar zxvf kubernetes-server-linux-amd64.tar.gz
    cd kubernetes/server/bin
    cp kube-apiserver kube-scheduler kube-controller-manager /opt/kubernetes/bin
    cp kubectl /usr/bin/

### 部署kube-apiserver ###

cat > /opt/kubernetes/cfg/kube-apiserver.conf  <<EOF
    KUBE_APISERVER_OPTS="--logtostderr=false \
    --v=2 \
    --log-dir=/opt/kubernetes/logs \
    --etcd-servers=https://192.168.1.120:2379,https://192.168.1.121:2379,https://192.168.1.122:2379 \
    --bind-address=192.168.1.120 \
    --secure-port=6443 \
    --advertise-address=192.168.1.120 \
    --allow-privileged=true \
    --service-cluster-ip-range=10.0.0.0/24 \
    --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
    --authorization-mode=RBAC,Node \
    --enable-bootstrap-token-auth=true \
    --token-auth-file=/opt/kubernetes/cfg/token.csv \
    --service-node-port-range=30000-32767 \
    --kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \
    --kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \
    --tls-cert-file=/opt/kubernetes/ssl/server.pem  \
    --tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
    --client-ca-file=/opt/kubernetes/ssl/ca.pem \
    --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
    --etcd-cafile=/opt/etcd/ssl/ca.pem \
    --etcd-certfile=/opt/etcd/ssl/server.pem \
    --etcd-keyfile=/opt/etcd/ssl/server-key.pem \
    --audit-log-maxage=30 \
    --audit-log-maxbackup=3 \
    --audit-log-maxsize=100 \
    --audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
    EOF
    
    # –logtostderr：启用日志
    # —v：日志等级
    # –log-dir：日志目录
    # –etcd-servers：etcd集群地址
    # –bind-address：监听地址
    # –secure-port：https安全端口
    # –advertise-address：集群通告地址
    # –allow-privileged：启用授权
    # –service-cluster-ip-range：Service虚拟IP地址段
    # –enable-admission-plugins：准入控制模块
    # –authorization-mode：认证授权，启用RBAC授权和节点自管理
    # –enable-bootstrap-token-auth：启用TLS bootstrap机制
    # –token-auth-file：bootstrap token文件
    # –service-node-port-range：Service nodeport类型默认分配端口范围
    # –kubelet-client-xxx：apiserver访问kubelet客户端证书
    # –tls-xxx-file：apiserver https证书
    # –etcd-xxxfile：连接Etcd集群证书
    # –audit-log-xxx：审计日志

##### 拷贝刚刚生成的证书 #####

cp ~/TLS/k8s/ca*pem ~/TLS/k8s/server*pem /opt/kubernetes/ssl/

##### 启用TLS Bootstrapping机制 #####

TLS Bootstraping：Master apiserver启用TLS认证后，Node节点kubelet和kube-proxy要与kube-apiserver进行通信，必须使用CA签发的有效证书才可以，当Node节点很多时，这种客户端证书颁发需要大量工作，同样也会增加集群扩展复杂度。为了简化流程，Kubernetes引入了TLS bootstraping机制来自动颁发客户端证书，kubelet会以一个低权限用户自动向apiserver申请证书，kubelet的证书由apiserver动态签署。所以强烈建议在Node上使用这种方式，目前主要用于kubelet，kube-proxy还是由我们统一颁发一个证书

![08658211c7600c977815553a82d35761.png][]

##### 生成随机token #####

head -c 16 /dev/urandom | od -An -t x | tr -d ' '
    # 177a34de29f27865f81274b4bd75c615

##### 创建上述配置文件中的token文件 #####

cat > /opt/kubernetes/cfg/token.csv << EOF
    177a34de29f27865f81274b4bd75c615,kubelet-bootstrap,10001,"system:node-bootstrapper"
    EOF
    
    # 格式：token，用户名，UID，用户组

##### systemd管理apiserver #####

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
    [Unit]
    Description=Kubernetes API Server
    Documentation=https://github.com/kubernetes/kubernetes
    [Service]
    EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf
    ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
    Restart=on-failure
    [Install]
    WantedBy=multi-user.target
    EOF

##### 启动设置开机启动 #####

systemctl daemon-reload
    systemctl start kube-apiserver
    systemctl enable kube-apiserver

##### 授权kubelet-bootstrap用户允许请求证书 #####

kubectl create clusterrolebinding kubelet-bootstrap \
    --clusterrole=system:node-bootstrapper \
    --user=kubelet-bootstrap

### 部署kube-controller-manager ###

##### 创建配置文件 #####

cat > /opt/kubernetes/cfg/kube-controller-manager.conf <<EOF
    KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \
    --v=2 \
    --log-dir=/opt/kubernetes/logs \
    --leader-elect=true \
    --master=127.0.0.1:8080 \
    --bind-address=127.0.0.1 \
    --allocate-node-cidrs=true \
    --cluster-cidr=10.244.0.0/16 \
    --service-cluster-ip-range=10.0.0.0/24 \
    --cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \
    --cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem  \
    --root-ca-file=/opt/kubernetes/ssl/ca.pem \
    --service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \
    --experimental-cluster-signing-duration=87600h0m0s"
    EOF
    
    # –master：通过本地非安全本地端口8080连接apiserver。
    # –leader-elect：当该组件启动多个时，自动选举（HA）
    # –cluster-signing-cert-file/–cluster-signing-key-file：自动为kubelet颁发证书的CA，与apiserver保持一致

##### systemd管理controller-manager #####

cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
    [Unit]
    Description=Kubernetes Controller Manager
    Documentation=https://github.com/kubernetes/kubernetes
    [Service]
    EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf
    ExecStart=/opt/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
    Restart=on-failure
    [Install]
    WantedBy=multi-user.target
    EOF

##### 启动设置开机启动 #####

systemctl daemon-reload
    systemctl start kube-controller-manager
    systemctl enable kube-controller-manager

### 部署kube-scheduler ###

cat >/opt/kubernetes/cfg/kube-scheduler.conf <<EOF
    KUBE_SCHEDULER_OPTS="--logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect --master=127.0.0.1:8080 --bind-address=127.0.0.1"
    EOF
    
    # –master：通过本地非安全本地端口8080连接apiserver。
    # –leader-elect：当该组件启动多个时，自动选举（HA）

##### systemd管理scheduler #####

cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
    [Unit]
    Description=Kubernetes Scheduler
    Documentation=https://github.com/kubernetes/kubernetes
    [Service]
    EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf
    ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
    Restart=on-failure
    [Install]
    WantedBy=multi-user.target
    EOF

##### 启动并设置开机自启动 #####

systemctl daemon-reload
    systemctl restart kube-scheduler
    systemctl enable kube-scheduler

##### 查看集群状态 #####

# 所有组件都已经启动成功，通过kubectl工具查看当前集群组件状态：
    kubectl get cs

![32b7aad034a0aa970f06717f66750ed3.png][]

### 部署worker node 在n01和n02上操作 ###

mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}

##### 从master上拷贝 #####

cd kubernetes/server/bin
    cp kubelet kube-proxy /opt/kubernetes/bin   # 本地拷贝,注释这里操作还是master节点,

### 部署kubelet ###

##### 创建配置文件 #####

cat > /opt/kubernetes/cfg/kubelet.conf <<EOF
    KUBELET_OPTS="--logtostderr=false \
    --v=2 \
    --log-dir=/opt/kubernetes/logs \
    --hostname-override=master \
    --network-plugin=cni \
    --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
    --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
    --config=/opt/kubernetes/cfg/kubelet-config.yml \
    --cert-dir=/opt/kubernetes/ssl \
    --pod-infra-container-image=lizhenliang/pause-amd64:3.0"
    EOF
    
    # –hostname-override：显示名称，集群中唯一
    # –network-plugin：启用CNI
    # –kubeconfig：空路径，会自动生成，后面用于连接apiserver
    # –bootstrap-kubeconfig：首次启动向apiserver申请证书
    # –config：配置参数文件
    # –cert-dir：kubelet证书生成目录
    # –pod-infra-container-image：管理Pod网络容器的镜像

##### 配置参数文件 #####

cat > /opt/kubernetes/cfg/kubelet-config.yml << EOF
    kind: KubeletConfiguration
    apiVersion: kubelet.config.k8s.io/v1beta1
    address: 0.0.0.0
    port: 10250
    readOnlyPort: 10255
    cgroupDriver: cgroupfs
    clusterDNS:
    - 10.0.0.2
    clusterDomain: cluster.local 
    failSwapOn: false
    authentication:
      anonymous:
        enabled: false
      webhook:
        cacheTTL: 2m0s
        enabled: true
      x509:
        clientCAFile: /opt/kubernetes/ssl/ca.pem 
    authorization:
      mode: Webhook
      webhook:
        cacheAuthorizedTTL: 5m0s
        cacheUnauthorizedTTL: 30s
    evictionHard:
      imagefs.available: 15%
      memory.available: 100Mi
      nodefs.available: 10%
      nodefs.inodesFree: 5%
    maxOpenFiles: 1000000
    maxPods: 110
    EOF

##### 生成bootstrap.kubeconfig文件 #####

KUBE_APISERVER="https://192.168.1.120:6443" # apiserver IP:PORT
    TOKEN="177a34de29f27865f81274b4bd75c615" # 与token.csv里保持一致
    # 生成 kubelet bootstrap kubeconfig 配置文件
    kubectl config set-cluster kubernetes \
      --certificate-authority=/opt/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=${KUBE_APISERVER} \
      --kubeconfig=bootstrap.kubeconfig
      
      
    kubectl config set-credentials "kubelet-bootstrap" \
      --token=${TOKEN} \
      --kubeconfig=bootstrap.kubeconfig
      
      
    kubectl config set-context default \
      --cluster=kubernetes \
      --user="kubelet-bootstrap" \
      --kubeconfig=bootstrap.kubeconfig
      
      
    kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
    
    # 拷贝到配置文件路径
    cp bootstrap.kubeconfig /opt/kubernetes/cfg

##### systemd 管理kubelet #####

cat > /usr/lib/systemd/system/kubelet.service << EOF
    [Unit]
    Description=Kubernetes Kubelet
    After=docker.service
    [Service]
    EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
    ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
    Restart=on-failure
    LimitNOFILE=65536
    [Install]
    WantedBy=multi-user.target
    EOF

##### 启动并设置开机启动 #####

systemctl daemon-reload
    systemctl start kubelet
    systemctl enable kubelet

##### 批准kubelet证书并加入集群 #####

# 查看kubelet证书请求
    kubectl get csr
    
    
    # 批准申请
    kubectl certificate approve 
    
    # 查看节点 这里没有安装CNI会显示not ready
    kubectl get node
    
    
    # 由于网络插件还没有部署，节点会没有准备就绪 NotReady

![8eef58e1941a41d9726f20f2d78fd0be.png][]

如果get csr提示No resources found. 一般是master端接收不到node申请加入kubernetes的请求信息。可以查看/opt/kubernetes/logs下的日志排查

![7b7bc07929533c06129bd2170429ae96.png][]

### 部署CNI网络 ###

wget https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz
      
    # 解压二进制文件并移动到默认工作目录
    mkdir -p /opt/cni/bin
    tar zxvf cni-plugins-linux-amd64-v0.8.6.tgz -C /opt/cni/bin
    
    # 部署cni网络
    wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
    sed -i -r "s#quay.io/coreos/flannel:.*-amd64#lizhenliang/flannel:v0.12.0-amd64#g" kube-flannel.yml
    
    # 默认镜像地址无法访问，修改为docker hub镜像仓库。
    kubectl apply -f kube-flannel.yml
    
    kubectl get pods -n kube-system
    
    
    kubectl get node
    
    # 部署好网络插件,Node准备就绪

### 部署kube-proxy ###

##### 创建配置文件 #####

cat > /opt/kubernetes/cfg/kube-proxy.conf << EOF
    KUBE_PROXY_OPTS="--logtostderr=false \\
    --v=2 \\
    --log-dir=/opt/kubernetes/logs \\
    --config=/opt/kubernetes/cfg/kube-proxy-config.yml"
    EOF

##### 创建参数文件 #####

cat > /opt/kubernetes/cfg/kube-proxy-config.yml << EOF
    kind: KubeProxyConfiguration
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    bindAddress: 0.0.0.0
    metricsBindAddress: 0.0.0.0:10249
    clientConnection:
      kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
    hostnameOverride: k8s-master
    clusterCIDR: 10.0.0.0/24
    EOF

##### 生成kube-proxy.kubeconfig文件 #####

# 生成kube-prox证书
    # 切换工作目录
    cd ～/TLS/k8s
    
    # 创建证书请求文件
    cat > kube-proxy-csr.json << EOF
    {
      "CN": "system:kube-proxy",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "L": "BeiJing",
          "ST": "BeiJing",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }
    EOF
    
    # 生成证书
    cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
    
    ls kube-proxy*pem

##### 生成kubeconfig文件 #####

KUBE_APISERVER="https://192.168.1.120:6443"
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/opt/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=${KUBE_APISERVER} \
      --kubeconfig=kube-proxy.kubeconfig
      
    kubectl config set-credentials kube-proxy \
      --client-certificate=./kube-proxy.pem \
      --client-key=./kube-proxy-key.pem \
      --embed-certs=true \
      --kubeconfig=kube-proxy.kubeconfig
      
    kubectl config set-context default \
      --cluster=kubernetes \
      --user=kube-proxy \
      --kubeconfig=kube-proxy.kubeconfig
      
    kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
    
    # 拷贝配置文件到指定路径
    cp kube-proxy.kubeconfig /opt/kubernetes/cfg/

##### systemd管理kube-proxy #####

cat > /usr/lib/systemd/system/kube-proxy.service << EOF
    [Unit]
    Description=Kubernetes Proxy
    After=network.target
    [Service]
    EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
    ExecStart=/opt/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
    Restart=on-failure
    LimitNOFILE=65536
    [Install]
    WantedBy=multi-user.target
    EOF

##### 启动并设置开机启动 #####

systemctl daemon-reload
    systemctl start kube-proxy
    systemctl enable kube-proxy

##### 授权apiserver访问kubelet #####

cat > apiserver-to-kubelet-rbac.yaml << EOF
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      annotations:
        rbac.authorization.kubernetes.io/autoupdate: "true"
      labels:
        kubernetes.io/bootstrapping: rbac-defaults
      name: system:kube-apiserver-to-kubelet
    rules:
      - apiGroups:
          - ""
        resources:
          - nodes/proxy
          - nodes/stats
          - nodes/log
          - nodes/spec
          - nodes/metrics
          - pods/log
        verbs:
          - "*"
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRoleBinding
    metadata:
      name: system:kube-apiserver
      namespace: ""
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: system:kube-apiserver-to-kubelet
    subjects:
      - apiGroup: rbac.authorization.k8s.io
        kind: User
        name: kubernetes
    EOF
    
    kubectl apply -f apiserver-to-kubelet-rbac.yaml

##### 新增加worker node #####

# 拷贝已部署好的node相关文件到新节点
    scp -r /opt/kubernetes/ op-jmx-n01:/opt/
    scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service op-jmx-n01:/usr/lib/systemd/system
    scp -r /opt/cni/ op-jmx-n01:/opt/
    scp /opt/kubernetes/ssl/ca.pem op-jmx-n01:/opt/kubernetes/ssl
    
    scp -r /opt/kubernetes/ op-jmx-n02:/opt/
    scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service op-jmx-n02:/usr/lib/systemd/system
    scp -r /opt/cni/ op-jmx-n02:/opt/
    scp /opt/kubernetes/ssl/ca.pem op-jmx-n02:/opt/kubernetes/ssl

##### 删除新node上的kubelet证书和kubeconfig文件 // 在op-jmx-n01 op-jmx-n02上操作 #####

rm -f /opt/kubernetes/cfg/kubelet.kubeconfig 
    rm -f /opt/kubernetes/ssl/kubelet*
    
    # 这几个文件是证书申请审批后自动生成的，每个Node不同，必须删除重新生成。

##### 修改主机名并设置开机自启动 #####

# 修改/opt/kubernetes/cfg/kubelet.conf  hostname-override: 为主机名
    sed -i  's/hostname-override=master/hostname-override=op-jmx-n01/g' /opt/kubernetes/cfg/kubelet.conf 
    
    # 修改/opt/kubernetes/cfg/kube-proxy-config.yml 里hostnameOverride: 为主机名
    sed -i  's/k8s-master/op-jmx-n01/g' /opt/kubernetes/cfg/kube-proxy-config.yml
    
    # 修改/opt/kubernetes/cfg/kubelet.conf  hostname-override: 为主机名
    sed -i  's/hostname-override=master/hostname-override=op-jmx-n02/g' /opt/kubernetes/cfg/kubelet.conf 
    
    # 修改/opt/kubernetes/cfg/kube-proxy-config.yml 里hostnameOverride: 为主机名
    sed -i  's/k8s-master/op-jmx-n02/g' /opt/kubernetes/cfg/kube-proxy-config.yml
    
    
    
    # 启动并设置开机启动
    systemctl daemon-reload
    systemctl restart kubelet
    systemctl enable kubelet
    systemctl restart kube-proxy
    systemctl enable kube-proxy

##### 在master上批准新node的Node kubelet证书申请 #####

kubectl get csr
    
    # 这里需要将原来的CNI镜像准备好
    kubectl certificate approve  node-csr-RSYsrzdjJbRL3dSSMkM63DFJ1XCiJpM2JAExI-jaLp8
    kubectl certificate approve  node-csr-iSf2PR72kePu_6IRhUd2fNg3EY67PTrWlA_Q0uPl1yw
    
    
    # 查看node状态
    kubectl get node

![c44d18ab8100b54605f3fab46e75a50c.png][]

### 部署CoreDNS ###

##### CoreDNS用于集群内部service名称解析 #####

cat > coredns.yaml<<EOF
    # __MACHINE_GENERATED_WARNING__
    
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: coredns
      namespace: kube-system
      labels:
          kubernetes.io/cluster-service: "true"
          addonmanager.kubernetes.io/mode: Reconcile
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      labels:
        kubernetes.io/bootstrapping: rbac-defaults
        addonmanager.kubernetes.io/mode: Reconcile
      name: system:coredns
    rules:
    - apiGroups:
      - ""
      resources:
      - endpoints
      - services
      - pods
      - namespaces
      verbs:
      - list
      - watch
    - apiGroups:
      - ""
      resources:
      - nodes
      verbs:
      - get
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRoleBinding
    metadata:
      annotations:
        rbac.authorization.kubernetes.io/autoupdate: "true"
      labels:
        kubernetes.io/bootstrapping: rbac-defaults
        addonmanager.kubernetes.io/mode: EnsureExists
      name: system:coredns
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: system:coredns
    subjects:
    - kind: ServiceAccount
      name: coredns
      namespace: kube-system
    ---
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: coredns
      namespace: kube-system
      labels:
          addonmanager.kubernetes.io/mode: EnsureExists
    data:
      Corefile: |
        .:53 {
            errors
            health
            kubernetes cluster.local in-addr.arpa ip6.arpa {
                pods insecure
                upstream
                fallthrough in-addr.arpa ip6.arpa
                ttl 30
            }
            prometheus :9153
            forward . /etc/resolv.conf
            cache 30
            loop
            reload
            loadbalance
        }
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: coredns
      namespace: kube-system
      labels:
        k8s-app: kube-dns
        kubernetes.io/cluster-service: "true"
        addonmanager.kubernetes.io/mode: Reconcile
        kubernetes.io/name: "CoreDNS"
    spec:
      # replicas: not specified here:
      # 1. In order to make Addon Manager do not reconcile this replicas parameter.
      # 2. Default is 1.
      # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
      replicas: 2
      strategy:
        type: RollingUpdate
        rollingUpdate:
          maxUnavailable: 1
      selector:
        matchLabels:
          k8s-app: kube-dns
      template:
        metadata:
          labels:
            k8s-app: kube-dns
          annotations:
            seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
        spec:
          priorityClassName: system-cluster-critical
          serviceAccountName: coredns
          tolerations:
            - key: "CriticalAddonsOnly"
              operator: "Exists"
          nodeSelector:
            beta.kubernetes.io/os: linux
          containers:
          - name: coredns
            image: coredns/coredns:1.5.0
            imagePullPolicy: IfNotPresent
            resources:
              limits:
                memory: 70Mi
              requests:
                cpu: 100m
                memory: 70Mi
            args: [ "-conf", "/etc/coredns/Corefile" ]
            volumeMounts:
            - name: config-volume
              mountPath: /etc/coredns
              readOnly: true
            ports:
            - containerPort: 53
              name: dns
              protocol: UDP
            - containerPort: 53
              name: dns-tcp
              protocol: TCP
            - containerPort: 9153
              name: metrics
              protocol: TCP
            livenessProbe:
              httpGet:
                path: /health
                port: 8080
                scheme: HTTP
              initialDelaySeconds: 60
              timeoutSeconds: 5
              successThreshold: 1
              failureThreshold: 5
            readinessProbe:
              httpGet:
                path: /health
                port: 8080
                scheme: HTTP
            securityContext:
              allowPrivilegeEscalation: false
              capabilities:
                add:
                - NET_BIND_SERVICE
                drop:
                - all
              readOnlyRootFilesystem: true
          dnsPolicy: Default
          volumes:
            - name: config-volume
              configMap:
                name: coredns
                items:
                - key: Corefile
                  path: Corefile
    ---
    apiVersion: v1
    kind: Service
    metadata:
      name: kube-dns
      namespace: kube-system
      annotations:
        prometheus.io/port: "9153"
        prometheus.io/scrape: "true"
      labels:
        k8s-app: kube-dns
        kubernetes.io/cluster-service: "true"
        addonmanager.kubernetes.io/mode: Reconcile
        kubernetes.io/name: "CoreDNS"
    spec:
      selector:
        k8s-app: kube-dns
        # 这个IP地址需要在kube-apiserver启动参数–service-cluster-ip-range指定的IP地址范围内 以前默认应该是 10.96.0.10
      clusterIP: 10.0.0.10
      ports:
      - name: dns
        port: 53
        protocol: UDP
      - name: dns-tcp
        p
    kubectl apply -f coredns.yaml

![f89a2017f435b183f0975aacd6e8b9df.png][]

##### DNS解析测试 #####

kubectl run -it --rm dns-test --image=busybox:1.28.4 sh
    # 进入容器后输入一下命令
    / # nslookup kubernetes

![54cf4eee574a4432b8c83e053bf5cc47.png][]

参考

https://www.cnblogs.com/you-men/p/13192086.html

[3a882e6f05d5f16ffcb10e1001f2c159.png]: /images/20221120/50a35e099a364250a63a6c6723fbb04a.png
[08658211c7600c977815553a82d35761.png]: /images/20221120/fad4038b61a44fdb8a719263003cde53.png
[32b7aad034a0aa970f06717f66750ed3.png]: /images/20221120/e99898b305f94b749aa5b330a3499213.png
[8eef58e1941a41d9726f20f2d78fd0be.png]: /images/20221120/1a3153f5391d4075bdd766592c523f5d.png
[7b7bc07929533c06129bd2170429ae96.png]: /images/20221120/ec27ba9ffc6341f9b60f86e62e905527.png
[c44d18ab8100b54605f3fab46e75a50c.png]: /images/20221120/087cc07c2ade446cb0bf801e6f2082de.png
[f89a2017f435b183f0975aacd6e8b9df.png]: /images/20221120/7d240d7f65c64961aad41c6970a7ef63.png
[54cf4eee574a4432b8c83e053bf5cc47.png]: https://img-blog.csdnimg.cn/img_convert/54cf4eee574a4432b8c83e053bf5cc47.png