二进制安装k8s(单master)

淩亂°似流年 2022-12-21 06:05 95阅读 0赞

在CentOS7上以二进制方式安装单master的k8s集群，适用于1.17.x，1.18.x，1.19.x  
本方案部署的软硬件环境如下：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center]  
kubernetes集群环境使用了3台机器用于部署，具体如下：  
![在这里插入图片描述][2020111508393249.png_pic_center]

# 一、 基础环境配置 #

### 1. 配置主机名，host文件 ###

hostnamectl --static set-hostname  master1
    hostnamectl --static set-hostname  node1
    hostnamectl --static set-hostname  node2
    cat /etc/hosts
    192.168.1.11 master
    192.168.1.12 node1
    192.168.1.13 node2

### 2. 关闭防火墙，禁用SELinux，关闭swap分区 ###

systemctl disable firewalld.service && systemctl stop firewalld.service

修改/etc/selinux/config 文件，将SELINUX=enforcing改为SELINUX=disabled

sed -i 's/^SELINUX=enforcing$/SELINUX= disabled /' /etc/selinux/config

该项设置需要重启后才能生效。

swapoff -a
    cp /etc/fstab /etc/fstab_bak
    cat /etc/fstab_bak |grep -v swap > /etc/fstab

### 3. 配置免秘钥 ###

ssh-keygen -t rsa -b 2048 -N '' -f /root/.ssh/id_rsa #一键生成秘钥

把所有节点的id\_rsa.pub追加到authorized\_keys，把最终的authorized\_keys分发到所有主机

cd /root/.ssh/  &&  cp id_rsa.pub authorized_keys	
    for i in { 11..13}; do scp authorized_keys 192.168.1.$i: /root/.ssh/ ;done

在每台节点上验证免秘钥

### 4. 配置阿里云CentOS7、docker yum源，安装基础软件包 ###

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
    sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
    yum -y install bash-completion vim-enhanced net-tools tree wget curl make cmake gcc gcc-c++  screen createrepo yum-utils device-mapper-persistent-data lvm2 git
    yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
    yum makecache fast	#查询docker可用版本
    yum list docker-ce.x86_64 --showduplicates | sort -r 
    yum -y install docker-ce-[VERSION]	#安装具体版本的docker

由于新版kubelet建议使用systemd，所以可以把docker的CgroupDriver改成systemd

mkdir -p /etc/docker && vim /etc/docker/daemon.json
    { 
    "exec-opts": ["native.cgroupdriver=systemd"],
    "registry-mirrors": ["https://registry.docker-cn.com","http://hub-mirror.c.163.com",
    "https://docker.mirrors.ustc.edu.cn", "https://lt2ws3tf.mirror.aliyuncs.com", 
    "https://mirror.baidubce.com","https://registry.docker-cn.com"] 
    }

重新加载docker

systemctl daemon-reload;systemctl restart docker; systemctl enable docker

查看docker版本信息  
docker version  
docker info

### 5. 配置内核参数 ###

] vim /etc/sysctl.d/k8s.conf
    net.ipv4.ip_forward = 1
    net.bridge.bridge-nf-call-iptables = 1
    net.bridge.bridge-nf-call-ip6tables = 1
    fs.may_detach_mounts = 1
    vm.overcommit_memory=1
    vm.panic_on_oom=0
    fs.inotify.max_user_watches=89100
    fs.file-max=52706963
    fs.nr_open=52706963
    net.netfilter.nf_conntrack_max=2310720
    net.ipv4.tcp_keepalive_time = 600
    net.ipv4.tcp_keepalive_probes = 3
    net.ipv4.tcp_keepalive_intvl =15
    net.ipv4.tcp_max_tw_buckets = 36000
    net.ipv4.tcp_tw_reuse = 1
    net.ipv4.tcp_max_orphans = 327680
    net.ipv4.tcp_orphan_retries = 3
    net.ipv4.tcp_syncookies = 1
    net.ipv4.tcp_max_syn_backlog = 16384
    net.ipv4.ip_conntrack_max = 65536
    net.ipv4.tcp_max_syn_backlog = 16384
    net.ipv4.tcp_timestamps = 0
    net.core.somaxconn = 16384

为了保证br\_netfilter模块加载，我们需要执行以下命令使参数生效

] sysctl --system && modprobe br_netfilter

### 6. 下载k8s源码包 ###

https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG  
打开页面后选择对应版本的Server Binaries下载  
![在这里插入图片描述][20201115084107828.png_pic_center]  
![在这里插入图片描述][20201115084116900.png_pic_center]

### 7. 下载etcd安装包 ###

wget https://github.com/etcd-io/etcd/releases/download/v3.4.3/etcd-v3.4.3-linux-amd64.tar.gz
    tar -zxf kubernetes-server-linux-amd64.tar.gz;cd kubernetes/server/bin/
    cp kube{ let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} /usr/local/bin/
    tar -zxf etcd-v3.4.3-linux-amd64.tar.gz;cd etcd-v3.4.3-linux-amd64/
    cp etcd etcdctl /usr/local/bin/
    查看版本
    kubelet --version
    etcdctl version
    将组件复制到其他节点
    cd /usr/local/bin/
    WorkNodes='node1 node2'
    for NODE in $WorkNodes; do scp /usr/local/bin/etcd* $NODE:/usr/local/bin/;scp kube{ let,-proxy} $NODE:/usr/local/bin/ ; done	
    ls /usr/local/bin/	#master节点安装etcd以及k8s所有组件
    etcd etcdctl kube-apiserver kube-controller-manager kubectl kubelet kube-proxy kube-scheduler
    ls /usr/local/bin/		#node节点
    etcd etcdctl kubelet  kube-proxy

### 8. 安装CNI插件 ###

所有节点创建/opt/cni/bin目录

mkdir -p /opt/cni/bin; cd /opt/cni/bin
    wget https://github.com/containernetworking/plugins/releases/download/v0.8.5/cni-plugins-linux-amd64-v0.8.5.tgz
    tar -zxf cni-plugins-linux-amd64-v0.8.5.tgz -C /opt/cni/bin/
    for NODE in $WorkNodes; do scp /opt/cni/bin/* $NODE:/opt/cni/bin/; done
    ls /opt/cni/bin/
    bandwidth  bridge  dhcp  firewall  flannel  host-device  host-local  ipvlan  loopback  macvlan  portmap  ptp  sbr  static  tuning  vlan

# 二、 配置etcd集群 #

在master上生成证书，并分发到其他节点

### 1. 生成etcd证书 ###

wget "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64" -O /usr/local/bin/cfssl
    wget "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
    chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

所有节点创建etcd证书目录

mkdir -p /etc/kubernetes/pki  /etc/etcd/ssl

准备证书文件，下载到/root目录下  
链接：https://pan.baidu.com/s/1EwUdqH8RjS1BdfOd3M2v-Q 提取码：g5cq

在master上生成etcd证书

cd /root/k8s-ha-install/pki; ls
    admin-csr.json ca-config.json etcd-ca-csr.json front-proxy-ca-csr.json kubelet-csr.json    manager-csr.json apiserver-csr.json ca-csr.json etcd-csr.json front-proxy-client-csr.json kube-proxy-csr.json scheduler-csr.json
    cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca
    cfssl gencert -ca=/etc/etcd/ssl/etcd-ca.pem -ca-key=/etc/etcd/ssl/etcd-ca-key.pem -config=ca-config.json -hostname=127.0.0.1,master,node1,node2,192.168.1.11,192.168.1.12,192.168.1.13 -profile=kubernetes etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd

将etcd证书分发到其他master节点

cd /etc/etcd/ssl; ls
    etcd-ca.csr  etcd-ca-key.pem  etcd-ca.pem  etcd.csr  etcd-key.pem  etcd.pem
    WorkNodes=''node1 node2' 
    for NODE in $WorkNodes; do scp etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem $NODE:/etc/etcd/ssl/;done

### 2. 配置etcd集群 ###

master节点etcd配置文件  
**master，注意修改主机名和IP地址**

] vim /etc/etcd/etcd.config.yml
    name: 'master'
    data-dir: /var/lib/etcd
    wal-dir: /var/lib/etcd/wal
    snapshot-count: 5000
    heartbeat-interval: 100
    election-timeout: 1000
    quota-backend-bytes: 0
    listen-peer-urls: 'https://192.168.1.11:2380'
    listen-client-urls: 'https://192.168.1.11:2379,http://127.0.0.1:2379'
    max-snapshots: 3
    max-wals: 5
    cors:
    initial-advertise-peer-urls: 'https://192.168.1.11:2380'
    advertise-client-urls: 'https://192.168.1.11:2379'
    discovery:
    discovery-fallback: 'proxy'
    discovery-proxy:
    discovery-srv:
    initial-cluster: 'master=https://192.168.1.11:2380,node1=https://192.168.1.12:2380,node2=https://192.168.1.13:2380'
    initial-cluster-token: 'etcd-k8s-cluster'
    initial-cluster-state: 'new'
    strict-reconfig-check: false
    enable-v2: true
    enable-pprof: true
    proxy: 'off'
    proxy-failure-wait: 5000
    proxy-refresh-interval: 30000
    proxy-dial-timeout: 1000
    proxy-write-timeout: 5000
    proxy-read-timeout: 0
    client-transport-security:
      cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
      key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
      client-cert-auth: true
      trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
      auto-tls: true
    peer-transport-security:
      cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
      key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
      peer-client-cert-auth: true
      trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
      auto-tls: true
    debug: false
    log-package-levels:
    log-outputs: [default]
    force-new-cluster: false

**node1，注意修改主机名和IP地址**

] vim /etc/etcd/etcd.config.yml
    name: 'node1'
    data-dir: /var/lib/etcd
    wal-dir: /var/lib/etcd/wal
    snapshot-count: 5000
    heartbeat-interval: 100
    election-timeout: 1000
    quota-backend-bytes: 0
    listen-peer-urls: 'https://192.168.1.12:2380'
    listen-client-urls: 'https://192.168.1.12:2379,http://127.0.0.1:2379'
    max-snapshots: 3
    max-wals: 5
    cors:
    initial-advertise-peer-urls: 'https://192.168.1.12:2380'
    advertise-client-urls: 'https://192.168.1.12:2379'
    discovery:
    discovery-fallback: 'proxy'
    discovery-proxy:
    discovery-srv:
    initial-cluster: 'master=https://192.168.1.11:2380,node1=https://192.168.1.12:2380,node2=https://192.168.1.13:2380'
    initial-cluster-token: 'etcd-k8s-cluster'
    initial-cluster-state: 'new'
    strict-reconfig-check: false
    enable-v2: true
    enable-pprof: true
    proxy: 'off'
    proxy-failure-wait: 5000
    proxy-refresh-interval: 30000
    proxy-dial-timeout: 1000
    proxy-write-timeout: 5000
    proxy-read-timeout: 0
    client-transport-security:
      cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
      key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
      client-cert-auth: true
      trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
      auto-tls: true
    peer-transport-security:
      cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
      key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
      peer-client-cert-auth: true
      trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
      auto-tls: true
    debug: false
    log-package-levels:
    log-outputs: [default]
    force-new-cluster: false

**node2，注意修改主机名和IP地址**

] vim /etc/etcd/etcd.config.yml
    name: 'node2'
    data-dir: /var/lib/etcd
    wal-dir: /var/lib/etcd/wal
    snapshot-count: 5000
    heartbeat-interval: 100
    election-timeout: 1000
    quota-backend-bytes: 0
    listen-peer-urls: 'https://192.168.1.13:2380'
    listen-client-urls: 'https://192.168.1.13:2379,http://127.0.0.1:2379'
    max-snapshots: 3
    max-wals: 5
    cors:
    initial-advertise-peer-urls: 'https://192.168.1.13:2380'
    advertise-client-urls: 'https://192.168.1.13:2379'
    discovery:
    discovery-fallback: 'proxy'
    discovery-proxy:
    discovery-srv:
    initial-cluster: 'master=https://192.168.1.11:2380,node1=https://192.168.1.12:2380,node2=https://192.168.1.13:2380'
    initial-cluster-token: 'etcd-k8s-cluster'
    initial-cluster-state: 'new'
    strict-reconfig-check: false
    enable-v2: true
    enable-pprof: true
    proxy: 'off'
    proxy-failure-wait: 5000
    proxy-refresh-interval: 30000
    proxy-dial-timeout: 1000
    proxy-write-timeout: 5000
    proxy-read-timeout: 0
    client-transport-security:
      cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
      key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
      client-cert-auth: true
      trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
      auto-tls: true
    peer-transport-security:
      cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
      key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
      peer-client-cert-auth: true
      trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
      auto-tls: true
    debug: false
    log-package-levels:
    log-outputs: [default]
    force-new-cluster: false

### 3. 启动etcd ###

所有节点创建etcd service并启动

] vim /usr/lib/systemd/system/etcd.service
    [Unit]
    Description=Etcd Service
    Documentation=https://coreos.com/etcd/docs/latest/
    After=network.target
    [Service]
    Type=notify
    ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
    Restart=on-failure
    RestartSec=10
    LimitNOFILE=65536
    [Install]
    WantedBy=multi-user.target
    Alias=etcd3.service
    ] mkdir /etc/kubernetes/pki/etcd
    ] ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
    ] ls /etc/kubernetes/pki/etcd/
    etcd-ca.csr  etcd-ca-key.pem  etcd-ca.pem  etcd.csr  etcd-key.pem  etcd.pem
    ] systemctl daemon-reload; systemctl start etcd; systemctl enable etcd
    ] systemctl status etcd	#etcd服务状态为：active(running)

### 4. 验证etcd集群 ###

]# etcdctl --endpoints="192.168.1.11:2379,192.168.1.12:2379,192.168.1.13:2379" \
    --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem \
    --key=/etc/kubernetes/pki/etcd/etcd-key.pem  endpoint status --write-out=table

![在这里插入图片描述][20201115084717959.png_pic_center]

# 三、 生成kubernetes证书 #

所有节点创建kubernetes相关目录  
mkdir -p /etc/kubernetes/pki  
在master节点上生成kubernetes证书

### 1. 生成apiserver证书 ###

cd /root/k8s-ha-1.17/pki
    cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
    cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -hostname=10.96.0.1,192.168.1.11,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.1.11 -profile=kubernetes apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

### 2. 生成聚合证书 ###

cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca
    cfssl gencert -ca=/etc/kubernetes/pki/front-proxy-ca.pem -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem -config=ca-config.json -profile=kubernetes front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

### 3. 生成controller-manager证书 ###

cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem \
       -config=ca-config.json -profile=kubernetes \
       manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager
    ] kubectl config set-cluster kubernetes  --certificate-authority=/etc/kubernetes/pki/ca.pem \
      --embed-certs=true --server=https://192.168.1.11:6443 \
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

在kubeconfig文件中设置上下文环境，并设置为默认环境

] kubectl config set-context system:kube-controller-manager@kubernetes \
        --cluster=kubernetes  --user=system:kube-controller-manager \
    --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
    ] kubectl config use-context system:kube-controller-manager@kubernetes \
    --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

设置一个用户项

] kubectl config set-credentials system:kube-controller-manager \
         --client-certificate=/etc/kubernetes/pki/controller-manager.pem \
         --client-key=/etc/kubernetes/pki/controller-manager-key.pem \
         --embed-certs=true \
         --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

### 4. 生成scheduler证书 ###

] cfssl gencert  -ca=/etc/kubernetes/pki/ca.pem  -ca-key=/etc/kubernetes/pki/ca-key.pem \
       -config=ca-config.json  -profile=kubernetes \
       scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler
    ] kubectl config set-cluster kubernetes  --certificate-authority=/etc/kubernetes/pki/ca.pem \
         --embed-certs=true  --server=https://192.168.1.11:6443 \
         --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
    ] kubectl config set-credentials system:kube-scheduler \
         --client-certificate=/etc/kubernetes/pki/scheduler.pem \
         --client-key=/etc/kubernetes/pki/scheduler-key.pem \
         --embed-certs=true  --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
    ] kubectl config set-context system:kube-scheduler@kubernetes \
         --cluster=kubernetes  --user=system:kube-scheduler \
         --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
    ] kubectl config use-context system:kube-scheduler@kubernetes \
         --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

### 5. 生成admin管理员用户证书 ###

] cfssl gencert  -ca=/etc/kubernetes/pki/ca.pem  -ca-key=/etc/kubernetes/pki/ca-key.pem \
       -config=ca-config.json  -profile=kubernetes \
       admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin
    ] kubectl config set-cluster kubernetes  --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true  --server=https://192.168.1.11:6443 \
     --kubeconfig=/etc/kubernetes/admin.kubeconfig
    ] kubectl config set-credentials kubernetes-admin \
      --client-certificate=/etc/kubernetes/pki/admin.pem \
     --client-key=/etc/kubernetes/pki/admin-key.pem \
     --embed-certs=true  --kubeconfig=/etc/kubernetes/admin.kubeconfig
    ] kubectl config set-context kubernetes-admin@kubernetes \
      --cluster=kubernetes  --user=kubernetes-admin \
      --kubeconfig=/etc/kubernetes/admin.kubeconfig
    ] kubectl config use-context kubernetes-admin@kubernetes \
      --kubeconfig=/etc/kubernetes/admin.kubeconfig

### 6. 创建ServiceAccount Key ###

openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
    openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub

# 四、 Kubernetes组件配置 #

所有节点创建相关目录

mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes

### 1. 创建kube-apiserver service ###

在Master节点创建kube-apiserver service

vim /usr/lib/systemd/system/kube-apiserver.service 
    [Unit]
    Description=Kubernetes API Server
    Documentation=https://github.com/kubernetes/kubernetes
    After=network.target
    
    [Service]
    ExecStart=/usr/local/bin/kube-apiserver \
          --v=2  \
          --logtostderr=true  \
          --allow-privileged=true  \
          --bind-address=0.0.0.0  \
          --secure-port=6443  \
          --insecure-port=0  \
          --advertise-address=192.168.1.11 \
          --service-cluster-ip-range=10.96.0.0/12  \
          --service-node-port-range=30000-32767  \
          --etcd-servers=https://192.168.1.11:2379,https://192.168.1.12:2379,https://192.168.1.13:2379 \
          --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \
          --etcd-certfile=/etc/etcd/ssl/etcd.pem  \
          --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \
          --client-ca-file=/etc/kubernetes/pki/ca.pem  \
          --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \
          --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \
          --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \
          --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \
          --service-account-key-file=/etc/kubernetes/pki/sa.pub  \
          --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
          --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
          --authorization-mode=Node,RBAC  \
          --enable-bootstrap-token-auth=true  \
          --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \
          --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
          --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \
          --requestheader-allowed-names=aggregator  \
          --requestheader-group-headers=X-Remote-Group  \
          --requestheader-extra-headers-prefix=X-Remote-Extra-  \
          --requestheader-username-headers=X-Remote-User
          # --token-auth-file=/etc/kubernetes/token.csv
    
    Restart=on-failure
    RestartSec=10s
    LimitNOFILE=65535
    
    [Install]
    WantedBy=multi-user.target

### 2. 启动kube-apiserver ###

systemctl daemon-reload && systemctl enable kube-apiserver; systemctl start kube-apiserver
    检测kube-server状态，应为running
    systemctl status kube-apiserver
    Active: active (running)

### 3. 创建kube-controller-manager service ###

在Master节点配置kube-controller-manager service

vim /usr/lib/systemd/system/kube-controller-manager.service
    [Unit]
    Description=Kubernetes Controller Manager
    Documentation=https://github.com/kubernetes/kubernetes
    After=network.target
    
    [Service]
    ExecStart=/usr/local/bin/kube-controller-manager \
          --v=2 \
          --logtostderr=true \
          --address=127.0.0.1 \
          --root-ca-file=/etc/kubernetes/pki/ca.pem \
          --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \
          --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \
          --service-account-private-key-file=/etc/kubernetes/pki/sa.key \
          --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \
          --leader-elect=true \
          --use-service-account-credentials=true \
          --node-monitor-grace-period=40s \
          --node-monitor-period=5s \
          --pod-eviction-timeout=2m0s \
          --controllers=*,bootstrapsigner,tokencleaner \
          --allocate-node-cidrs=true \
          --cluster-cidr=10.244.0.0/16 \
          --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
          --node-cidr-mask-size=24
          
    Restart=always
    RestartSec=10s
    
    [Install]
    WantedBy=multi-user.target

### 4. 启动kube-controller-manager ###

在Master节点启动kube-controller-manager

systemctl daemon-reload; 
     systemctl enable kube-controller-manager; systemctl start kube-controller-manager
    检测kube-controller-manager 状态，应为running
    systemctl status kube-controller-manager
    Active: active (running)

### 5. 创建kube-scheduler service ###

在Master节点配置kube-scheduler service

vim /usr/lib/systemd/system/kube-scheduler.service 
    [Unit]
    Description=Kubernetes Scheduler
    Documentation=https://github.com/kubernetes/kubernetes
    After=network.target
    
    [Service]
    ExecStart=/usr/local/bin/kube-scheduler \
          --v=2 \
          --logtostderr=true \
          --address=127.0.0.1 \
          --leader-elect=true \
          --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
    
    Restart=always
    RestartSec=10s
    
    [Install]
    WantedBy=multi-user.target

### 6. 启动kube-scheduler ###

systemctl daemon-reload;systemctl enable kube-scheduler; systemctl start kube-scheduler
    检测kube-scheduler状态，应为running
    systemctl status kube-scheduler
    Active: active (running)

### 7. TLS Bootstrapping配置 ###

在master创建bootstrap

cd /root/k8s-ha-install/bootstrap
    ] kubectl config set-cluster kubernetes  --certificate-authority=/etc/kubernetes/pki/ca.pem \
      --embed-certs=true --server=https://192.168.1.11:6443 \
      --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
      ] kubectl config set-credentials tls-bootstrap-token-user  --token=c8ad9c.2e4d610cf3e7426e \
    --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

c8ad9c.2e4d610cf3e7426e是生成的随机序列，可通过以下命令生成，把生成的token写入bootstrap.secret.yaml文件中

echo "$(head -c 6 /dev/urandom | md5sum | head -c 6)"."$(head -c 16 /dev/urandom | md5sum | head -c 16)"
    c8ad9c.2e4d610cf3e7426e

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 1]

] kubectl config set-context tls-bootstrap-token-user@kubernetes --cluster=kubernetes \
     --user=tls-bootstrap-token-user   --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
    ] kubectl config use-context tls-bootstrap-token-user@kubernetes \
      --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
    mkdir -p /root/.kube
    cp -i /etc/kubernetes/admin.kubeconfig /root/.kube/config
    chown $(id -u):$(id -g) $HOME/.kube/config
    kubectl apply -f bootstrap.secret.yaml

![在这里插入图片描述][20201115085337841.png_pic_center]

添加加载环境变量，tab补全kubelet命令

echo "source <(kubectl completion bash)" >> ~/.bash_profile
    source /root/.bash_profile
    source /usr/share/bash-completion/bash_completion
    source <(kubectl completion bash)

# 五、 Node节点配置 #

把master上的证书复制到Node节点  
所有节点创建etcd证书目录

mkdir -p /etc/etcd/ssl/
    cd /etc/etcd/ssl/
    for NODE in node1 node2; do scp etcd-ca.pem etcd.pem etcd-key.pem $NODE:/etc/etcd/ssl/; done
    cd /etc/kubernetes/pki/
    for NODE in node1 node2; do scp ca.pem ca-key.pem front-proxy-ca.pem $NODE:/etc/kubernetes/pki;done
    cd /etc/kubernetes/
    for NODE in node1 node2; do scp bootstrap-kubelet.kubeconfig $NODE:/etc/kubernetes/;done
    所有Node节点创建相关目录
    mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/

### 1. 创建kubelet service ###

所有节点配置kubelet service（Master节点不部署Pod也可无需配置）

vim  /usr/lib/systemd/system/kubelet.service
    [Unit]
    Description=Kubernetes Kubelet
    Documentation=https://github.com/kubernetes/kubernetes
    After=docker.service
    Requires=docker.service
    
    [Service]
    ExecStart=/usr/local/bin/kubelet
    
    Restart=always
    StartLimitInterval=0
    RestartSec=10
    
    [Install]
    WantedBy=multi-user.target

### 2. 配置10-kubelet.conf ###

vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf
    [Service]
    Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
    Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
    Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
    Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
    ExecStart=
    ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS

### 3. 配置kubelet-conf.yml ###

vim /etc/kubernetes/kubelet-conf.yml
    apiVersion: kubelet.config.k8s.io/v1beta1
    kind: KubeletConfiguration
    address: 0.0.0.0
    port: 10250
    readOnlyPort: 10255
    authentication:
      anonymous:
        enabled: false
      webhook:
        cacheTTL: 2m0s
        enabled: true
      x509:
        clientCAFile: /etc/kubernetes/pki/ca.pem
    authorization:
      mode: Webhook
      webhook:
        cacheAuthorizedTTL: 5m0s
        cacheUnauthorizedTTL: 30s
    cgroupDriver: systemd
    cgroupsPerQOS: true
    clusterDNS:
    - 10.96.0.10
    clusterDomain: cluster.local
    containerLogMaxFiles: 5
    containerLogMaxSize: 10Mi
    contentType: application/vnd.kubernetes.protobuf
    cpuCFSQuota: true
    cpuManagerPolicy: none
    cpuManagerReconcilePeriod: 10s
    enableControllerAttachDetach: true
    enableDebuggingHandlers: true
    enforceNodeAllocatable:
    - pods
    eventBurst: 10
    eventRecordQPS: 5
    evictionHard:
      imagefs.available: 15%
      memory.available: 100Mi
      nodefs.available: 10%
      nodefs.inodesFree: 5%
    evictionPressureTransitionPeriod: 5m0s
    failSwapOn: true
    fileCheckFrequency: 20s
    hairpinMode: promiscuous-bridge
    healthzBindAddress: 127.0.0.1
    healthzPort: 10248
    httpCheckFrequency: 20s
    imageGCHighThresholdPercent: 85
    imageGCLowThresholdPercent: 80
    imageMinimumGCAge: 2m0s
    iptablesDropBit: 15
    iptablesMasqueradeBit: 14
    kubeAPIBurst: 10
    kubeAPIQPS: 5
    makeIPTablesUtilChains: true
    maxOpenFiles: 1000000
    maxPods: 110
    nodeStatusUpdateFrequency: 10s
    oomScoreAdj: -999
    podPidsLimit: -1
    registryBurst: 10
    registryPullQPS: 5
    resolvConf: /etc/resolv.conf
    rotateCertificates: true
    runtimeRequestTimeout: 2m0s
    serializeImagePulls: true
    staticPodPath: /etc/kubernetes/manifests
    streamingConnectionIdleTimeout: 4h0m0s
    syncFrequency: 1m0s
    volumeStatsAggPeriod: 1m0s

### 4. 启动kubelet ###

所有节点启动kubelet

systemctl daemon-reload;systemctl enable kubelet; systemctl start kubelet
    systemctl status kubelet.service
    Active: active (running)
    
    此时系统日志/var/log/messages, 显示只有如下信息为正常
    Unable to update cni config: no networks found in /etc/cni/net.d 
    查看集群状态
    ]# kubectl get node

![在这里插入图片描述][20201115085740947.png_pic_center]

### 5. Kube-Proxy配置（在master节点上） ###

cd /root/k8s-ha-1.17/
    kubectl -n kube-system create serviceaccount kube-proxy
    ] kubectl create clusterrolebinding system:kube-proxy   --clusterrole system:node-proxier \
    --serviceaccount kube-system:kube-proxy
    SECRET=$(kubectl -n kube-system get sa/kube-proxy --output=jsonpath='{.secrets[0].name}')
    JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET --output=jsonpath='{.data.token}' | base64 -d)
    ] kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem \
    --embed-certs=true  --server=https://192.168.1.11:6443 \
    --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
    ] kubectl config set-credentials kubernetes  --token=${JWT_TOKEN}  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
    ] kubectl config set-context kubernetes --cluster=kubernetes --user=kubernetes  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
    ] kubectl config use-context kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig

赋值Service文件

for NODE in master node1 node2; do
       scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
       scp kube-proxy/kube-proxy.conf $NODE:/etc/kubernetes/kube-proxy.conf
       scp kube-proxy/kube-proxy.service $NODE:/usr/lib/systemd/system/kube-proxy.service
     done

所有节点启动kube-proxy

systemctl daemon-reload;systemctl enable kube-proxy; systemctl start kube-proxy
    systemctl status kube-proxy
    Active: active (running)

# 六、 安装插件 #

### 1. 安装calico ###

cd /root/k8s-ha-1.17/Calico/  
kubectl apply -f calico.yaml  
kubectl get pod -n kube-system -owide  
![在这里插入图片描述][20201115090018369.png_pic_center]

kubectl get node  
![在这里插入图片描述][20201115090013719.png_pic_center]

### 2. 安装CoreDNS ###

安装最新版CoreDNS

cd /root/k8s-ha-1.17/CoreDNS/
    kubectl apply -f coredns.yaml
    kubectl get pod -n kube-system

![在这里插入图片描述][2020111509000166.png_pic_center]

### 3. 安装Metrics Server ###

cd /root/k8s-ha-1.17/
    kubectl apply -f metrics-server-3.6.1/
    kubectl get pod -n kube-system -l k8s-app=metrics-server
    kubectl get pod -n kube-system

![在这里插入图片描述][20201115085953220.png_pic_center]

![在这里插入图片描述][20201115085936967.png_pic_center]

### 4. 安装Dashboard ###

kubectl apply -f dashboard/
    kubectl get pod -A

![在这里插入图片描述][20201115085855606.png_pic_center]

kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')

在谷歌浏览器（Chrome）启动文件中加入启动参数，用于解决无法访问Dashboard的问题，  
–test-type --ignore-certificate-errors  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 2]

访问Dashboard：https://192.168.1.11:30000，选择登录方式为令牌（即token方式），  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 3]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 4]

七、 集群验证  
Pod必须能解析Service  
Pod必须能解解析跨namespace的Service  
每个节点都必须能访问集群的kubernetes的SVC(443端口)和kube-dns的svc(53端口)  
Pod于pod之间要能通信：同namespace间，跨namespace间，跨机器能通信  
kubectl get svc  
创建一个busybox的pod

vim busybox.yaml
    apiVersion: v1
    kind: Pod
    metadata:
      name: busybox
      namespace: default
    spec:
      containers:
      - name: busybox
        image: busybox:1.28
        command:
          - sleep
          - "3600"
        imagePullPolicy: IfNotPresent
      restartPolicy: Always
    kubectl apply -f busybox.yaml
    kubectl exec  busybox -n default -- nslookup kubernetes
    kubectl exec  busybox -n default -- nslookup kube-dns.kube-system

![在这里插入图片描述][20201115085804934.png_pic_center]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center]: /images/20221120/a71490b72f33479bb32b3b0fe9794f7b.png
[2020111508393249.png_pic_center]: /images/20221120/f2354b5042154acc9f349bb6767eb475.png
[20201115084107828.png_pic_center]: /images/20221120/fc161a33832946d9a329c0701eddf781.png
[20201115084116900.png_pic_center]: /images/20221120/5fb2952f30ea44deab5eac1964e6c10d.png
[20201115084717959.png_pic_center]: /images/20221120/38a2e086b5ec4f13ab820f9841e42e86.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221120/73a654828445454faa3b1295d79ffbb1.png
[20201115085337841.png_pic_center]: /images/20221120/5bd709de1f8448de8fbbeb8127a31589.png
[20201115085740947.png_pic_center]: https://img-blog.csdnimg.cn/20201115085740947.png#pic_center
[20201115090018369.png_pic_center]: https://img-blog.csdnimg.cn/20201115090018369.png#pic_center
[20201115090013719.png_pic_center]: https://img-blog.csdnimg.cn/20201115090013719.png#pic_center
[2020111509000166.png_pic_center]: https://img-blog.csdnimg.cn/2020111509000166.png#pic_center
[20201115085953220.png_pic_center]: https://img-blog.csdnimg.cn/20201115085953220.png#pic_center
[20201115085936967.png_pic_center]: https://img-blog.csdnimg.cn/20201115085936967.png#pic_center
[20201115085855606.png_pic_center]: https://img-blog.csdnimg.cn/20201115085855606.png#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 2]: https://img-blog.csdnimg.cn/20201115085848564.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 3]: https://img-blog.csdnimg.cn/20201115085831312.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU_size_16_color_FFFFFF_t_70_pic_center 4]: https://img-blog.csdnimg.cn/2020111508583814.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0hscm9saXU=,size_16,color_FFFFFF,t_70#pic_center
[20201115085804934.png_pic_center]: https://img-blog.csdnimg.cn/20201115085804934.png#pic_center