kubernetes v1.11.0 从代码编译到部署文档-部署

妖狐艹你老母 2022-05-15 02:45 173阅读 0赞

## 系统环境 ##

[root@master-47-35 ansible]# uname -a 
    Linux master-47-35 3.10.0-327.4.5.el7.x86_64 #1 SMP Mon Jan 25 22:07:14 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
    [root@master-47-35 ansible]# cat /etc/redhat-release 
    CentOS Linux release 7.5.1804 (Core)

## 机器信息 ##

10.39.47.12 slave-47-12
    10.39.47.35 master-47-35
    10.39.47.33 slave-47-33 
    10.39.47.32 slave-47-32
    10.39.47.34 master-47-34
    10.39.47.36 master-47-36

--------------------

## 安装详情 ##

**制作证书**

1.安装 cfssl

mkdir -p /opt/local/cfssl
    
    cd /opt/local/cfssl
    
    wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
    mv cfssl_linux-amd64 cfssl
    
    wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
    mv cfssljson_linux-amd64 cfssljson
    
    wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
    mv cfssl-certinfo_linux-amd64 cfssl-certinfo
    
    chmod +x *

安装结果

[root@master-47-35 cfssl]# ls -l
    total 18808
    -rwxr-xr-x 1 root root 10376657 Mar 30  2016 cfssl
    -rwxr-xr-x 1 root root  6595195 Mar 30  2016 cfssl-certinfo
    -rwxr-xr-x 1 root root  2277873 Mar 30  2016 cfssljson

--------------------

创建 CA 证书配置

vi  config.json
    {
      "signing": {
        "default": {
          "expiry": "87600h"
        },
        "profiles": {
          "kubernetes": {
            "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ],
            "expiry": "87600h"
          }
        }
      }
    }

vi csr.json
    
    {
      "CN": "kubernetes",
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "Beijing",
          "L": "Beijing",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

生成 CA 证书和私钥

cd /opt/ssl/
    
    /opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca

[root@master-47-35 ssl]# ls -lt
    total 20
    -rw-r--r-- 1 root root 1001 Aug 23 17:08 ca.csr
    -rw------- 1 root root 1679 Aug 23 17:08 ca-key.pem
    -rw-r--r-- 1 root root 1359 Aug 23 17:08 ca.pem
    -rw-r--r-- 1 root root  208 Aug 23 17:07 csr.json
    -rw-r--r-- 1 root root  292 Aug 23 17:05 config.json

分发证书(ansible工具同步,关于如何使用ansible工具请自行研究)

[root@master-47-35 ansible]# cat hosts 
    [etcd]
    10.39.47.35
    10.39.47.34
    10.39.47.36
    
    [master]
    10.39.47.35
    10.39.47.36
    10.39.47.34
    
    [node]
    10.39.47.32
    10.39.47.12
    10.39.47.33
    
    [all]
    10.39.47.32
    10.39.47.34
    10.39.47.36
    10.39.47.35
    10.39.47.12
    10.39.47.33

[root@master-47-35 ansible]# ansible -i hosts all -m copy -a 'src=/opt/ssl/ dest=/etc/kubernetes/ssl'
    10.39.47.32 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.12 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.34 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.35 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.36 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.33 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }

--------------------

在每台机器上安装docker

ansible -i hosts all -m shell -a "yum install docker -y"
    
    ansible -i hosts all -m shell -a "systemctl enable docker"
    
    ansible -i hosts all -m shell -a "systemctl start docker"

查看docker 状态

ansible -i hosts all -m shell -a "systemctl status docker"

--------------------

## 安装etcd 集群 ##

下载etcd

wget https://github.com/coreos/etcd/releases/download/v3.2.18/etcd-v3.2.18-linux-amd64.tar.gz
    tar zxvf etcd-v3.2.18-linux-amd64.tar.gz 
    
    [root@master-47-35 ansible]# ansible -i hosts etcd -m copy -a 'src=./etcd/etcdctl dest=/usr/bin/'
    [root@master-47-35 ansible]# ansible -i hosts etcd -m copy -a 'src=./etcd/etcdctl dest=/usr/bin/'

创建 etcd 证书

cd /opt/ssl/
    
    vi etcd-csr.json
    {
      "CN": "etcd",
      "hosts": [
        "127.0.0.1",
        "10.39.47.34",
        "10.39.47.36",
        "10.39.47.35"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "Beijing",
          "L": "Beijing",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

# 生成 etcd   密钥
    
    /opt/local/cfssl/cfssl gencert -ca=/opt/ssl/ca.pem \
      -ca-key=/opt/ssl/ca-key.pem \
      -config=/opt/ssl/config.json \
      -profile=kubernetes etcd-csr.json | /opt/local/cfssl/cfssljson -bare etcd
    
    [root@master-47-35 ssl]# ls -lt etcd*
    -rw-r--r-- 1 root root 1062 Aug 23 18:20 etcd.csr
    -rw------- 1 root root 1679 Aug 23 18:20 etcd-key.pem
    -rw-r--r-- 1 root root 1436 Aug 23 18:20 etcd.pem
    -rw-r--r-- 1 root root  293 Aug 23 18:19 etcd-csr.json

# 检查证书
    
    [root@master-47-35 ssl]# /opt/local/cfssl/cfssl-certinfo -cert etcd.pem
    
    
    
    # 拷贝到etcd服务器
    
    [root@master-47-35 ansible]# ansible -i hosts etcd -m copy -a 'src=/opt/ssl/ dest=/etc/kubernetes/ssl/'
    10.39.47.35 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.34 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    10.39.47.36 | SUCCESS => {
        "changed": true, 
        "dest": "/etc/kubernetes/ssl/", 
        "src": "/opt/ssl/"
    }
    
    [root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'ls -l /etc/kubernetes/ssl/'
    10.39.47.34 | SUCCESS | rc=0 >>
    total 36
    -rw-r--r-- 1 root root 1001 Aug 23 17:12 ca.csr
    -rw-r--r-- 1 root root 1679 Aug 23 17:12 ca-key.pem
    -rw-r--r-- 1 root root 1359 Aug 23 17:12 ca.pem
    -rw-r--r-- 1 root root  292 Aug 23 17:12 config.json
    -rw-r--r-- 1 root root  208 Aug 23 17:12 csr.json
    -rw-r--r-- 1 root root 1062 Aug 23 18:30 etcd.csr
    -rw-r--r-- 1 root root  293 Aug 23 18:30 etcd-csr.json
    -rw-r--r-- 1 root root 1679 Aug 23 18:30 etcd-key.pem
    -rw-r--r-- 1 root root 1436 Aug 23 18:30 etcd.pem
    
    10.39.47.35 | SUCCESS | rc=0 >>
    total 36
    -rw-r--r-- 1 root root 1001 Aug 23 17:12 ca.csr
    -rw-r--r-- 1 root root 1679 Aug 23 17:12 ca-key.pem
    -rw-r--r-- 1 root root 1359 Aug 23 17:12 ca.pem
    -rw-r--r-- 1 root root  292 Aug 23 17:12 config.json
    -rw-r--r-- 1 root root  208 Aug 23 17:12 csr.json
    -rw-r--r-- 1 root root 1062 Aug 23 18:30 etcd.csr
    -rw-r--r-- 1 root root  293 Aug 23 18:30 etcd-csr.json
    -rw-r--r-- 1 root root 1679 Aug 23 18:30 etcd-key.pem
    -rw-r--r-- 1 root root 1436 Aug 23 18:30 etcd.pem
    
    10.39.47.36 | SUCCESS | rc=0 >>
    total 36
    -rw-r--r-- 1 root root 1001 Aug 23 17:12 ca.csr
    -rw-r--r-- 1 root root 1679 Aug 23 17:12 ca-key.pem
    -rw-r--r-- 1 root root 1359 Aug 23 17:12 ca.pem
    -rw-r--r-- 1 root root  292 Aug 23 17:12 config.json
    -rw-r--r-- 1 root root  208 Aug 23 17:12 csr.json
    -rw-r--r-- 1 root root 1062 Aug 23 18:30 etcd.csr
    -rw-r--r-- 1 root root  293 Aug 23 18:30 etcd-csr.json
    -rw-r--r-- 1 root root 1679 Aug 23 18:30 etcd-key.pem
    -rw-r--r-- 1 root root 1436 Aug 23 18:30 etcd.pem
    
    
    # 如果 etcd 非 root 用户，读取证书会提示没权限
    
    [root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'chmod 644 /etc/kubernetes/ssl/etcd*'
     [WARNING]: Consider using file module with mode rather than running chmod
    
    10.39.47.36 | SUCCESS | rc=0 >>
    
    
    10.39.47.34 | SUCCESS | rc=0 >>
    
    
    10.39.47.35 | SUCCESS | rc=0 >>

修改 etcd 配置

[root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'useradd etcd;mkdir -p /opt/etcd;chown -R etcd:etcd /opt/etcd'

# etcd-1
    
    
    vi /etc/systemd/system/etcd.service
    
    
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    
    [Service]
    Type=notify
    WorkingDirectory=/opt/etcd/
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/usr/bin/etcd \
      --name=etcd-47-34 \
      --cert-file=/etc/kubernetes/ssl/etcd.pem \
      --key-file=/etc/kubernetes/ssl/etcd-key.pem \
      --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
      --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
      --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
      --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
      --initial-advertise-peer-urls=https://10.39.47.34:2380 \
      --listen-peer-urls=https://10.39.47.34:2380 \
      --listen-client-urls=https://10.39.47.34:2379,http://127.0.0.1:2379 \
      --advertise-client-urls=https://10.39.47.34:2379 \
      --initial-cluster-token=k8s-etcd-cluster \
      --initial-cluster=etcd-47-34=https://10.39.47.34:2380,etcd-47-35=https://10.39.47.35:2380,etcd-47-36=https://10.39.47.36:2380 \
      --initial-cluster-state=new \
      --data-dir=/opt/etcd/
    Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

# etcd-2
    
    
    vi /etc/systemd/system/etcd.service
    
    
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    
    [Service]
    Type=notify
    WorkingDirectory=/opt/etcd/
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/usr/bin/etcd \
      --name=etcd-47-35 \
      --cert-file=/etc/kubernetes/ssl/etcd.pem \
      --key-file=/etc/kubernetes/ssl/etcd-key.pem \
      --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
      --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
      --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
      --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
      --initial-advertise-peer-urls=https://10.39.47.35:2380 \
      --listen-peer-urls=https://10.39.47.35:2380 \
      --listen-client-urls=https://10.39.47.35:2379,http://127.0.0.1:2379 \
      --advertise-client-urls=https://10.39.47.35:2379 \
      --initial-cluster-token=k8s-etcd-cluster \
      --initial-cluster=etcd-47-34=https://10.39.47.34:2380,etcd-47-35=https://10.39.47.35:2380,etcd-47-36=https://10.39.47.36:2380 \
      --initial-cluster-state=new \
      --data-dir=/opt/etcd/
    Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

# etcd-3
    
    
    vi /etc/systemd/system/etcd.service
    
    
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    
    [Service]
    Type=notify
    WorkingDirectory=/opt/etcd/
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/usr/bin/etcd \
      --name=etcd-47-36 \
      --cert-file=/etc/kubernetes/ssl/etcd.pem \
      --key-file=/etc/kubernetes/ssl/etcd-key.pem \
      --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
      --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
      --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
      --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
      --initial-advertise-peer-urls=https://10.39.47.36:2380 \
      --listen-peer-urls=https://10.39.47.36:2380 \
      --listen-client-urls=https://10.39.47.36:2379,http://127.0.0.1:2379 \
      --advertise-client-urls=https://10.39.47.36:2379 \
      --initial-cluster-token=k8s-etcd-cluster \
      --initial-cluster=etcd-47-34=https://10.39.47.34:2380,etcd-47-35=https://10.39.47.35:2380,etcd-47-36=https://10.39.47.36:2380 \
      --initial-cluster-state=new \
      --data-dir=/opt/etcd/
    Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

改权限etcd 二进制文件的权限

[root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'chown -R etcd:etcd /usr/bin/etcd'
     [WARNING]: Consider using file module with owner rather than running chown
    [root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'chmod +x /usr/bin/etcd'
     [WARNING]: Consider using file module with mode rather than running chmod
    
    10.39.47.35 | SUCCESS | rc=0 >>
    
    
    10.39.47.36 | SUCCESS | rc=0 >>
    
    
    10.39.47.34 | SUCCESS | rc=0 >>

启动 etcd

[root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'systemctl start etcd'

查看状态

[root@master-47-35 ansible]# ansible -i hosts etcd -m shell -a 'systemctl status etcd'

查看 etcd 集群状态

etcdctl --endpoints=https://10.39.47.34:2379,https://10.39.47.35:2379,https://10.39.47.36:2379\
            --cert-file=/etc/kubernetes/ssl/etcd.pem \
            --ca-file=/etc/kubernetes/ssl/ca.pem \
            --key-file=/etc/kubernetes/ssl/etcd-key.pem \
            cluster-health
    
    member 33e2f0f1818f1c75 is healthy: got healthy result from https://10.39.47.35:2379
    member 3bf13ec35a47ecea is healthy: got healthy result from https://10.39.47.34:2379
    member 7ac684c93b663782 is healthy: got healthy result from https://10.39.47.36:2379
    cluster is healthy

查看 etcd 集群成员

etcdctl --endpoints=https://10.39.47.34:2379,https://10.39.47.35:2379,https://10.39.47.36:2379\
            --cert-file=/etc/kubernetes/ssl/etcd.pem \
            --ca-file=/etc/kubernetes/ssl/ca.pem \
            --key-file=/etc/kubernetes/ssl/etcd-key.pem \
            member list
    
    33e2f0f1818f1c75: name=etcd-47-35 peerURLs=https://10.39.47.35:2380 clientURLs=https://10.39.47.35:2379 isLeader=false
    3bf13ec35a47ecea: name=etcd-47-34 peerURLs=https://10.39.47.34:2380 clientURLs=https://10.39.47.34:2379 isLeader=true
    7ac684c93b663782: name=etcd-47-36 peerURLs=https://10.39.47.36:2380 clientURLs=https://10.39.47.36:2379 isLeader=false

--------------------

## 配置 Kubernetes 集群 ##

安装`kubectl`

[root@master-47-35 ansible]# ansible -i hosts etcd -m copy -a 'src=kubectl dest=/usr/bin'

创建 admin 证书

kubectl 与 kube-apiserver 的安全端口通信，需要为安全通信提供 TLS 证书和秘钥。

cd /opt/ssl/
    
    vi admin-csr.json
    
    
    {
      "CN": "admin",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "Beijing",
          "L": "Beijing",
          "O": "system:masters",
          "OU": "System"
        }
      ]
    }

# 生成 admin 证书和私钥
    cd /opt/ssl/
    
    /opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/opt/ssl/config.json \
      -profile=kubernetes admin-csr.json | /opt/local/cfssl/cfssljson -bare admin
    
    
    # 查看生成
    
    [root@master-47-35 ssl]# ls admin*
    admin.csr  admin-csr.json  admin-key.pem  admin.pem
    
    #分配证书
    [root@master-47-35 ansible]# ansible -i hosts all -m copy -a 'src=/opt/ssl/ dest=/etc/kubernetes/ssl'

生成 kubernetes 配置文件,生成证书相关的配置文件存储与 /root/.kube 目录中

# 配置 kubernetes 集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=https://10.39.47.35:6443 #10.39.47.34:6443 10.39.47.36:6443 一样的道理
    
    
    # 配置 客户端认证
    
    kubectl config set-credentials admin \
      --client-certificate=/etc/kubernetes/ssl/admin.pem \
      --embed-certs=true \
      --client-key=/etc/kubernetes/ssl/admin-key.pem
      
    
    
    kubectl config set-context kubernetes \
      --cluster=kubernetes \
      --user=admin
    
    
    kubectl config use-context kubernetes

创建 kubernetes 证书

cd /opt/ssl
    
    vi kubernetes-csr.json
    
    {
      "CN": "kubernetes",
      "hosts": [
        "127.0.0.1",
        "10.39.47.34",
        "10.39.47.35",
        "10.39.47.36",
        "10.254.0.1",
        "kubernetes",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "Beijing",
          "L": "Beijing",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }
    
    
    ## 这里 hosts 字段中 三个 IP 分别为 127.0.0.1 本机， 10.39.47.34 10.39.47.35 和 10.39.47.36 为 Master 的IP，多个Master需要写多个。  10.254.0.1 为 kubernetes SVC 的 IP， 一般是 部署网络的第一个IP , 如: 10.254.0.1 ， 在启动完成后，我们使用   kubectl get svc ， 就可以查看到

生成 kubernetes 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/opt/ssl/config.json \
      -profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes
    
    # 查看生成
    
    [root@master-47-35 ssl]# ls -l kubernetes*
    -rw-r--r-- 1 root root 1261 Aug 23 19:46 kubernetes.csr
    -rw-r--r-- 1 root root  473 Aug 23 19:46 kubernetes-csr.json
    -rw------- 1 root root 1679 Aug 23 19:46 kubernetes-key.pem
    -rw-r--r-- 1 root root 1627 Aug 23 19:46 kubernetes.pem
    
    # 拷贝到目录
    ansible -i hosts etcd -m copy -a 'src=/opt/ssl/ dest=/etc/kubernetes/ssl'

配置 kube-apiserver

kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求，kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token 一致，如果一致则自动为 kubelet生成证书和秘钥。

# 生成 token
    
    [root@master-47-35 ansible]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
    3af352ae41e9c7d8d700825f075602ec
    
    
    # 创建 encryption-config.yaml 配置
    
    cat > encryption-config.yaml <<EOF
    kind: EncryptionConfig
    apiVersion: v1
    resources:
      - resources:
          - secrets
        providers:
          - aescbc:
              keys:
                - name: key1
                  secret: 3af352ae41e9c7d8d700825f075602ec
          - identity: {}
    EOF
    
    
    # 拷贝
    
    [root@master-47-35 ansible]# ansible -i hosts all -m shell -a 'ls -l /etc/kubernetes/'

创建 目录`/etc/kubernetes/manifests`以及`kube-apiserver.json`文件

[root@master-47-35 manifests]# cat kube-apiserver.json 
    {
      "kind": "Pod",
      "apiVersion": "v1",
      "metadata": {
        "name": "kube-apiserver",
        "namespace": "kube-system",
        "creationTimestamp": null,
        "labels": {
          "component": "kube-apiserver",
          "tier": "control-plane"
        }
      },
      "spec": {
        "volumes": [
          {
            "name": "certs",
            "hostPath": {
              "path": "/etc/ssl/certs"
            }
          },
          {
            "name": "pki",
            "hostPath": {
              "path": "/etc/kubernetes"
            }
          }
        ],
        "containers": [
          {
            "name": "kube-apiserver",
            "image": "harbor.enncloud.cn/enncloud/hyperkube-amd64:v1.11.2",
            "imagePullPolicy": "Always",
            "command": [
              "/apiserver",
              "--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota",
              "--anonymous-auth=false",
              "--experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml", 
              "--advertise-address=10.39.47.35",
              "--allow-privileged=true",
              "--apiserver-count=3",
              "--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
              "--audit-log-maxage=30",
              "--audit-log-maxbackup=3",
              "--audit-log-maxsize=100",
              "--audit-log-path=/var/log/kubernetes/audit.log",
              "--authorization-mode=Node,RBAC",
              "--bind-address=10.39.47.35",
              "--secure-port=6443",
              "--client-ca-file=/etc/kubernetes/ssl/ca.pem",
              "--kubelet-client-certificate=/etc/kubernetes/ssl/kubernetes.pem", 
              "--kubelet-client-key=/etc/kubernetes/ssl/kubernetes-key.pem",
              "--enable-swagger-ui=true",
              "--etcd-cafile=/etc/kubernetes/ssl/ca.pem",
              "--etcd-certfile=/etc/kubernetes/ssl/etcd.pem",
              "--etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem",
              "--etcd-servers=https://10.39.47.35:2379,https://10.39.47.34:2379,https://10.39.47.36:2379", 
              "--event-ttl=1h",
              "--kubelet-https=true", 
              "--insecure-bind-address=127.0.0.1",
              "--insecure-port=8080",
              "--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem",
              "--service-cluster-ip-range=10.254.0.0/18",
              "--service-node-port-range=30000-32000",
              "--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem", 
              "--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem",
              "--enable-bootstrap-token-auth",
              "--v=2"
            ],
            "resources": {
              "requests": {
                "cpu": "250m"
              }
            },
            "volumeMounts": [
              {
                "name": "certs",
                "mountPath": "/etc/ssl/certs"
              },
              {
                "name": "pki",
                "readOnly": true,
                "mountPath": "/etc/kubernetes/"
              }
            ],
            "livenessProbe": {
              "httpGet": {
                "path": "/healthz",
                "port": 8080,
                "host": "127.0.0.1"
              },
              "initialDelaySeconds": 15,
              "timeoutSeconds": 15,
              "failureThreshold": 8
            }
          }
        ],
        "hostNetwork": true
      }
    }

创建`kube-controller-manager.json` 文件

[root@master-47-35 manifests]# cat kube-controller-manager.json 
    {
      "kind": "Pod",
      "apiVersion": "v1",
      "metadata": {
        "name": "kube-controller-manager",
        "namespace": "kube-system",
        "labels": {
          "component": "kube-controller-manager",
          "tier": "control-plane"
        }
      },
      "spec": {
        "volumes": [
          {
            "name": "certs",
            "hostPath": {
              "path": "/etc/ssl/certs"
            }
          },
          {
            "name": "pki",
            "hostPath": {
              "path": "/etc/kubernetes"
            }
          },
          {
            "name": "plugin",
            "hostPath": {
              "path": "/usr/libexec/kubernetes/kubelet-plugins"
            }
          },
          {
            "name": "qingcloud",
            "hostPath": {
              "path": "/etc/qingcloud"
            }
          }
        ],
        "containers": [
          {
            "name": "kube-controller-manager",
            "image": "harbor.enncloud.cn/enncloud/hyperkube-amd64:v1.11.2",
            "imagePullPolicy": "Always",
            "command": [
              "/controller-manager",
              "--address=127.0.0.1",
              "--master=http://127.0.0.1:8080",
              "--allocate-node-cidrs=false",
              "--service-cluster-ip-range=10.254.0.0/18",
              "--cluster-cidr=10.254.64.0/18",
              "--cluster-name=kubernetes",
              "--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem",
              "--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem",
              "--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem",
              "--root-ca-file=/etc/kubernetes/ssl/ca.pem",
              "--experimental-cluster-signing-duration=86700h0m0s",
              "--leader-elect=true",
              "--controllers=*,tokencleaner,bootstrapsigner",
              "--feature-gates=RotateKubeletServerCertificate=true",
              "--horizontal-pod-autoscaler-use-rest-clients",
              "--horizontal-pod-autoscaler-sync-period=60s",
              "--node-monitor-grace-period=40s",
              " --node-monitor-period=5s",
              "--pod-eviction-timeout=5m0s",
              "--v=10"
            ],
            "resources": {
              "requests": {
                "cpu": "200m"
              }
            },
            "volumeMounts": [
              {
                "name": "certs",
                "mountPath": "/etc/ssl/certs"
              },
              {
                "name": "pki",
                "readOnly": true,
                "mountPath": "/etc/kubernetes/"
              }, 
                            {
                "name": "plugin",
                "mountPath": "/usr/libexec/kubernetes/kubelet-plugins"
              },
              {
                "name": "qingcloud",
                "readOnly": true,
                "mountPath": "/etc/qingcloud"
              }
    
            ],
            "livenessProbe": {
              "httpGet": {
                "path": "/healthz",
                "port": 10252,
                "host": "127.0.0.1"
              },
              "initialDelaySeconds": 15,
              "timeoutSeconds": 15,
              "failureThreshold": 8
            }
          }
        ],
        "hostNetwork": true
      }
    }

创建`kube-scheduler.json` 文件

{
      "kind": "Pod",
      "apiVersion": "v1",
      "metadata": {
        "name": "kube-scheduler",
        "namespace": "kube-system",
        "creationTimestamp": null,
        "labels": {
          "component": "kube-scheduler",
          "tier": "control-plane"
        }
      },
      "spec": {
        "containers": [
          {
            "name": "kube-scheduler",
            "image": "harbor.enncloud.cn/enncloud/hyperkube-amd64:v1.11.2",
            "imagePullPolicy": "Always",
            "command": [
              "/scheduler",
              "--address=127.0.0.1",
              "--master=http://127.0.0.1:8080",
              "--leader-elect=true",
              "--v=2"
            ],
            "resources": {
              "requests": {
                "cpu": "100m"
              }
            },
            "livenessProbe": {
              "httpGet": {
                "path": "/healthz",
                "port": 10251,
                "host": "127.0.0.1"
              },
              "initialDelaySeconds": 15,
              "timeoutSeconds": 15,
              "failureThreshold": 8
            }
          }
        ],
        "hostNetwork": true
      }
    }

--------------------

**配置 kubelet 认证**

kubelet 授权 kube-apiserver 的一些操作 exec run logs 等

生成 bootstrap.kubeconfig

# 配置集群参数
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=https://10.39.47.35:6443 \
      --kubeconfig=bootstrap.kubeconfig
    
    # 配置客户端认证
    
    kubectl config set-credentials kubelet-bootstrap \
      --token=3af352ae41e9c7d8d700825f075602ec \
      --kubeconfig=bootstrap.kubeconfig
    
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \
      --user=kubelet-bootstrap \
      --kubeconfig=bootstrap.kubeconfig
      
      
    # 配置默认关联
    kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
    
    # 拷贝生成的 bootstrap.kubeconfig 文件
    
    [root@master-47-35 ansible]# ansible -i hosts all -m copy -a 'src=/etc/kubernetes dest=/etc/kubernetes'

mkdir /var/lib/kubelet

创建`kubelet.service`

[root@master-47-35 kubernetes]# cat /etc/systemd/system/kubelet.service 
    [Unit]
    Description=Kubernetes Kubelet
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=docker.service
    Requires=docker.service
    
    [Service]
    WorkingDirectory=/var/lib/kubelet
    ExecStart=/usr/local/bin/kubelet \
      --hostname-override=master-47-35 \
      --pod-infra-container-image=harbor.enncloud.cn/paas/pause-amd64:3.1 \
      --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
      --pod-manifest-path=/etc/kubernetes/manifests  \
      --kubeconfig=/etc/kubernetes/kubelet.config \
      --config=/etc/kubernetes/kubelet.config.json \
      --cert-dir=/etc/kubernetes/pki \
      --allow-privileged=true \
      --kube-reserved cpu=500m,memory=512m \
      --image-gc-high-threshold=85  --image-gc-low-threshold=70 \
      --logtostderr=true \
      --enable-controller-attach-detach=true --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ \
      --v=2
    
    [Install]
    WantedBy=multi-user.target

创建完成之后，启动kubelet  
/etc/kubernetes/kubelet.config.jsonn内容

cat /etc/kubernetes/kubelet.config.json
    [root@master-47-35 prome]# cat /etc/kubernetes/kubelet.config.json
    {
      "kind": "KubeletConfiguration",
      "apiVersion": "kubelet.config.k8s.io/v1beta1",
      "authentication": {
        "x509": {
          "clientCAFile": "/etc/kubernetes/ssl/ca.pem"
        },
        "webhook": {
          "enabled": true,
          "cacheTTL": "2m0s"
        },
        "anonymous": {
          "enabled": false
        }
      },
      "authorization": {
        "mode": "Webhook",
        "webhook": {
          "cacheAuthorizedTTL": "5m0s",
          "cacheUnauthorizedTTL": "30s"
        }
      },
      "address": "10.39.47.35",
      "port": 10250,
      "readOnlyPort": 10255,
      "cgroupDriver": "systemd",
      "hairpinMode": "promiscuous-bridge",
      "serializeImagePulls": false,
      "RotateCertificates": true,
      "featureGates": {
        "RotateKubeletClientCertificate": true,
        "RotateKubeletServerCertificate": true
      },
      "MaxPods": "512",
      "failSwapOn": false,
      "containerLogMaxSize": "10Mi",
      "containerLogMaxFiles": 5,
      "clusterDomain": "cluster.local",
      "clusterDNS": ["10.254.0.2"]
    }

# RBAC 只需创建一次就可以
    
    kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

组件信息

[root@master-47-35 kubernetes]# kubectl get cs 
    NAME                 STATUS    MESSAGE              ERROR
    scheduler            Healthy   ok                   
    controller-manager   Healthy   ok                   
    etcd-1               Healthy   {"health": "true"}   
    etcd-2               Healthy   {"health": "true"}   
    etcd-0               Healthy   {"health": "true"}

创建 bootstrap kubeconfig 文件

mkdir -p /var/log/filelog/containers

部署的过程中出现了这个问题

Aug 24 15:34:54 master-47-35 kubelet[25499]: E0824 15:34:54.766732   25499 summary.go:102] Failed to get system container stats for "/system.slice/kubelet.service": failed to get cgroup stats for "/system.slice/kubelet.service": failed to get container info for "/system.slice/kubelet.service": unknown container "/system.slice/kubelet.service"
    Aug 24 15:34:54 master-47-35 kubelet[25499]: E0824 15:34:54.766760   25499 summary.go:102] Failed to get system container stats for "/system.slice/docker.service": failed to get cgroup stats for "/system.slice/docker.service": failed to get container info for "/system.slice/docker.service": unknown container "/system.slice/docker.service"

解决办法,相关[issues/4049][issues_4049]  
[issues/56850][issues_56850]

创建`/etc/systemd/system/kubelet.service.d/11-cgroups.conf`文件，内容如下

[Service]
    CPUAccounting=true
    MemoryAccounting=true

`kubelet`启动参数,添加以下两个启动参数

--runtime-cgroups=/systemd/system.slice \
    --kubelet-cgroups=/systemd/system.slice \

内容如下

[root@master-47-35 ~]# cat /etc/systemd/system/kubelet.service
    [Unit]
    Description=Kubernetes Kubelet
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=docker.service
    Requires=docker.service
    
    [Service]
    WorkingDirectory=/var/lib/kubelet
    ExecStart=/usr/local/bin/kubelet \
      --hostname-override=master-47-35 \
      --pod-infra-container-image=harbor.enncloud.cn/paas/pause-amd64:3.1 \
      --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
      --pod-manifest-path=/etc/kubernetes/manifests  \
      --kubeconfig=/etc/kubernetes/kubelet.config \
      --config=/etc/kubernetes/kubelet.config.json \
      --cert-dir=/etc/kubernetes/pki \
      --runtime-cgroups=/systemd/system.slice \
      --kubelet-cgroups=/systemd/system.slice \
      --allow-privileged=true \
      --kube-reserved cpu=500m,memory=512m \
      --image-gc-high-threshold=85  --image-gc-low-threshold=70 \
      --logtostderr=true \
      --enable-controller-attach-detach=true --volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ \
      --v=2
    
    [Install]
    WantedBy=multi-user.target

重启`kubelet`

systemctl daemon-reload
    systemctl restart kubelet

查看token list

kubeadm token list --kubeconfig ~/.kube/config

--------------------

配置token

# 生成 10.39.47.36 的 bootstrap.kubeconfig
    
    # 配置集群参数
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=https://10.39.47.35:6443 \
      --kubeconfig=master-47-36-bootstrap.kubeconfig
    
    # 配置客户端认证
    
    kubectl config set-credentials kubelet-bootstrap \
      --token=7uwvcs.5ai2ygtwsofeuxa5 \
      --kubeconfig=master-47-36-bootstrap.kubeconfig
    
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \
      --user=kubelet-bootstrap \
      --kubeconfig=master-47-36-bootstrap.kubeconfig
      
      
    # 配置默认关联
    kubectl config use-context default --kubeconfig=master-47-36-bootstrap.kubeconfig
    
    # 拷贝生成的 master-47-36-bootstrap.kubeconfig 文件
    
    mv master-47-36-bootstrap.kubeconfig /etc/kubernetes/bootstrap.kubeconfig

[root@master-47-34 kubernetes]# kubectl get nodes 
    NAME           STATUS    ROLES     AGE       VERSION
    master-47-34   Ready     <none>    2m        v1.11.0-1+feebbe2b16f546
    master-47-35   Ready     <none>    2h        v1.11.0-1+feebbe2b16f546
    master-47-36   Ready     <none>    17m       v1.11.0-1+feebbe2b16f546

--------------------

## 配置 kube-proxy ##

# 证书方面由于我们node端没有装 cfssl
    # 我们回到 master 端 机器 去配置证书，然后拷贝过来
    
    
    vi kube-proxy-csr.json
    
    {
      "CN": "system:kube-proxy",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

生成 kube-proxy 证书和私钥

创建 kube-proxy kubeconfig 文件

# 配置集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \
      --embed-certs=true \
      --server=https://10.39.47.35:6443 \
      --kubeconfig=kube-proxy.kubeconfig
    
    
    # 配置客户端认证
    
    kubectl config set-credentials kube-proxy \
      --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
      --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
      --embed-certs=true \
      --kubeconfig=kube-proxy.kubeconfig
      
      
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \
      --user=kube-proxy \
      --kubeconfig=kube-proxy.kubeconfig
    
    
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
    
    # 拷贝到需要的 node 端里
    
       ansible -i hosts all -m copy -a 'src=kube-proxy.csr dest=/etc/kubernetes/pki/'
       ansible -i hosts all -m copy -a 'src=kube-proxy-csr.json dest=/etc/kubernetes/pki/'
       ansible -i hosts all -m copy -a 'src=kube-proxy-key.pem dest=/etc/kubernetes/pki/'
       ansible -i hosts all -m copy -a 'src=kube-proxy.pem dest=/etc/kubernetes/pki/'
    ansible -i hosts all -m copy -a 'src=kube-proxy.kubeconfig dest=/etc/kubernetes/'

## 安装calico ##

需要在kubelet服务启动参数添加`--network-plugin=cni \`参数

wget http://docs.projectcalico.org/v3.2/getting-started/kubernetes/installation/hosted/calico.yaml
    
    wget http://docs.projectcalico.org/v3.2/getting-started/kubernetes/installation/rbac.yaml
    
    #所需镜像
    [root@master-47-35 calico]# cat calico.yaml | grep "image"
              image: quay.io/calico/node:v3.2.1
              image: quay.io/calico/cni:v3.2.1
              image: quay.io/calico/kube-controllers:v3.2.1

配置 calico

etcd_endpoints: "https://10.39.47.34:2379,https://10.39.47.35:2379,https://10.39.47.36:2379"
    
      # If you're using TLS enabled etcd uncomment the following.
      # You must also populate the Secret below with these files.
      etcd_ca: "/calico-secrets/etcd-ca"
      etcd_cert: "/calico-secrets/etcd-cert"
      etcd_key: "/calico-secrets/etcd-key"
    
    
    # 这里面要写入 base64 的信息
    
    
    data:
      etcd-key: (cat /etc/kubernetes/ssl/etcd-key.pem | base64 | tr -d '\n')
      etcd-cert: (cat /etc/kubernetes/ssl/etcd.pem | base64 | tr -d '\n')
      etcd-ca: (cat /etc/kubernetes/ssl/ca.pem | base64 | tr -d '\n')
    
    
    
    
     - name: CALICO_IPV4POOL_CIDR
                  value: "10.253.0.0/18"

创建calico服务

[root@master-47-35 calico]# kubectl create -f .
    configmap/calico-config created
    secret/calico-etcd-secrets created
    daemonset.extensions/calico-node created
    serviceaccount/calico-node created
    deployment.extensions/calico-kube-controllers created
    serviceaccount/calico-kube-controllers created
    clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
    clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
    clusterrole.rbac.authorization.k8s.io/calico-node created
    clusterrolebinding.rbac.authorization.k8s.io/calico-node created

安装 Calicoctl

cd /usr/local/bin/
    
    wget -c  https://github.com/projectcalico/calicoctl/releases/download/v3.2.1/calicoctl
    
    
    chmod +x calicoctl
    
    [root@master-47-35 nginx]# 
    [root@master-47-35 nginx]# cat /etc/calico/calicoctl.cfg 
    apiVersion: projectcalico.org/v3
    kind: CalicoAPIConfig
    metadata:
    spec:
      datastoreType: "etcdv3"
      etcdEndpoints: "https://10.39.47.34:2379,https://10.39.47.35:2379,https://10.39.47.36:2379"
      etcdKeyFile: "/etc/kubernetes/ssl/etcd-key.pem"
      etcdCertFile: "/etc/kubernetes/ssl/etcd.pem"
      etcdCACertFile: "/etc/kubernetes/ssl/ca.pem"
      
    
    [root@master-47-35 nginx]# calicoctl node status
    Calico process is running.
    
    IPv4 BGP status
    +--------------+-------------------+-------+----------+-------------+
    | PEER ADDRESS |     PEER TYPE     | STATE |  SINCE   |    INFO     |
    +--------------+-------------------+-------+----------+-------------+
    | 10.39.47.36  | node-to-node mesh | up    | 03:58:03 | Established |
    | 10.39.47.34  | node-to-node mesh | up    | 03:58:21 | Established |
    | 10.39.47.32  | node-to-node mesh | up    | 03:58:45 | Established |
    | 10.39.47.12  | node-to-node mesh | up    | 03:58:48 | Established |
    | 10.39.47.33  | node-to-node mesh | up    | 06:29:56 | Established |
    +--------------+-------------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.

calico 不是ipip模式 是bgp模式

[root@master-47-35 nginx]# calicoctl get ippool -o yaml 
    apiVersion: projectcalico.org/v3
    items:
    - apiVersion: projectcalico.org/v3
      kind: IPPool
      metadata:
        creationTimestamp: 2018-08-25T03:55:15Z
        name: default-ipv4-ippool
        resourceVersion: "188049"
        uid: acd48c40-a81a-11e8-84d5-5254e98192ae
      spec:
        cidr: 10.253.0.0/16
        ipipMode: Never //Always, CrossSubnet, Never	
        natOutgoing: true
    kind: IPPoolList
    metadata:
      resourceVersion: "188881"

部署kube-dns  
[kube-dns.yaml.base][]

mv kube-dns.yaml.base kube-dns.yaml
    
    1. # clusterIP: __PILLAR__DNS__SERVER__ 修改为我们之前定义的 dns IP 192.254.0.2
    
    2. # 修改 --domain=__PILLAR__DNS__DOMAIN__.   为 我们之前 预定的 domain 名称 --domain=cluster.local.
    
    3. # 修改 --server=/__PILLAR__DNS__DOMAIN__/127.0.0.1#10053  中 domain 为我们之前预定的 --server=/cluster.local./127.0.0.1#10053
    
    4. # 修改 --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__, 中的 domain 为我们之前预定的  --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local.,
    
    5. # 修改 --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__,  中的 domain 为我们之前预定的  --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local.,
    
    
    [root@master-47-35 kube-dns]# kubectl create -f kube-dns.yaml 
    service/kube-dns created
    serviceaccount/kube-dns created
    configmap/kube-dns created
    deployment.extensions/kube-dns created

安装集群完成

[root@master-47-35 kube-dns]# kubectl get pods -n kube-system -o wide 
    NAME                                       READY     STATUS    RESTARTS   AGE       IP             NODE
    calico-kube-controllers-65945f849d-nqbz9   1/1       Running   0          2h        10.39.47.32    slave-47-32
    calico-node-4965k                          2/2       Running   0          2h        10.39.47.35    master-47-35
    calico-node-967xp                          2/2       Running   0          2h        10.39.47.36    master-47-36
    calico-node-bsp6p                          2/2       Running   0          2h        10.39.47.32    slave-47-32
    calico-node-gsdpq                          2/2       Running   0          2h        10.39.47.34    master-47-34
    calico-node-r7sh2                          2/2       Running   0          2h        10.39.47.12    slave-47-12
    calico-node-x2smd                          2/2       Running   0          2h        10.39.47.33    slave-47-33
    kube-apiserver-master-47-34                1/1       Running   0          21h       10.39.47.34    master-47-34
    kube-apiserver-master-47-35                1/1       Running   0          23h       10.39.47.35    master-47-35
    kube-apiserver-master-47-36                1/1       Running   0          21h       10.39.47.36    master-47-36
    kube-controller-manager-master-47-34       1/1       Running   0          21h       10.39.47.34    master-47-34
    kube-controller-manager-master-47-35       1/1       Running   0          23h       10.39.47.35    master-47-35
    kube-controller-manager-master-47-36       1/1       Running   0          21h       10.39.47.36    master-47-36
    kube-dns-58d745cd6d-zng2z                  3/3       Running   0          36s       10.253.109.1   slave-47-33
    kube-proxy-5xzbz                           1/1       Running   0          4h        10.39.47.12    slave-47-12
    kube-proxy-996nd                           1/1       Running   0          4h        10.39.47.32    slave-47-32
    kube-proxy-gxxdn                           1/1       Running   0          4h        10.39.47.33    slave-47-33
    kube-proxy-n6p9v                           1/1       Running   0          4h        10.39.47.34    master-47-34
    kube-proxy-w72bm                           1/1       Running   0          4h        10.39.47.35    master-47-35
    kube-proxy-zs6b2                           1/1       Running   0          4h        10.39.47.36    master-47-36
    kube-scheduler-master-47-34                1/1       Running   0          21h       10.39.47.34    master-47-34
    kube-scheduler-master-47-35                1/1       Running   0          23h       10.39.47.35    master-47-35
    kube-scheduler-master-47-36                1/1       Running   0          21h       10.39.47.36    master-47-36

验证集群

[root@master-47-35 nginx]# kubectl create -f nginx.yaml 
    deployment.extensions/nginx-dm created
    service/nginx-svc created
    [root@master-47-35 nginx]# kubectl get pods 
    NAME                       READY     STATUS    RESTARTS   AGE
    nginx-dm-fff68d674-dvhhx   1/1       Running   0          9s
    nginx-dm-fff68d674-t4xf2   1/1       Running   0          9s
    nginx-dm-fff68d674-wl2l4   1/1       Running   0          9s
    [root@master-47-35 nginx]# kubectl get pods  -owide 
    NAME                       READY     STATUS    RESTARTS   AGE       IP              NODE
    nginx-dm-fff68d674-dvhhx   1/1       Running   0          12s       10.253.109.3    slave-47-33
    nginx-dm-fff68d674-t4xf2   1/1       Running   0          12s       10.253.20.130   slave-47-32
    nginx-dm-fff68d674-wl2l4   1/1       Running   0          12s       10.253.87.130   slave-47-12
    [root@master-47-35 nginx]# curl 10.253.109.3
    <!DOCTYPE html>
    <html>
    <head>
    <title>Welcome to nginx!</title>
    <style>
        body {
            width: 35em;
            margin: 0 auto;
            font-family: Tahoma, Verdana, Arial, sans-serif;
        }
    </style>
    </head>
    <body>
    <h1>Welcome to nginx!</h1>
    <p>If you see this page, the nginx web server is successfully installed and
    working. Further configuration is required.</p>
    
    <p>For online documentation and support please refer to
    <a href="http://nginx.org/">nginx.org</a>.<br/>
    Commercial support is available at
    <a href="http://nginx.com/">nginx.com</a>.</p>
    
    <p><em>Thank you for using nginx.</em></p>
    </body>
    </html>

**安装dashboard**

curl -O https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml

root@master-47-35 nginx]# kubectl create -f kubernetes-dashboard.yaml 
    secret/kubernetes-dashboard-certs created
    serviceaccount/kubernetes-dashboard created
    role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
    rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
    deployment.apps/kubernetes-dashboard created
    service/kubernetes-dashboard created
    [root@master-47-35 nginx]# kubectl get pods -n kube-system 
    NAME                                       READY     STATUS    RESTARTS   AGE
    calico-kube-controllers-65945f849d-nqbz9   1/1       Running   0          6h
    calico-node-4965k                          2/2       Running   0          6h
    calico-node-967xp                          2/2       Running   0          6h
    calico-node-bsp6p                          2/2       Running   0          6h
    calico-node-gsdpq                          2/2       Running   0          6h
    calico-node-r7sh2                          2/2       Running   0          6h
    calico-node-x2smd                          2/2       Running   0          6h
    kube-apiserver-master-47-34                1/1       Running   0          1d
    kube-apiserver-master-47-35                1/1       Running   0          1d
    kube-apiserver-master-47-36                1/1       Running   0          1d
    kube-controller-manager-master-47-34       1/1       Running   0          1d
    kube-controller-manager-master-47-35       1/1       Running   0          1d
    kube-controller-manager-master-47-36       1/1       Running   0          1d
    kube-dns-58d745cd6d-zng2z                  3/3       Running   0          4h
    kube-proxy-5xzbz                           1/1       Running   0          8h
    kube-proxy-996nd                           1/1       Running   0          8h
    kube-proxy-gxxdn                           1/1       Running   0          8h
    kube-proxy-n6p9v                           1/1       Running   0          8h
    kube-proxy-w72bm                           1/1       Running   0          8h
    kube-proxy-zs6b2                           1/1       Running   0          8h
    kube-scheduler-master-47-34                1/1       Running   0          1d
    kube-scheduler-master-47-35                1/1       Running   0          1d
    kube-scheduler-master-47-36                1/1       Running   0          1d
    kubernetes-dashboard-7c85576668-s5c6f      1/1       Running   0          6s

[root@master-47-35 nginx]# kubectl edit svc kubernetes-dashboard -n kube-system
    service/kubernetes-dashboard edited
    [root@master-47-35 nginx]# kubectl get svc -n kube-system 
    NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
    kube-dns               ClusterIP   10.254.0.2      <none>        53/UDP,53/TCP   4h
    kubernetes-dashboard   NodePort    10.254.11.180   <none>        443:30968

查看日志  
发现查找失败heapster的服务失败

[root@master-47-35 ~]# kubectl logs kubernetes-dashboard-7c85576668-s5c6f -n kube-system 
    2018/08/25 10:41:00 Starting overwatch
    2018/08/25 10:41:00 Using in-cluster config to connect to apiserver
    2018/08/25 10:41:00 Using service account token for csrf signing
    2018/08/25 10:41:00 No request provided. Skipping authorization
    2018/08/25 10:41:00 Successful initial request to the apiserver, version: v1.11.0-1+feebbe2b16f546
    2018/08/25 10:41:00 Generating JWE encryption key
    2018/08/25 10:41:00 New synchronizer has been registered: kubernetes-dashboard-key-holder-kube-system. Starting
    2018/08/25 10:41:00 Starting secret synchronizer for kubernetes-dashboard-key-holder in namespace kube-system
    2018/08/25 10:41:00 Storing encryption key in a secret
    2018/08/25 10:41:00 Creating in-cluster Heapster client
    2018/08/25 10:41:00 Metric client health check failed: the server could not find the requested resource (get services heapster). Retrying in 30 seconds.
    2018/08/25 10:41:00 Auto-generating certificates
    2018/08/25 10:41:00 Successfully created certificates
    2018/08/25 10:41:00 Serving securely on HTTPS port: 8443
    2018/08/25 10:41:30 Metric client health check failed: the server could not find the requested resource (get services heapster). Retrying in 30 seconds.
    2018/08/25 10:42:00 Metric client health check failed: the server could not find the requested resource (get services heapster). Retrying in 30 seconds.
    2018/08/25 10:42:30 Metric client health check failed: the server could not find the requested resource (get services heapster). Retrying in 30 seconds.
    2018/08/25 10:43:00 Metric client health check failed: the server could not find the requested resource (get services heapster). Retrying in 30 seconds.

接下来部署heapster

--------------------

**部署 heapster**

下载 heapster 相关 yaml 文件

wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/grafana.yaml
    
    wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/influxdb.yaml
    
    wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/heapster.yaml
    
    wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/rbac/heapster-rbac.yaml

yaml 文件列表 以及所需镜像

[root@master-47-35 heapster]# ls -l
    total 16
    -rw-r--r-- 1 root root 2276 Aug 27 09:53 grafana.yaml
    -rw-r--r-- 1 root root  264 Aug 27 09:54 heapster-rbac.yaml
    -rw-r--r-- 1 root root 1100 Aug 27 09:54 heapster.yaml
    -rw-r--r-- 1 root root  960 Aug 27 09:53 influxdb.yaml
    [root@master-47-35 heapster]# cat  grafana.yaml | grep image
            image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
    [root@master-47-35 heapster]# cat  heapster.yaml | grep image
            image: k8s.gcr.io/heapster-amd64:v1.5.4
            imagePullPolicy: IfNotPresent
    [root@master-47-35 heapster]# cat influxdb.yaml | grep image
            image: k8s.gcr.io/heapster-influxdb-amd64:v1.5.2

部署heapster

[root@master-47-35 heapster]# kubectl create -f .
    deployment.extensions/monitoring-grafana created
    service/monitoring-grafana created
    clusterrolebinding.rbac.authorization.k8s.io/heapster created
    serviceaccount/heapster created
    deployment.extensions/heapster created
    service/heapster created
    deployment.extensions/monitoring-influxdb created
    service/monitoring-influxdb created

查看部署heapster结果

[root@master-47-35 heapster]# kubectl get pods -n kube-system 
    NAME                                       READY     STATUS    RESTARTS   AGE
    calico-kube-controllers-65945f849d-nqbz9   1/1       Running   0          1d
    calico-node-4965k                          2/2       Running   0          1d
    calico-node-967xp                          2/2       Running   0          1d
    calico-node-bsp6p                          2/2       Running   0          1d
    calico-node-gsdpq                          2/2       Running   0          1d
    calico-node-r7sh2                          2/2       Running   0          1d
    calico-node-x2smd                          2/2       Running   0          1d
    heapster-c8bf7dccb-qntmt                   1/1       Running   0          29s
    kube-apiserver-master-47-34                1/1       Running   0          2d
    kube-apiserver-master-47-35                1/1       Running   0          2d
    kube-apiserver-master-47-36                1/1       Running   0          2d
    kube-controller-manager-master-47-34       1/1       Running   0          2d
    kube-controller-manager-master-47-35       1/1       Running   0          2d
    kube-controller-manager-master-47-36       1/1       Running   0          2d
    kube-dns-58d745cd6d-zng2z                  3/3       Running   0          1d
    kube-proxy-5xzbz                           1/1       Running   0          2d
    kube-proxy-996nd                           1/1       Running   0          2d
    kube-proxy-gxxdn                           1/1       Running   0          2d
    kube-proxy-n6p9v                           1/1       Running   0          2d
    kube-proxy-w72bm                           1/1       Running   0          2d
    kube-proxy-zs6b2                           1/1       Running   0          2d
    kube-scheduler-master-47-34                1/1       Running   0          2d
    kube-scheduler-master-47-35                1/1       Running   0          2d
    kube-scheduler-master-47-36                1/1       Running   0          2d
    kubernetes-dashboard-7c85576668-s5c6f      1/1       Running   0          1d
    monitoring-grafana-65cc8c4799-bsjxk        1/1       Running   0          30s
    monitoring-influxdb-69797ddf88-q2vn7       1/1       Running   0          30s

W0827 02:19:25.000397       1 manager.go:152] Failed to get all responses in time (got 0/6)
    E0827 02:20:05.001983       1 manager.go:101] Error in scraping containers from kubelet:10.39.47.34:10255: failed to get all container stats from Kubelet URL "http://10.39.47.34:10255/stats/container/": Post http://10.39.47.34:10255/stats/container/: dial tcp 10.39.47.34:10255: getsockopt: connection refused
    E0827 02:20:05.003892       1 manager.go:101] Error in scraping containers from kubelet:10.39.47.36:10255: failed to get all container stats from Kubelet URL "http://10.39.47.36:10255/stats/container/": Post http://10.39.47.36:10255/stats/container/: dial tcp 10.39.47.36:10255: getsockopt: connection refused
    E0827 02:20:05.011008       1 manager.go:101] Error in scraping containers from kubelet:10.39.47.32:10255: failed to get all container stats from Kubelet URL "http://10.39.47.32:10255/stats/container/": Post http://10.39.47.32:10255/stats/container/: dial tcp 10.39.47.32:10255: getsockopt: connection refused

修改 yaml 文件

# heapster.yaml 文件
    
    #### 修改如下部分 #####
    
    因为 kubelet 启用了 https 所以如下配置需要增加 https 端口
    
            - --source=kubernetes:https://kubernetes.default
    
    修改为
    
            - --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true

# heapster-rbac.yaml  文件
    
    #### 修改为部分 #####
    
    将 serviceAccount kube-system:heapster 与 ClusterRole system:kubelet-api-admin 绑定，授予它调用 kubelet API 的权限；
    
    
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1beta1
    metadata:
      name: heapster
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: system:heapster
    subjects:
    - kind: ServiceAccount
      name: heapster
      namespace: kube-system
    ---
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1beta1
    metadata:
      name: heapster-kubelet-api
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: system:kubelet-api-admin
    subjects:
    - kind: ServiceAccount
      name: heapster
      namespace: kube-system

[root@master-47-35 heapster]# kubectl create -f .
    deployment.extensions/monitoring-grafana created
    service/monitoring-grafana created
    clusterrolebinding.rbac.authorization.k8s.io/heapster created
    clusterrolebinding.rbac.authorization.k8s.io/heapster-kubelet-api created
    serviceaccount/heapster created
    deployment.extensions/heapster created
    service/heapster created
    deployment.extensions/monitoring-influxdb created
    service/monitoring-influxdb created

[root@master-47-35 heapster]# kubectl logs -f heapster-7797bb6dd4-jvs6n -n kube-system 
    I0827 02:25:20.229353       1 heapster.go:78] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086
    I0827 02:25:20.229399       1 heapster.go:79] Heapster version v1.5.4
    I0827 02:25:20.230543       1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default" and version v1
    I0827 02:25:20.230569       1 configs.go:62] Using kubelet port 10250

[root@master-47-35 heapster]# kubectl top nodes 
    NAME           CPU(cores)   CPU%      MEMORY(bytes)   MEMORY%   
    master-47-34   89m          2%        2257Mi          61%       
    master-47-35   124m         3%        2503Mi          67%       
    master-47-36   87m          2%        2449Mi          66%       
    slave-47-12    38m          1%        1469Mi          39%       
    slave-47-32    52m          1%        1434Mi          38%       
    slave-47-33    42m          1%        1457Mi          39%

部署完成

[issues_4049]: https://github.com/kubernetes/kops/issues/4049
[issues_56850]: https://github.com/kubernetes/kubernetes/issues/56850
[kube-dns.yaml.base]: https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/kube-dns/kube-dns.yaml.base