kubernetes 1.8.8 二进制证书安装

男娘i 2022-05-26 10:26 260阅读 0赞

## 环境准备 ##

linux 系统

[root@master-01 ~]# cat /etc/hosts
    127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
    ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
    192.168.1.41 master-01
    192.168.1.42 master-02
    192.168.1.43 master-03
    
    [root@master-01 ~]# cat /etc/redhat-release 
    CentOS Linux release 7.4.1708 (Core)

## 创建 验证(证书) ##

安装 cfssl

yum install wget 
    
    mkdir -p /opt/local/cfssl
    
    cd /opt/local/cfssl
    
    wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
    mv cfssl_linux-amd64 cfssl
    
    wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
    mv cfssljson_linux-amd64 cfssljson
    
    wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
    mv cfssl-certinfo_linux-amd64 cfssl-certinfo
    
    chmod +x *
    
    [root@master-01 cfssl]# ls -l 总用量 18808 -rwxr-xr-x 1 root root 10376657 3月 30 2016 cfssl -rwxr-xr-x 1 root root 6595195 3月 30 2016 cfssl-certinfo -rwxr-xr-x 1 root root 2277873 3月 30 2016 cfssljson

创建 CA 证书配置

mkdir /opt/ssl
    
    cd /opt/ssl

# config.json 文件
    
    vi  config.json
    
    
    {
      "signing": {
        "default": {
          "expiry": "87600h"
        },
        "profiles": {
          "kubernetes": {
            "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ],
            "expiry": "87600h"
          }
        }
      }
    }

# csr.json 文件
    
    vi csr.json
    
    {
      "CN": "kubernetes",
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

生成 CA 证书和私钥

[root@master-01 ssl]# /opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca
    2018/04/29 18:07:11 [INFO] generating a new CA key and certificate from CSR
    2018/04/29 18:07:11 [INFO] generate received request
    2018/04/29 18:07:11 [INFO] received CSR
    2018/04/29 18:07:11 [INFO] generating key: rsa-2048
    2018/04/29 18:07:11 [INFO] encoded CSR
    2018/04/29 18:07:11 [INFO] signed certificate with serial number 291408426601725634356434729258994816542793925675

[root@master-01 ssl]# ls -l
    总用量 20
    -rw-r--r-- 1 root root 1005 4月  29 18:07 ca.csr
    -rw------- 1 root root 1675 4月  29 18:07 ca-key.pem
    -rw-r--r-- 1 root root 1363 4月  29 18:07 ca.pem
    -rw-r--r-- 1 root root  292 4月  29 18:05 config.json
    -rw-r--r-- 1 root root  210 4月  29 18:05 csr.json

分发证书(在每个节点上执行)

# 创建证书目录
    mkdir -p /etc/kubernetes/ssl
    
    # 拷贝所有文件到目录下
    cp *.pem /etc/kubernetes/ssl
    cp ca.csr /etc/kubernetes/ssl
    # 拷贝ca到其他节点
    scp * root@192.168.1.42:/etc/kubernetes/ssl
    scp * root@192.168.1.43:/etc/kubernetes/ssl

## 安装Docker ##

yum install docker -y

添加加速器

vi /etc/docker/daemon.json
    {
       "registry-mirrors": ["http://579fe187.m.daocloud.io","https://pee6w651.mirror.aliyuncs.com"]}

启动docker

systemctl enable docker
    systemctl start docker systemctl status docker

## 安装etcd集群 ##

在你所要安装etcd的节点执行

yum install etcd3 -y

创建 etcd 证书

cd /opt/ssl/
    
    vi etcd-csr.json
    {
      "CN": "etcd",
      "hosts": [
        "127.0.0.1",
        "192.168.1.41",
        "192.168.1.42",
        "192.168.1.43"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

# 生成 etcd   密钥
     /opt/local/cfssl/cfssl gencert -ca=/opt/ssl/ca.pem \
      -ca-key=/opt/ssl/ca-key.pem \   -config=/opt/ssl/config.json \   -profile=kubernetes etcd-csr.json | /opt/local/cfssl/cfssljson -bare etcd

# 查看生成
    
    [root@master-01 ssl]# ls -l etcd*
    -rw-r--r-- 1 root root 1066 4月  29 18:15 etcd.csr
    -rw-r--r-- 1 root root  298 4月  29 18:14 etcd-csr.json
    -rw------- 1 root root 1675 4月  29 18:15 etcd-key.pem
    -rw-r--r-- 1 root root 1440 4月  29 18:15 etcd.pem
    
    # 拷贝到etcd服务器
    
    # etcd-1 
    cp etcd*.pem /etc/kubernetes/ssl/
    
    # etcd-2
    scp etcd*.pem root@192.168.1.42:/etc/kubernetes/ssl/
    
    # etcd-3
    scp etcd*.pem root@192.168.1.42:/etc/kubernetes/ssl/
    
    
    
    # 如果 etcd 非 root 用户，读取证书会提示没权限
    
    chmod 644 /etc/kubernetes/ssl/etcd-key.pem

修改 etcd 配置

#etcd1
    # [member]
    ETCD_NAME=etcd1
    ETCD_DATA_DIR="/var/lib/etcd/etcd1.etcd"
    ETCD_WAL_DIR="/var/lib/etcd/wal"
    ETCD_SNAPSHOT_COUNT="100"
    ETCD_HEARTBEAT_INTERVAL="100"
    ETCD_ELECTION_TIMEOUT="1000"
    ETCD_LISTEN_PEER_URLS="https://192.168.1.41:2380"
    ETCD_LISTEN_CLIENT_URLS="https://192.168.1.41:2379,http://127.0.0.1:2379"
    ETCD_MAX_SNAPSHOTS="5"
    ETCD_MAX_WALS="5"
    #ETCD_CORS=""
    
    # [cluster]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.41:2380"
    # if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
    ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.41:2380,etcd2=https://192.168.1.42:2380,etcd3=https://192.168.1.43:2380"
    ETCD_INITIAL_CLUSTER_STATE="new"
    ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
    ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.41:2379"
    #ETCD_DISCOVERY=""
    #ETCD_DISCOVERY_SRV=""
    #ETCD_DISCOVERY_FALLBACK="proxy"
    #ETCD_DISCOVERY_PROXY=""
    #ETCD_STRICT_RECONFIG_CHECK="false"
    #ETCD_AUTO_COMPACTION_RETENTION="0"
    
    # [proxy]
    #ETCD_PROXY="off"
    #ETCD_PROXY_FAILURE_WAIT="5000"
    #ETCD_PROXY_REFRESH_INTERVAL="30000"
    #ETCD_PROXY_DIAL_TIMEOUT="1000"
    #ETCD_PROXY_WRITE_TIMEOUT="5000"
    #ETCD_PROXY_READ_TIMEOUT="0"
    
    # [security]
    ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
    ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
    ETCD_CLIENT_CERT_AUTH="true"
    ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
    ETCD_AUTO_TLS="true"
    ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
    ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
    ETCD_PEER_CLIENT_CERT_AUTH="true"
    ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
    ETCD_PEER_AUTO_TLS="true"
    
    # [logging]
    #ETCD_DEBUG="false"
    # examples for -log-package-levels etcdserver=WARNING,security=DEBUG
    #ETCD_LOG_PACKAGE_LEVELS=""

#etcd2
    # [member]
    ETCD_NAME=etcd2
    ETCD_DATA_DIR="/var/lib/etcd/etcd2.etcd"
    ETCD_WAL_DIR="/var/lib/etcd/wal"
    ETCD_SNAPSHOT_COUNT="100"
    ETCD_HEARTBEAT_INTERVAL="100"
    ETCD_ELECTION_TIMEOUT="1000"
    ETCD_LISTEN_PEER_URLS="https://192.168.1.42:2380"
    ETCD_LISTEN_CLIENT_URLS="https://192.168.1.42:2379,http://127.0.0.1:2379"
    ETCD_MAX_SNAPSHOTS="5"
    ETCD_MAX_WALS="5"
    #ETCD_CORS=""
    
    # [cluster]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.42:2380"
    # if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
    ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.41:2380,etcd2=https://192.168.1.42:2380,etcd3=https://192.168.1.43:2380"
    ETCD_INITIAL_CLUSTER_STATE="new"
    ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
    ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.42:2379"
    #ETCD_DISCOVERY=""
    #ETCD_DISCOVERY_SRV=""
    #ETCD_DISCOVERY_FALLBACK="proxy"
    #ETCD_DISCOVERY_PROXY=""
    #ETCD_STRICT_RECONFIG_CHECK="false"
    #ETCD_AUTO_COMPACTION_RETENTION="0"
    
    # [proxy]
    #ETCD_PROXY="off"
    #ETCD_PROXY_FAILURE_WAIT="5000"
    #ETCD_PROXY_REFRESH_INTERVAL="30000"
    #ETCD_PROXY_DIAL_TIMEOUT="1000"
    #ETCD_PROXY_WRITE_TIMEOUT="5000"
    #ETCD_PROXY_READ_TIMEOUT="0"
    
    # [security]
    ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
    ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
    ETCD_CLIENT_CERT_AUTH="true"
    ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
    ETCD_AUTO_TLS="true"
    ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
    ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
    ETCD_PEER_CLIENT_CERT_AUTH="true"
    ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
    ETCD_PEER_AUTO_TLS="true"
    
    # [logging]
    #ETCD_DEBUG="false"
    # examples for -log-package-levels etcdserver=WARNING,security=DEBUG
    #ETCD_LOG_PACKAGE_LEVELS=""

#etcd3
    # [member]
    ETCD_NAME=etcd3
    ETCD_DATA_DIR="/var/lib/etcd/etcd3.etcd"
    ETCD_WAL_DIR="/var/lib/etcd/wal"
    ETCD_SNAPSHOT_COUNT="100"
    ETCD_HEARTBEAT_INTERVAL="100"
    ETCD_ELECTION_TIMEOUT="1000"
    ETCD_LISTEN_PEER_URLS="https://192.168.1.43:2380"
    ETCD_LISTEN_CLIENT_URLS="https://192.168.1.43:2379,http://127.0.0.1:2379"
    ETCD_MAX_SNAPSHOTS="5"
    ETCD_MAX_WALS="5"
    #ETCD_CORS=""
    
    # [cluster]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.43:2380"
    # if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
    ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.41:2380,etcd2=https://192.168.1.42:2380,etcd3=https://192.168.1.43:2380"
    ETCD_INITIAL_CLUSTER_STATE="new"
    ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
    ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.43:2379"
    #ETCD_DISCOVERY=""
    #ETCD_DISCOVERY_SRV=""
    #ETCD_DISCOVERY_FALLBACK="proxy"
    #ETCD_DISCOVERY_PROXY=""
    #ETCD_STRICT_RECONFIG_CHECK="false"
    #ETCD_AUTO_COMPACTION_RETENTION="0"
    
    # [proxy]
    #ETCD_PROXY="off"
    #ETCD_PROXY_FAILURE_WAIT="5000"
    #ETCD_PROXY_REFRESH_INTERVAL="30000"
    #ETCD_PROXY_DIAL_TIMEOUT="1000"
    #ETCD_PROXY_WRITE_TIMEOUT="5000"
    #ETCD_PROXY_READ_TIMEOUT="0"
    
    # [security]
    ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
    ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
    ETCD_CLIENT_CERT_AUTH="true"
    ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
    ETCD_AUTO_TLS="true"
    ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
    ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
    ETCD_PEER_CLIENT_CERT_AUTH="true"
    ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
    ETCD_PEER_AUTO_TLS="true"
    
    # [logging]
    #ETCD_DEBUG="false"
    # examples for -log-package-levels etcdserver=WARNING,security=DEBUG
    #ETCD_LOG_PACKAGE_LEVELS=""

**启动 etcd**  
分别启动 所有节点的 etcd 服务

systemctl enable etcd
    systemctl start etcd
    systemctl status etcd
    
    # 如果报错 请使用
    journalctl -f -t etcd  和 journalctl -u etcd 来定位问题

**验证 etcd 集群状态**

查看 etcd 集群状态

etcdctl --endpoints=https://192.168.1.41:2379,https://192.168.1.42:2379,https://192.168.1.43:2379\
            --cert-file=/etc/kubernetes/ssl/etcd.pem \
            --ca-file=/etc/kubernetes/ssl/ca.pem \
            --key-file=/etc/kubernetes/ssl/etcd-key.pem \
            cluster-health
    
    member 46af81257a43d032 is healthy: got healthy result from https://192.168.1.43:2379
    member d1511678caec8c7e is healthy: got healthy result from https://192.168.1.42:2379
    member df617030b25aba3c is healthy: got healthy result from https://192.168.1.41:2379
    cluster is healthy

查看 etcd 集群成员

[root@master-03 ~]# etcdctl --endpoints=https://192.168.1.41:2379,https://192.168.1.42:2379,https://192.168.1.43:2379\
    >         --cert-file=/etc/kubernetes/ssl/etcd.pem \
    >         --ca-file=/etc/kubernetes/ssl/ca.pem \
    >         --key-file=/etc/kubernetes/ssl/etcd-key.pem \
    >         member list
    46af81257a43d032: name=etcd3 peerURLs=https://192.168.1.43:2380 clientURLs=https://192.168.1.43:2379 isLeader=true
    d1511678caec8c7e: name=etcd2 peerURLs=https://192.168.1.42:2380 clientURLs=https://192.168.1.42:2379 isLeader=false
    df617030b25aba3c: name=etcd1 peerURLs=https://192.168.1.41:2380 clientURLs=https://192.168.1.41:2379 isLeader=false

## 配置 Kubernetes 集群 ##

安装组件(每个节点都要安装)

# 从github 上下载版本
    
    cd /tmp
    
    wget https://dl.k8s.io/v1.8.8/kubernetes-server-linux-amd64.tar.gz
    
    tar -xzvf kubernetes-server-linux-amd64.tar.gz
    
    cd kubernetes
    
    cp -r server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler,kubectl,kube-proxy,kubelet} /usr/local/bin/

**创建 admin 证书**

vi admin-csr.json
    {
      "CN": "admin",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "system:masters",
          "OU": "System"
        }
      ]
    }

# 生成 admin 证书和私钥
    cd /opt/ssl/
     /opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \   -config=/opt/ssl/config.json \   -profile=kubernetes admin-csr.json | /opt/local/cfssl/cfssljson -bare admin 
    
    # 查看生成
    
    [root@master-01 ssl]# ls -l admin*
    -rw-r--r-- 1 root root 1013 4月 29 18:36 admin.csr -rw-r--r-- 1 root root 231 4月 29 18:35 admin-csr.json -rw------- 1 root root 1679 4月 29 18:36 admin-key.pem -rw-r--r-- 1 root root 1407 4月 29 18:36 admin.pem 
    cp admin*.pem /etc/kubernetes/ssl/
    
    scp admin*.pem root@192.168.1.42:/etc/kubernetes/ssl/
    
    scp admin*.pem root@192.168.1.43:/etc/kubernetes/ssl/

**配置 kubectl kubeconfig 文件**  
生成证书相关的配置文件存储与 /root/.kube 目录中

# 配置 kubernetes 集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 
    
    # 配置 客户端认证
    
    kubectl config set-credentials admin \
      --client-certificate=/etc/kubernetes/ssl/admin.pem \   --embed-certs=true \   --client-key=/etc/kubernetes/ssl/admin-key.pem 
    
    
    kubectl config set-context kubernetes \
      --cluster=kubernetes \   --user=admin 
    
    kubectl config use-context kubernetes
    
    mkdir ~/.kube/
    scp -r ~/.kube/ root@192.168.1.42:~/.kube/
    
    scp ~/.kube/ root@192.168.1.43:~/.kube/

**创建 kubernetes 证书**

cd /opt/ssl
    
    vi kubernetes-csr.json
    
    {
      "CN": "kubernetes",
      "hosts": [
        "127.0.0.1",
        "192.168.1.41",
        "192.168.1.42",
        "192.168.1.43",
        "192.254.0.1",
        "kubernetes",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }
    
    ## 这里 hosts 字段中 三个 IP 分别为 127.0.0.1 本机， 192.168.1.41 和 192.168.1.42,192.168.1.43 为 Master 的IP，多个Master需要写多个 192.254.0.1 为 kubernetes SVC 的 IP， 一般是 部署网络的第一个IP , 如: 192.254.0.1 ， 在启动完成后，我们使用 kubectl get svc ， 就可以查看到

生成 kubernetes 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \   -config=/opt/ssl/config.json \   -profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes 
    # 查看生成
    
    [root@master-01 ssl]# ls -l kubernetes*
    -rw-r--r-- 1 root root 1261 4月 29 18:51 kubernetes.csr -rw-r--r-- 1 root root 479 4月 29 18:50 kubernetes-csr.json -rw------- 1 root root 1675 4月 29 18:51 kubernetes-key.pem -rw-r--r-- 1 root root 1635 4月 29 18:51 kubernetes.pem 
    
    # 拷贝到目录
    cp kubernetes*.pem /etc/kubernetes/ssl/
    
    scp kubernetes*.pem root@192.168.1.42:/etc/kubernetes/ssl/
    
    scp kubernetes*.pem root@192.168.1.43:/etc/kubernetes/ssl/

**配置 kube-apiserver**  
kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求，kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token 一致，如果一致则自动为 kubelet生成证书和秘钥。

# 生成 token
    
    [root@master-01 ssl]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
    aeaddee30713d7c68ef64deb71c0ac24
    
    
    # 创建 token.csv 文件
    
    cd /opt/ssl
    
    vi token.csv
    
    aeaddee30713d7c68ef64deb71c0ac24,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
    
    
    # 拷贝
    
    cp token.csv /etc/kubernetes/
    
    scp token.csv root@192.168.1.42:/etc/kubernetes/
    
    scp token.csv root@192.168.1.42:/etc/kubernetes/

# 生成高级审核配置文件
    
    cd /etc/kubernetes
    
    
    cat >> audit-policy.yaml <<EOF
    # Log all requests at the Metadata level.
    apiVersion: audit.k8s.io/v1beta1
    kind: Policy
    rules:
    - level: Metadata
    EOF
    
    
    
    
    # 拷贝
    
    scp audit-policy.yaml root@192.168.1.42:/etc/kubernetes/
    
    scp audit-policy.yaml root@192.168.1.43:/etc/kubernetes/

创建 kube-apiserver.service 文件

# 自定义 系统 service 文件一般存于 /etc/systemd/system/ 下
    # 配置为 各自的本地 IP
    
    vi /etc/systemd/system/kube-apiserver.service
    
    [Unit]
    Description=Kubernetes API Server
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=network.target
    
    [Service]
    User=root
    ExecStart=/usr/local/bin/kube-apiserver \
      --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \   --advertise-address=192.168.1.41 \   --allow-privileged=true \   --apiserver-count=3 \   --audit-policy-file=/etc/kubernetes/audit-policy.yaml \   --audit-log-maxage=30 \   --audit-log-maxbackup=3 \   --audit-log-maxsize=100 \   --audit-log-path=/var/log/kubernetes/audit.log \   --authorization-mode=Node,RBAC \   --bind-address=0.0.0.0 \   --secure-port=6443 \   --client-ca-file=/etc/kubernetes/ssl/ca.pem \   --enable-swagger-ui=true \   --etcd-cafile=/etc/kubernetes/ssl/ca.pem \   --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \   --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \   --etcd-servers=https://192.168.1.41:2379,https://192.168.1.42:2379,https://192.168.1.43:2379 \   --event-ttl=1h \   --kubelet-https=true \   --insecure-bind-address=127.0.0.1 \   --insecure-port=8080 \   --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \   --service-cluster-ip-range=192.254.0.0/16 \   --service-node-port-range=30000-32000 \   --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \   --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \   --enable-bootstrap-token-auth \   --token-auth-file=/etc/kubernetes/token.csv \   --v=2 Restart=on-failure
    RestartSec=5
    Type=notify
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

# k8s 1.8 添加 --authorization-mode=Node
    # k8s 1.8 添加 --admission-control=NodeRestriction
    # k8s 1.8 添加 --audit-policy-file=/etc/kubernetes/audit-policy.yaml
    
    # 这里面要注意的是 --service-node-port-range=30000-32000
    # 这个地方是 映射外部端口时 的端口范围，随机映射也在这个范围内映射，指定映射端口必须也在这个范围内。

启动 kube-apiserver

systemctl daemon-reload
    systemctl enable kube-apiserver
    systemctl start kube-apiserver
    systemctl status kube-apiserver

**配置 kube-controller-manager**

# 创建 kube-controller-manager.service 文件
    
    vi /etc/systemd/system/kube-controller-manager.service
    [Unit]
    Description=Kubernetes Controller Manager
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    
    [Service]
    ExecStart=/usr/local/bin/kube-controller-manager \
      --address=0.0.0.0 \   --master=http://127.0.0.1:8080 \   --allocate-node-cidrs=true \   --service-cluster-ip-range=192.254.0.0/16 \   --cluster-cidr=192.233.0.0/16 \   --cluster-name=kubernetes \   --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \   --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \   --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \   --root-ca-file=/etc/kubernetes/ssl/ca.pem \   --leader-elect=true \   --v=2 Restart=on-failure
    RestartSec=5
    
    [Install]
    WantedBy=multi-user.target

启动 kube-controller-manager

systemctl daemon-reload
    systemctl enable kube-controller-manager
    systemctl start kube-controller-manager
    systemctl status kube-controller-manager

**配置 kube-scheduler**

# 创建 kube-cheduler.service 文件
    
    vi /etc/systemd/system/kube-scheduler.service
    [Unit]
    Description=Kubernetes Scheduler
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    
    [Service]
    ExecStart=/usr/local/bin/kube-scheduler \
      --address=0.0.0.0 \   --master=http://127.0.0.1:8080 \   --leader-elect=true \   --v=2 Restart=on-failure
    RestartSec=5
    
    [Install]
    WantedBy=multi-user.target

启动 kube-scheduler

systemctl daemon-reload
    systemctl enable kube-scheduler
    systemctl start kube-scheduler
    systemctl status kube-scheduler

**验证 Master 节点**

[root@master-01 kubernetes]# kubectl get cs
    NAME                 STATUS    MESSAGE              ERROR
    controller-manager   Healthy   ok                   
    scheduler            Healthy   ok                   
    etcd-2               Healthy   {
       "health": "true"}   
    etcd-0               Healthy   {
       "health": "true"}   
    etcd-1               Healthy   {
       "health": "true"} 
    
    
    [root@master-02 kubernetes]# kubectl get cs
    NAME                 STATUS    MESSAGE              ERROR
    controller-manager   Healthy   ok                   
    scheduler            Healthy   ok                   
    etcd-1               Healthy   {
       "health": "true"}   
    etcd-0               Healthy   {
       "health": "true"}   
    etcd-2               Healthy   {
       "health": "true"}  
    
    
    [root@master-03 kubernetes]# kubectl get cs
    NAME                 STATUS    MESSAGE              ERROR
    scheduler            Healthy   ok                   
    controller-manager   Healthy   ok                   
    etcd-2               Healthy   {
       "health": "true"}   
    etcd-0               Healthy   {
       "health": "true"}   
    etcd-1               Healthy   {
       "health": "true"}

**配置 kubelet**  
kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要先将 bootstrap token 文件中的 kubelet-bootstrap 用户赋予 system:node-bootstrapper 角色，然后 kubelet 才有权限创建认证请求(certificatesigningrequests)。

# 先创建认证请求
    # user 为 master 中 token.csv 文件里配置的用户
    # 只需创建一次就可以
    
    kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

创建 kubelet kubeconfig 文件

# 配置集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 \   --kubeconfig=bootstrap.kubeconfig 
    # 配置客户端认证
    
    kubectl config set-credentials kubelet-bootstrap \
      --token=aeaddee30713d7c68ef64deb71c0ac24 \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \   --user=kubelet-bootstrap \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
    
    # 拷贝生成的 bootstrap.kubeconfig 文件
    
    mv bootstrap.kubeconfig /etc/kubernetes/

**创建 kubelet.service 文件**

# 创建 kubelet 目录
    
    > 配置为 node 本机 IP
    
    mkdir /var/lib/kubelet
    
    vi /etc/systemd/system/kubelet.service
    
    
    [Unit]
    Description=Kubernetes Kubelet
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=docker.service
    Requires=docker.service
    
    [Service]
    WorkingDirectory=/var/lib/kubelet
    ExecStart=/usr/local/bin/kubelet \
      --cgroup-driver=systemd \   --hostname-override=master-01 \   --pod-infra-container-image=jicki/pause-amd64:3.0 \   --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \   --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \   --cert-dir=/etc/kubernetes/ssl \   --cluster_dns=192.254.0.2 \   --cluster_domain=cluster.local. \   --hairpin-mode promiscuous-bridge \   --allow-privileged=true \   --fail-swap-on=false \   --serialize-image-pulls=false \   --logtostderr=true \   --max-pods=512 \   --v=2 
    [Install]
    WantedBy=multi-user.target

# 如上配置:
    master-01    本机hostname
    192.254.0.2       预分配的 dns 地址
    cluster.local.   为 kubernetes 集群的 domain
    jicki/pause-amd64:3.0  这个是 pod 的基础镜像，既 gcr 的 gcr.io/google_containers/pause-amd64:3.0 镜像， 下载下来修改为自己的仓库中的比较快。

启动kubelet

systemctl daemon-reload
    systemctl enable kubelet
    systemctl start kubelet
    systemctl status kubelet
    
    # 如果报错 请使用
    journalctl -f -t kubelet  和 journalctl -u kubelet 来定位问题

**配置 TLS 认证**

[root@master-03 kubernetes]# kubectl get csr
    NAME                                                  AGE      REQUESTOR           CONDITION
    node-csr-9EWT-ZMXLm83nfGkXUWbf5BfAbHT4DoIwiUzKdMxhe8   4m        kubelet-bootstrap   Pending
    node-csr-GOarlm9yyjQdX2QglkvG6_T8QmSo2EvEqULxYQY3TFg   14s       kubelet-bootstrap   Pending
    node-csr-H2srX4AqkJU-FtOPco1jMJTqrhRnOF3ogLulLVfAYeE   31s       kubelet-bootstrap   Pending
    
    # 增加 认证
    
    kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve

**验证 nodes**

[root@master-03 kubernetes]# kubectl get nodes
    NAME        STATUS    ROLES     AGE       VERSION
    master-01   Ready     <none>    2m        v1.8.8
    master-02   Ready     <none>    1m        v1.8.8
    master-03   Ready     <none>    15s       v1.8.8
    
    
    [root@master-03 kubernetes]# ls /etc/kubernetes/kubelet.kubeconfig 
    /etc/kubernetes/kubelet.kubeconfig
    
    root@master-03 kubernetes]# ls /etc/kubernetes/ssl/kubelet*
    /etc/kubernetes/ssl/kubelet-client.crt  /etc/kubernetes/ssl/kubelet-client.key  /etc/kubernetes/ssl/kubelet.crt  /etc/kubernetes/ssl/kubelet.key

**配置 kube-proxy**

创建 kube-proxy 证书

cd /opt/ssl
     vi kube-proxy-csr.json
    
    {
      "CN": "system:kube-proxy",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

生成 kube-proxy 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/opt/ssl/config.json \
      -profile=kubernetes  kube-proxy-csr.json | /opt/local/cfssl/cfssljson -bare kube-proxy
    
    # 查看生成
    ls kube-proxy*
    kube-proxy.csr  kube-proxy-csr.json  kube-proxy-key.pem  kube-proxy.pem
    
    # 拷贝到目录
    cp kube-proxy*.pem /etc/kubernetes/ssl/
    
    scp kube-proxy*.pem root@192.168.1.42:/etc/kubernetes/ssl/
    
    scp kube-proxy*.pem root@192.168.1.43:/etc/kubernetes/ssl/

创建 kube-proxy kubeconfig 文件

# 配置集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 \   --kubeconfig=kube-proxy.kubeconfig 
    
    # 配置客户端认证
    
    kubectl config set-credentials kube-proxy \
      --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \   --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \   --embed-certs=true \   --kubeconfig=kube-proxy.kubeconfig 
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \   --user=kube-proxy \   --kubeconfig=kube-proxy.kubeconfig 
    
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
    
    # 拷贝到目录
    mv kube-proxy.kubeconfig /etc/kubernetes/

创建 kube-proxy.service 文件

# 创建 kube-proxy 目录
    
    mkdir -p /var/lib/kube-proxy
    
    
    vi /etc/systemd/system/kube-proxy.service
    [Unit]
    Description=Kubernetes Kube-Proxy Server
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=network.target
    
    [Service]
    WorkingDirectory=/var/lib/kube-proxy
    ExecStart=/usr/local/bin/kube-proxy \
      --bind-address=192.168.1.41 \   --hostname-override=master-01 \   --cluster-cidr=192.254.0.0/16 \   --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \   --logtostderr=true \   --v=2 Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

启动 kube-proxy

systemctl daemon-reload
    systemctl enable kube-proxy
    systemctl start kube-proxy
    systemctl status kube-proxy

# 如果报错 请使用
    journalctl -f -t kube-proxy  和 journalctl -u kube-proxy 来定位问题

## Node 端 ##

单 Node 部分 需要部署的组件有 docker calico kubectl kubelet kube-proxy 这几个组件。 Node 节点 基于 Nginx 负载 API 做 Master HA

# master 之间除 api server 以外其他组件通过 etcd 选举，api server 默认不作处理；在每个 node 上启动一个 nginx，每个 nginx 反向代理所有 api server，node 上 kubelet、kube-proxy 连接本地的 nginx 代理端口，当 nginx 发现无法连接后端时会自动踢掉出问题的 api server，从而实现 api server 的 HA

安装组件

cd /tmp
    
    wget https://dl.k8s.io/v1.8.3/kubernetes-server-linux-amd64.tar.gz
    
    tar -xzvf kubernetes-server-linux-amd64.tar.gz
    
    cd kubernetes
    
    cp -r server/bin/{kube-proxy,kubelet,kubectl} /usr/local/bin/

# ALL node
    
    mkdir -p /etc/kubernetes/ssl/
    
    scp ca.pem kube-proxy.pem kube-proxy-key.pem root@192.168.1.44:/etc/kubernetes/ssl/

配置 kubelet or kube-proxy

# kubelet
    
    # 首先 创建 kubelet kubeconfig 文件
    
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置客户端认证
    
    kubectl config set-credentials kubelet-bootstrap \
      --token=aeaddee30713d7c68ef64deb71c0ac24 \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \   --user=kubelet-bootstrap \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
    
    # 拷贝生成的 bootstrap.kubeconfig 文件
    
    mv bootstrap.kubeconfig /etc/kubernetes/

# 创建 kube-proxy kubeconfig 文件
    
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 \   --kubeconfig=kube-proxy.kubeconfig 
    
    # 配置客户端认证
    
    kubectl config set-credentials kube-proxy \
      --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \   --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \   --embed-certs=true \   --kubeconfig=kube-proxy.kubeconfig 
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \   --user=kube-proxy \   --kubeconfig=kube-proxy.kubeconfig 
    
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
    
    # 拷贝到目录
    mv kube-proxy.kubeconfig /etc/kubernetes/

## 创建Nginx 代理 ##

在每个 node 都必须创建一个 Nginx 代理， 这里特别注意， 当 Master 也做为 Node 的时候 不需要配置 Nginx-proxy

# 创建配置目录
    mkdir -p /etc/nginx
    
    # 写入代理配置
    cat << EOF >> /etc/nginx/nginx.conf
    error_log stderr notice;
    
    worker_processes auto;
    events {
      multi_accept on;
      use epoll;
      worker_connections 1024;
    }
    
    stream {
        upstream kube_apiserver {
            least_conn;
            server 192.168.1.43:6443;
            server 192.168.1.42:6443;
            server 192.168.1.41:6443;
        }
    
        server {
            listen        0.0.0.0:6443;
            proxy_pass    kube_apiserver;
            proxy_timeout 10m;
            proxy_connect_timeout 1s;
        }
    }
    EOF
    
    # 更新权限
    chmod +r /etc/nginx/nginx.conf

# 配置 Nginx 基于 docker 进程，然后配置 systemd 来启动
    
    cat << EOF >> /etc/systemd/system/nginx-proxy.service
    [Unit]
    Description=kubernetes apiserver docker wrapper
    Wants=docker.socket
    After=docker.service
    
    [Service]
    User=root
    PermissionsStartOnly=true
    ExecStart=/usr/bin/docker run -p 127.0.0.1:6443:6443 \\
                                  -v /etc/nginx:/etc/nginx \\
                                  --name nginx-proxy \\
                                  --net=host \\
                                  --restart=on-failure:5 \\
                                  --memory=512M \\
                                  nginx:1.13.5-alpine
    ExecStartPre=-/usr/bin/docker rm -f nginx-proxy
    ExecStop=/usr/bin/docker stop nginx-proxy
    Restart=always
    RestartSec=15s
    TimeoutStartSec=30s
    
    [Install]
    WantedBy=multi-user.target
    EOF

# 启动 Nginx
    
    systemctl daemon-reload
    systemctl start nginx-proxy
    systemctl enable nginx-proxy
    systemctl status nginx-proxy
    
    
    # 重启 Node 的 kubelet 与 kube-proxy
    
    systemctl restart kubelet
    systemctl status kubelet
    
    systemctl restart kube-proxy
    systemctl status kube-proxy

Master 配置 TLS 认证

# 查看 csr 的名称
    
    [root@master-03 ssl]# kubectl get csr
    NAME                                                   AGE       REQUESTOR           CONDITION
    node-csr-9EWT-ZMXLm83nfGkXUWbf5BfAbHT4DoIwiUzKdMxhe8   13h       kubelet-bootstrap   Approved,Issued
    node-csr-GOarlm9yyjQdX2QglkvG6_T8QmSo2EvEqULxYQY3TFg   13h       kubelet-bootstrap   Approved,Issued
    node-csr-H2srX4AqkJU-FtOPco1jMJTqrhRnOF3ogLulLVfAYeE   13h       kubelet-bootstrap   Approved,Issued
    node-csr-mFuz_pMF88dpDjLhpo5oh3_4Y9Wdh1YLZjIGRo5lwZ8   19s       kubelet-bootstrap   Pending
    
    kubectl certificate approve node-csr-mFuz_pMF88dpDjLhpo5oh3_4Y9Wdh1YLZjIGRo5lwZ8
    
    
    # 增加 认证
    
    [root@k8s-master-64 ~]# kubectl certificate approve NAME

[root@master-03 ssl]# kubectl get nodes 
    NAME        STATUS    ROLES     AGE       VERSION
    master-01   Ready     <none>    13h       v1.8.8
    master-02   Ready     <none>    13h       v1.8.8
    master-03   Ready     <none>    13h       v1.8.8
    node-01     Ready     <none>    5m        v1.8.8

## 配置 Calico 网络 ##

[Calico官网地址][Calico]

# 下载 yaml 文件
    
    wget http://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml wget http://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/rbac.yaml # 下载 镜像 # 国外镜像 有墙 quay.io/calico/node:v2.6.0 quay.io/calico/cni:v1.11.0 quay.io/calico/kube-controllers:v1.0.0 # 国内镜像 jicki/node:v2.6.0 jicki/cni:v1.11.0 jicki/kube-controllers:v1.0.0 docker pull jicki/node:v2.6.0 && docker tag jicki/node:v2.6.0 quay.io/calico/node:v2.6.0 docker pull jicki/cni:v1.11.0 && docker tag jicki/cni:v1.11.0 quay.io/calico/cni:v1.11.0 docker pull jicki/kube-controllers:v1.0.0 && docker tag jicki/kube-controllers:v1.0.0 quay.io/calico/kube-controllers:v1.0.0

**配置 calico**

vi calico.yaml
    
    # 注意修改如下选项:
    
    
      etcd_endpoints: "https://192.168.1.41:2379,https://192.168.1.42:2379,https://192.168.1.43:2379"
    
        etcd_ca: "/calico-secrets/etcd-ca"  
        etcd_cert: "/calico-secrets/etcd-cert"
        etcd_key: "/calico-secrets/etcd-key"  
    
    
    # 这里面要写入 base64 的信息
    
    
    
    data:
      etcd-key: (cat /etc/kubernetes/ssl/etcd-key.pem | base64 | tr -d '\n')
      etcd-cert: (cat /etc/kubernetes/ssl/etcd.pem | base64 | tr -d '\n')
      etcd-ca: (cat /etc/kubernetes/ssl/ca.pem | base64 | tr -d '\n')
    
    
        - name: CALICO_IPV4POOL_CIDR
          value: "192.233.0.0/16"

导入 yaml 文件

[root@master-01 calico]# ls -l
    总用量 24
    -rw-r--r-- 1 root root 17855 4月  30 08:50 calico.yaml
    -rw-r--r-- 1 root root  1131 4月  26 21:45 rbac.yaml
    [root@master-01 calico]# pwd
    /root/calico
    [root@master-01 calico]# ls -l
    总用量 24
    -rw-r--r-- 1 root root 17855 4月  30 08:50 calico.yaml
    -rw-r--r-- 1 root root  1131 4月  26 21:45 rbac.yaml
    [root@master-01 calico]# kubectl apply -f .
    configmap "calico-config" created
    secret "calico-etcd-secrets" created
    daemonset "calico-node" created
    deployment "calico-kube-controllers" created
    deployment "calico-policy-controller" created
    serviceaccount "calico-kube-controllers" created
    serviceaccount "calico-node" created
    clusterrole "calico-kube-controllers" created
    clusterrolebinding "calico-kube-controllers" created
    clusterrole "calico-node" created
    clusterrolebinding "calico-node" created

查看calico是否启动成功

[root@master-01 calico]# kubectl get pods -n kube-system 
    NAME                                      READY     STATUS    RESTARTS   AGE
    calico-kube-controllers-565ff95f7-28zr8   1/1       Running   0          7m
    calico-node-fggk6                         2/2       Running   0          7m
    calico-node-jmpbc                         2/2       Running   0          7m
    calico-node-pld72                         2/2       Running   0          7m
    calico-node-q5tbz                         2/2       Running   0          7m

配置 kubelet.service

vi /etc/systemd/system/kubelet.service
    
    # 增加 如下配置
    
      --network-plugin=cni \
    
    
    # 重新加载配置
    systemctl daemon-reload
    systemctl restart kubelet.service
    systemctl status kubelet.service

安装 Calicoctl

cd /usr/local/bin/
    
    wget -c  https://github.com/projectcalico/calicoctl/releases/download/v1.6.1/calicoctl
    
    chmod +x calicoctl
    
    apiVersion: v1
    kind: calicoApiConfig
    metadata:
    spec:
     datastoreType: "etcdv3"
     etcdEndpoints: "https://192.168.1.41:2379,https://192.168.1.42:2379,https://192.168.1.43:2379"
     etcdKeyFile: "/etc/kubernetes/ssl/etcd-key.pem"
     etcdCertFile: "/etc/kubernetes/ssl/etcd.pem"
     etcdCACertFile: "/etc/kubernetes/ssl/ca.pem"
    
    [root@master-01 ~]# calicoctl node status
    Calico process is running.
    
    IPv4 BGP status +--------------+-------------------+-------+----------+-------------+
    | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+-------------------+-------+----------+-------------+
    | 192.168.1.42 | node-to-node mesh | up    | 01:26:03 | Established |
    | 192.168.1.44 | node-to-node mesh | up    | 01:30:00 | Established |
    | 192.168.1.43 | node-to-node mesh | up | 01:30:03 | Established | +--------------+-------------------+-------+----------+-------------+
    
    IPv6 BGP status
    No IPv6 peers found.

## 测试集群 ##

# 创建一个 nginx deplyment
    
    apiVersion: extensions/v1beta1 
    kind: Deployment 
    metadata: 
      name: nginx-dm
    spec: 
      replicas: 3
      template: 
        metadata: 
          labels: 
            name: nginx 
        spec: 
          containers: 
            - name: nginx 
              image: nginx:alpine 
              imagePullPolicy: IfNotPresent
              ports: 
                - containerPort: 80
    
    ---
    
    apiVersion: v1 
    kind: Service
    metadata: 
      name: nginx-svc 
    spec: 
      ports: 
        - port: 80
          targetPort: 80
          protocol: TCP 
      selector: 
        name: nginx

[root@master-01 nginx]# kubectl get pods -owide
    NAME                        READY     STATUS    RESTARTS   AGE       IP                NODE
    nginx-dm-55b58f68b6-2xzfj   1/1       Running   0          41s       192.168.222.1     master-02
    nginx-dm-55b58f68b6-5cfsn   1/1       Running   0          41s       192.168.184.65    master-01
    nginx-dm-55b58f68b6-jqwl2   1/1       Running   0          41s       192.168.133.129   master-03

# 在 node 里 curl
    
    [root@node-01 kubernetes]# curl 192.168.184.65
    <!DOCTYPE html>
    <html>
    <head>
    <title>Welcome to nginx!</title>
    <style> body { width: 35em; margin: 0 auto; font-family: Tahoma, Verdana, Arial, sans-serif; } </style>
    </head>
    <body>
    <h1>Welcome to nginx!</h1>
    <p>If you see this page, the nginx web server is successfully installed and
    working. Further configuration is required.</p>
    
    <p>For online documentation and support please refer to
    <a href="http://nginx.org/">nginx.org</a>.<br/>
    Commercial support is available at
    <a href="http://nginx.com/">nginx.com</a>.</p>
    
    <p><em>Thank you for using nginx.</em></p>
    </body>
    </html>

## 配置 KubeDNS ##

[官方 github yaml][github yaml]

下载镜像

# 官方镜像
    k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10
    k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10
    k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10

下载 yaml 文件

curl -O https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/kube-dns.yaml.base
    
    # 修改后缀
    
    mv kube-dns.yaml.base kube-dns.yaml

修改 kube-dns.yaml

1. # clusterIP: __PILLAR__DNS__SERVER__ 修改为我们之前定义的 dns IP 192.254.0.2
    
    2. # 修改 --domain=__PILLAR__DNS__DOMAIN__.   为 我们之前 预定的 domain 名称 --domain=cluster.local.
    
    3. # 修改 --server=/__PILLAR__DNS__DOMAIN__/127.0.0.1#10053  中 domain 为我们之前预定的 --server=/cluster.local./127.0.0.1#10053
    
    4. # 修改 --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__, 中的 domain 为我们之前预定的  --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local.,
    
    5. # 修改 --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__,  中的 domain 为我们之前预定的  --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local.,

# 导入
    
    [root@master-01 nginx]# kubectl create -f kube-dns.yaml
    service "kube-dns" created
    serviceaccount "kube-dns" created
    configmap "kube-dns" created
    deployment "kube-dns" created

查看 kubedns 服务

[root@master-01 ~]# kubectl get pods -n kube-system -owide
    NAME                                      READY     STATUS    RESTARTS   AGE       IP              NODE
    calico-kube-controllers-565ff95f7-28zr8   1/1       Running   0          54m       10.0.2.15       node-01
    calico-node-fggk6                         2/2       Running   0          54m       192.168.1.41    master-01
    calico-node-jmpbc                         2/2       Running   0          54m       10.0.2.15       node-01
    calico-node-pld72                         2/2       Running   0          54m       192.168.1.42    master-02
    calico-node-q5tbz                         2/2       Running   0          54m       192.168.1.43    master-03
    kube-dns-b9b47f6c4-5bb7c                  3/3       Running   0          17m       192.168.190.1   node-01

**验证 dns 服务**  
在验证 dns 之前，在 dns 未部署之前创建的 pod 与 deployment 等，都必须删除，重新部署，否则无法解析

[root@master-01 nginx]# kubectl delete -f nginx-deployment.yaml 
    deployment "nginx-dm" deleted
    service "nginx-svc" deleted
    [root@master-01 nginx]# kubectl create -f nginx-deployment.yaml 
    deployment "nginx-dm" created
    service "nginx-svc" created
    [root@master-01 nginx]# kubectl get pods 
    NAME                        READY     STATUS    RESTARTS   AGE
    nginx-dm-55b58f68b6-6dx8w   1/1       Running   0          5s
    nginx-dm-55b58f68b6-lwhk4   1/1       Running   0          5s
    nginx-dm-55b58f68b6-xdr4n   1/1       Running   0          5s
    
    [root@master-01 nginx]# kubectl get svc -owide
    NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE       SELECTOR
    kubernetes   ClusterIP   192.254.0.1     <none>        443/TCP   15h       <none>
    nginx-svc    ClusterIP   192.254.66.27   <none>        80/TCP    46s       name=nginx
    
    
    # 创建一个 pods 来测试一下 dns 
    [root@master-01 nginx]# vi test-dns.yaml
    
    apiVersion: v1
    kind: Pod
    metadata:
      name: alpine
    spec:
      containers:
      - name: alpine
        image: alpine
        command:
        - sh
        - -c
        - while true; do sleep 1; done
    
    
    [root@master-01 nginx]# 
    [root@master-01 nginx]# kubectl create -f test-dns.yaml 
    pod "alpine" created
    [root@master-01 nginx]# kubectl get pods -owide
    NAME                        READY     STATUS    RESTARTS   AGE       IP                NODE
    alpine                      1/1       Running   0          9s        192.168.222.3     master-02
    nginx-dm-55b58f68b6-6dx8w   1/1       Running   0          2m        192.168.184.66    master-01
    nginx-dm-55b58f68b6-lwhk4   1/1       Running   0          2m        192.168.222.2     master-02
    nginx-dm-55b58f68b6-xdr4n   1/1       Running   0          2m        192.168.133.130   master-03
    
    # 测试
    
    [root@master-01 nginx]# kubectl exec -it alpine nslookup nginx-svc
    nslookup: can't resolve '(null)': Name does not resolve Name: nginx-svc Address 1: 192.254.66.27 nginx-svc.default.svc.cluster.local

## 部署 Ingress 与 Dashboard ##

[官方 dashboard 的github][dashboard _github]

下载 dashboard 镜像

k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3

下载yaml文件

全部的yaml文件在https://github.com/kubernetes/kubernetes/blob/v1.8.8/cluster/addons/dashboard/dashboard-secret.yaml 路径下

导入 yaml

# 导入文件
    
    [root@master-01 dashboard]# kubectl apply -f .
    configmap "kubernetes-dashboard-settings" created
    serviceaccount "kubernetes-dashboard" created
    deployment "kubernetes-dashboard" created
    role "kubernetes-dashboard-minimal" created
    rolebinding "kubernetes-dashboard-minimal" created
    secret "kubernetes-dashboard-certs" created
    service "kubernetes-dashboard" created
    
    
    
    
    # 查看 svc 与 pod
    
    [root@master-01 dashboard]# kubectl get svc -n kube-system
    NAME                   TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)         AGE
    kube-dns               ClusterIP   192.254.0.2       <none>        53/UDP,53/TCP   1h
    kubernetes-dashboard   ClusterIP   192.254.235.241   <none>        443/TCP         3m
    [root@master-01 dashboard]# kubectl get pods -n kube-system -owide
    NAME                                      READY     STATUS    RESTARTS   AGE       IP               NODE
    calico-kube-controllers-565ff95f7-28zr8   1/1       Running   0          2h        10.0.2.15        node-01
    calico-node-fggk6                         2/2       Running   0          2h        192.168.1.41     master-01
    calico-node-jmpbc                         2/2       Running   0          2h        10.0.2.15        node-01
    calico-node-pld72                         2/2       Running   0          2h        192.168.1.42     master-02
    calico-node-q5tbz                         2/2       Running   0          2h        192.168.1.43     master-03
    kube-dns-b9b47f6c4-5bb7c                  3/3       Running   0          1h        192.168.190.1    node-01
    kubernetes-dashboard-768854d6dc-zb9rc     1/1       Running   2          2m        192.168.184.71   master-01

如何访问dashboard,请参考:[Accessing-Dashboard][]  
我是通过nodePort方式去访问

[root@master-01 dashboard]# kubectl -n kube-system edit service kubernetes-dashboard
    service "kubernetes-dashboard" edited
    [root@master-01 dashboard]# kubectl -n kube-system get service kubernetes-dashboard
    NAME                   TYPE       CLUSTER-IP        EXTERNAL-IP   PORT(S)         AGE
    kubernetes-dashboard   NodePort   192.254.235.241   <none>        443:31545/TCP   11m

访问masterip:31545 就可以了  
![dash][]

之所以出现以上的问题，是因为没有配置token或者kubeconfig，所以没权限去访问  
RBAC 的问题,解决办法拿到dashboard的token登陆就可以解决(需要base64解码):

root@master-01 ~]# kubectl get secret --all-namespaces
    NAMESPACE       NAME                                       TYPE                                  DATA      AGE
    default         default-token-k6hqr                        kubernetes.io/service-account-token   3         22h
    ingress-nginx   default-token-9v48d                        kubernetes.io/service-account-token   3         5h
    ingress-nginx   nginx-ingress-serviceaccount-token-lbhxx   kubernetes.io/service-account-token   3         5h
    kube-public     default-token-zgtrk                        kubernetes.io/service-account-token   3         22h
    kube-system     calico-etcd-secrets                        Opaque                                3         8h
    kube-system     calico-kube-controllers-token-d46bn        kubernetes.io/service-account-token   3         8h
    kube-system     calico-node-token-6s4hv                    kubernetes.io/service-account-token   3         8h
    kube-system     dashboard-token-g55wc                      kubernetes.io/service-account-token   3         6h
    kube-system     default-token-xbw22                        kubernetes.io/service-account-token   3         22h
    kube-system     heapster-token-wt55t                       kubernetes.io/service-account-token   3         25m
    kube-system     kube-dns-token-xbw5c                       kubernetes.io/service-account-token   3         7h
    kube-system     kubernetes-dashboard-certs                 Opaque                                0         5h
    kube-system     kubernetes-dashboard-key-holder            Opaque                                2         5h
    kube-system     kubernetes-dashboard-token-x8hvz           kubernetes.io/service-account-token   3         5h

[root@master-01 ~]# kubectl get secret -n kube-system dashboard-token-g55wc -oyaml
    apiVersion: v1
    data:
      ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR3akNDQXFxZ0F3SUJBZ0lVTXdzMHJpRi9WYzlKODhzWFNCV3NkN1JTQkNzd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1p6RUxNQWtHQTFVRUJoTUNRMDR4RVRBUEJnTlZCQWdUQ0ZOb1pXNWFhR1Z1TVJFd0R3WURWUVFIRXdoVAphR1Z1V21obGJqRU1NQW9HQTFVRUNoTURhemh6TVE4d0RRWURWUVFMRXdaVGVYTjBaVzB4RXpBUkJnTlZCQU1UCkNtdDFZbVZ5Ym1WMFpYTXdIaGNOTVRnd05ESTVNVEF3TWpBd1doY05Nak13TkRJNE1UQXdNakF3V2pCbk1Rc3cKQ1FZRFZRUUdFd0pEVGpFUk1BOEdBMVVFQ0JNSVUyaGxibHBvWlc0eEVUQVBCZ05WQkFjVENGTm9aVzVhYUdWdQpNUXd3Q2dZRFZRUUtFd05yT0hNeER6QU5CZ05WQkFzVEJsTjVjM1JsYlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1ClpYUmxjekNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFNUHVuWDIvdk4wUWJQdzcKNGo3cDBXNW5GdENRTmlxTkVENDZEbXpCNFA0SVRQRUk2Q1pyczFPTnlJWVhWUUN0djN2UG82d0N0cVFOeU5wbgpMcGg0M2p3YkprRHdEbnhKVEFzb1ZVeXU3Ym5YRVVTOGpiQ2I3cCtzNEFzVFQxQjNLWHdPMURFREFZRkFJZWtYClZMZWdVUUZhVVJyRndYMHJ4cGI0QlBLN04zbzQzK281VmhDa0ZOQTU3VjRFUytFNEZvcGQ2ekFEYTBreFo1QXgKanNLd2s3a20rUnV6NkRNeWplbnl5dXVtZzAxWGRnMU9PaXFpbjczWGs3NWFSZDF1aDZEbmdWSnlRQW9qSDRrawpHUmtqM1NsZGVYSFR2L0xPNmEvU1NiNW1Tczd1alJvUHZURmZUbk0ybWFXRmZCNytnVHZMcGxRRmJKMVdBelJRCklTbW5NNWNDQXdFQUFhTm1NR1F3RGdZRFZSMFBBUUgvQkFRREFnRUdNQklHQTFVZEV3RUIvd1FJTUFZQkFmOEMKQVFJd0hRWURWUjBPQkJZRUZHajFieVJKQlFJbDdRcWZQSjF6UVlrZENlMndNQjhHQTFVZEl3UVlNQmFBRkdqMQpieVJKQlFJbDdRcWZQSjF6UVlrZENlMndNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUFGL0Z5RHhiTmNZcTVMCjY3cTZValI2aE5NNHVpbVVVVVBwUkhhU2VUWmNDNlVGdndQeU5ZNzlmQThPSjJrVFViM0d3bmN6UjJ4bzNab1YKYitXNnBKUWI5TjVnRDV6ekJWelpHN05CQWVoL1pvc2dEZXczdjFpZUdmU0N6a0ZCWFJQRUVVRUlPYkphYWxFVwpRZThuOWJXSHNqNGI3L2VZVVhaMWVqRU1Wd2kvMTVubGNDT2dIZVpYWHNvQ0RmdmVBbUxhWm1LRVhyOHJhOU9OCk5STWRwcGg4eTJidmxUYnpvVGRoV3pRWjY0MUcySExYS2wzMXpVSzhEYnQ0THlpaEsxcjVpRHVRaEdRMENtTzcKVlVvdktsaVlmWURQWFVsdnBWZUN3S2grVWpNSDNIYjdtcWk3akJ2OTdnckxiQmh1UXlhaC9wK004ZHFRcDQ2LwpEMVlHNGRWOAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
      namespace: a3ViZS1zeXN0ZW0=
      token: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUpyZFdKbExYTjVjM1JsYlNJc0ltdDFZbVZ5Ym1WMFpYTXVhVzh2YzJWeWRtbGpaV0ZqWTI5MWJuUXZjMlZqY21WMExtNWhiV1VpT2lKa1lYTm9ZbTloY21RdGRHOXJaVzR0WnpVMWQyTWlMQ0pyZFdKbGNtNWxkR1Z6TG1sdkwzTmxjblpwWTJWaFkyTnZkVzUwTDNObGNuWnBZMlV0WVdOamIzVnVkQzV1WVcxbElqb2laR0Z6YUdKdllYSmtJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5elpYSjJhV05sTFdGalkyOTFiblF1ZFdsa0lqb2lOek15TlRFNE9ESXROR015TmkweE1XVTRMVGhrWm1JdE1EZ3dNREkzWmpWaE9EWXpJaXdpYzNWaUlqb2ljM2x6ZEdWdE9uTmxjblpwWTJWaFkyTnZkVzUwT210MVltVXRjM2x6ZEdWdE9tUmhjMmhpYjJGeVpDSjkuSjRYMHR4REVSZUxQNmZ4eFlKT2VJeUJucDRla0FSd3ZKc0ZFbXBxSFZtaEhKWUZadzVRcU5iY1k5ZEU0Y0kya1NqUmhFSFpTTzR5RHVfOGRwSzBzUlgtMjRpWmE0N05IVXVGZU0xTDl6cTZ5ODVkYXlTelg1N05BTVBpYng0MHNleE5BZ2VIZUNESmxQWThiQVhCTG90TGk5WmdRS2FNQldFOFJ2Y0pja1A5VlliVndCSmhtYVNxcXVXaHlwWXVVMmpiZWowVTQ0VmlUNy1xaUpSMTJQUUsycVgySVBxNVlOTmVpcElUdEFvaDVpZUstRmtJenVnMjRSYmxNTTBQYjY4S3VSUFFNbi1mcDk3cEZvS3lKWVhFc1NWQ1FWV2FlYlAtaGpzYlVvWm9iTWhMWWJQUFVWY0Z0cmltWExWS2lhZTRMNjZsRExiOUg5SUQ2ZzNocklB
    kind: Secret
    metadata:
      annotations:
        kubernetes.io/service-account.name: dashboard
        kubernetes.io/service-account.uid: 73251882-4c26-11e8-8dfb-080027f5a863
      creationTimestamp: 2018-04-30T03:27:45Z
      name: dashboard-token-g55wc
      namespace: kube-system
      resourceVersion: "76660"
      selfLink: /api/v1/namespaces/kube-system/secrets/dashboard-token-g55wc
      uid: 732bdb8a-4c26-11e8-8dfb-080027f5a863
    type: kubernetes.io/service-account-token

![dash01][]  
**部署 heapster 插件**  
下载安装文件

wget https://github.com/kubernetes/heapster/archive/v1.5.0.tar.gz
    tar xzvf ./v1.5.0.tar.gz
    cd ./heapster-1.5.0/
    
    kubectl create -f deploy/kube-config/influxdb/
    kubectl create -f deploy/kube-config/rbac/heapster-rbac.yaml

访问 grafana  
通过kube-proxy访问

kubectl proxy --address='192.168.1.43' --port=8086 --accept-hosts='^*$'
    Starting to serve on 192.168.1.43:8086

![gra][]

**部署 Nginx Ingress**  
Kubernetes 暴露服务的方式目前只有三种：LoadBlancer Service、NodePort Service、Ingress； 什么是 Ingress ? Ingress 就是利用 Nginx Haproxy 等负载均衡工具来暴露 Kubernetes 服务。

[官方 Nginx Ingress github][Nginx Ingress github]

配置 调度 node

# ingress 有多种方式 1. deployment 自由调度 replicas
                         2.  daemonset 全局调度 分配到所有node里
    
    
    # deployment 自由调度过程中，由于我们需要 约束 controller 调度到指定的 node 中，所以需要对 node 进行 label 标签
    
    
    # 默认如下:
    [root@master-01 dashboard]# kubectl get nodes --show-labels
    NAME        STATUS    ROLES     AGE       VERSION   LABELS
    master-01   Ready     <none>    16h       v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master-01
    master-02   Ready     <none>    16h       v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master-02
    master-03   Ready     <none>    16h       v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master-03
    node-01     Ready     <none>    2h        v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node-01
    
    
    # 对 41 与 42 打上 label
    
    [root@master-01 dashboard]# kubectl label nodes master-01 ingress=proxy
    node "master-01" labeled
    [root@master-01 dashboard]# kubectl label nodes master-02 ingress=proxy
    node "master-02" labeled
    [root@master-01 dashboard]# kubectl get nodes --show-labels
    NAME        STATUS    ROLES     AGE       VERSION   LABELS
    master-01   Ready     <none>    16h       v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,ingress=proxy,kubernetes.io/hostname=master-01
    master-02   Ready     <none>    16h       v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,ingress=proxy,kubernetes.io/hostname=master-02
    master-03   Ready     <none>    16h       v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=master-03
    node-01     Ready     <none>    2h        v1.8.8    beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=node-01

# 部署 Nginx backend , Nginx backend 用于统一转发 没有的域名 到指定页面。
    
    curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/namespace.yaml
    
    curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/default-backend.yaml
    
    [root@master-01 ingress]# kubectl create -f namespace.yaml default-backend.yaml 
    deployment "default-http-backend" created
    service "default-http-backend" created
    
    
    [root@master-01 ingress]# kubectl get deployment -n ingress-nginx
    NAME                   DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
    default-http-backend   1         1         1            1           4m
    
    [root@master-01 ingress]# kubectl get pods -n ingress-nginx -owide
    NAME                                    READY     STATUS    RESTARTS   AGE       IP              NODE
    default-http-backend-66b447d9cf-mmxrl   1/1       Running   0          3m        192.168.222.8   master-02

# 部署 Ingress RBAC 认证
    
    curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/rbac.yaml

# 部署 Ingress Controller 组件
    
    # 下载 yaml 文件
    
    curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/with-rbac.yaml
    
    
    # 上面 对 两个 node 打了 label 所以配置 replicas: 2
    # 修改 yaml 文件 增加 rbac 认证 , hostNetwork 还有 nodeSelector, 第二个 spec 下 增加。
    
    spec:
      replicas: 2
      ....
        spec:
          hostNetwork: true
          serviceAccountName: nginx-ingress-serviceaccount
          nodeSelector:
            ingress: proxy
        ....
    
    kubectl create -f with-rbac.yaml
    
    [root@master-01 ingress]# kubectl get pods -n ingress-nginx -owide
    NAME                                       READY    STATUS   RESTARTS  AGE      IP               NODE
    default-http-backend-66b447d9cf-mmxrl       1/1       Running   0          29m       192.168.222.8    master-02
    nginx-ingress-controller-6c995c59d9-dsmfw   1/1       Running   0          1m        192.168.222.10   master-02
    nginx-ingress-controller-6c995c59d9-rdvf7   1/1       Running   0          1m        192.168.184.72   master-01

# 查看我们原有的 svc
    [root@master-01 ingress]# kubectl get svc -n kube-system
    NAME                   TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)         AGE
    kube-dns               ClusterIP   192.254.0.2       <none>        53/UDP,53/TCP   2h
    kubernetes-dashboard   NodePort    192.254.235.241   <none>        443:31545/TCP   53m

# 创建 yaml 文件
    
    vi dashboard-ingress.yaml
    
    apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
      name: dashboard-ingress
      namespace: kube-system
    spec:
      rules:
      - host: dashboard.jicki.me
        http:
          paths:
          - backend:
              serviceName: kubernetes-dashboard
              servicePort: 80
    
    
    
    # 导入 yaml
    
    [root@master-01 ingress]# kubectl apply -f dashboard-ingress.yaml 
    ingress "dashboard-ingress" created
    
    [root@master-01 ingress]# kubectl get ingress -n kube-system -o wide
    NAME                HOSTS                ADDRESS   PORTS     AGE
    dashboard-ingress   dashboard.jicki.me             80        24s
    
    验证
     curl -I dashboard.jicki.me

END  
参考:  
[kubernetes-1.8.3][]

[Calico]: http://docs.projectcalico.org/v2.6/gettingstarted/kubernetes/installation/hosted/hosted
[github yaml]: https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/dns
[dashboard _github]: https://github.com/kubernetes/dashboard
[Accessing-Dashboard]: https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above
[dash]: /images/20220526/001338b34d88458fbfbd05f62158aad9.png
[dash01]: /images/20220526/a9ffc9c9e2e048debdb47c3b70291e46.png
[gra]: /images/20220526/e72de3bd964b40f1882772a6d9f166b4.png
[Nginx Ingress github]: https://github.com/kubernetes/ingress/tree/master/examples/deployment/nginx
[kubernetes-1.8.3]: https://jicki.me/kubernetes/2017/11/17/kubernetes-1.8.3.html