Kubernetes双向TLS配置-Centos7

桃扇骨 2022-07-15 09:17 195阅读 0赞

## 简介： ##

kubernetes 提供了多种安全认证机制，

其中对于集群通讯间可采用 TLS(https) 双向认证机制，也可采用基于 Token 或用户名密码的单向 tls 认证。

[kubernetes][] 一般在内网部署，采用私有 IP 地址进行通讯，权威 CA 貌似只能签署域名证书，所以我们采用自建CA。

## 目标： ##

在kubernetes集群中，自建CA并配置各组件间的双向TLS认证

## 环境： ##

> k8s-master：192.168.12.177
> 
> k8s-node01：192.168.12.178
> 
> 软件版本：
> 
> k8s：v1.3.4
> 
> 实施步骤：

1.master主机

1.1使用自建的CA签署证书

#新建CA
    [root@master ~]# cd key/
    [root@master key]# openssl genrsa -out ca-key.pem 2048
    [root@master key]# openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"
    [root@master key]# ll
    total 8
    -rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
    -rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
    
    #新建openssl.conf文件
    [root@master key]# vi openssl.cnf 
    [req]
    req_extensions = v3_req
    distinguished_name = req_distinguished_name
    [req_distinguished_name]
    [ v3_req ]
    basicConstraints = CA:FALSE
    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
    subjectAltName = @alt_names
    [alt_names]
    DNS.1 = kubernetes
    DNS.2 = kubernetes.default
    DNS.3 = kubernetes.default.svc
    DNS.4 = kubernetes.default.svc.test01.com
    IP.1 = 10.254.0.1
    IP.2 = 192.168.12.177
    
    #制作apiserver key
    [root@master key]# openssl genrsa -out apiserver-key.pem 2048
    [root@master key]# openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf
    
    #签署apiserver证书
    [root@master key]# openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
    [root@master key]# ll
    total 28
    -rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
    -rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
    -rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
    -rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
    -rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
    -rw-r--r-- 1 root root 17 Sep 21 11:12 ca.srl
    -rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf
    
    #新建node01的openssl.conf
    [root@master key]# vi worker-openssl-001.cnf
    [req]
    req_extensions = v3_req
    distinguished_name = req_distinguished_name
    [req_distinguished_name]
    [ v3_req ]
    basicConstraints = CA:FALSE
    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
    subjectAltName = @alt_names
    [alt_names]
    IP.1 = 192.168.12.178
    
    #制作node01 key
    [root@master key]# openssl genrsa -out k8s-node001-worker-key.pem 2048
    [root@master key]# openssl req -new -key k8s-node001-worker-key.pem -out k8s-node001-worker.csr -subj "/CN=k8s-node001" -config worker-openssl-001.cnf
    
    #签署node01证书
    [root@master key]# openssl x509 -req -in k8s-node001-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out k8s-node001-worker.pem -days 365 -extensions v3_req -extfile worker-openssl-001.cnf
    [root@master key]# ll
    total 44
    -rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
    -rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
    -rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
    -rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
    -rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
    -rw-r--r-- 1 root root 17 Sep 21 11:15 ca.srl
    -rw-r--r-- 1 root root 972 Sep 21 11:14 k8s-node001-worker.csr
    -rw-r--r-- 1 root root 1679 Sep 21 11:14 k8s-node001-worker-key.pem
    -rw-r--r-- 1 root root 1046 Sep 21 11:15 k8s-node001-worker.pem
    -rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf
    -rw-r--r-- 1 root root 262 Sep 21 11:13 worker-openssl-001.cnf

1.2master主机修改相关配置

#复制apiserver的pem文件到ssl目录
    [root@master key]# mkdir -p /etc/kubernetes/ssl
    [root@master key]# cd /etc/kubernetes/ssl
    [root@master ssl]# cp /root/key/ca.pem .
    [root@master ssl]# cp /root/key/apiserver.pem .
    [root@master ssl]# cp /root/key/apiserver-key.pem .
    
    #新建kubeconfig文件
    [root@master ~]# vi /etc/kubernetes/cm-kubeconfig.yaml 
    apiVersion: v1
    kind: Config
    clusters:
    - name: local
     cluster:
     certificate-authority: /etc/kubernetes/ssl/ca.pem
    users:
    - name: controllermanager
     user:
     client-certificate: /etc/kubernetes/ssl/apiserver.pem
     client-key: /etc/kubernetes/ssl/apiserver-key.pem
    contexts:
    - context:
     cluster: local
     user: controllermanager
     name: kubelet-context
    current-context: kubelet-context
    
    #按下面配置，修改各组件
    #因controllmanager、scheduler与apiserver在同一台主机上，也可以使用非安全端口8080）
    [root@master ~]# grep -v ^# /etc/kubernetes/config 
    KUBE_LOGTOSTDERR="--logtostderr=true"
    KUBE_LOG_LEVEL="--v=0"
    KUBE_ALLOW_PRIV="--allow-privileged=false"
    KUBE_MASTER="--master=https://192.168.12.177:6443"
    [root@master ~]# grep -v ^# /etc/kubernetes/apiserver 
    KUBE_API_ADDRESS="--bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1"
    KUBE_API_PORT="--secure-port=6443 --insecure-port=8080"
    KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.12.177:2379"
    KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
    KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
    KUBE_API_ARGS="--client-ca-file=/etc/kubernetes/ssl/ca.pem \
     --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
     --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem \
     --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem"
    [root@master ~]# grep -v ^# /etc/kubernetes/controller-manager 
    KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
     --root-ca-file=/etc/kubernetes/ssl/ca.pem \
     --master=http://127.0.0.1:8080 \
     --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"
    [root@master ~]# grep -v ^# /etc/kubernetes/scheduler 
    KUBE_SCHEDULER_ARGS="--master=http://127.0.0.1:8080 \
     --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"
    
    #重启各组件
    [root@master kubernetes]# systemctl restart kube-controller-manager
    [root@master kubernetes]# systemctl restart kube-scheduler
    [root@master kubernetes]# systemctl status kube-apiserver
    
    #查看状态
    [root@master kubernetes]# systemctl status kube-controller-manager -l 
    [root@master kubernetes]# systemctl status kube-scheduler -l 
    [root@master kubernetes]# systemctl status kube-apiserver -l
    
    #验证证书
    [root@master kubernetes]# curl https://192.168.12.177:6443/api/v1/nodes --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem --cacert /etc/kubernetes/ssl/ca.pem

2.node01主机

#将master主机生成的node01证书复制到ssl目录
    [root@node01 ~]# mkdir -p /etc/kubernetes/ssl/
    [root@node01 ~]# cd /etc/kubernetes/ssl/
    [root@node01 ssl]# ll
    total 12
    -rw-r--r-- 1 root root 1090 Sep 21 11:49 ca.pem
    -rw-r--r-- 1 root root 1679 Sep 21 11:49 k8s-node001-worker-key.pem
    -rw-r--r-- 1 root root 1046 Sep 21 11:50 k8s-node001-worker.pem
    
    #做一个超链接
    [root@node01 ssl]# ln -s k8s-node001-worker.pem worker.pem
    [root@node01 ssl]# ln -s k8s-node001-worker-key.pem worker-key.pem
    
    #新建kubeconfig
    [root@node01 ssl]# vi /etc/kubernetes/worker-kubeconfig.yaml 
    apiVersion: v1
    kind: Config
    clusters:
    - name: local
     cluster:
     certificate-authority: /etc/kubernetes/ssl/ca.pem
    users:
    - name: kubelet
     user:
     client-certificate: /etc/kubernetes/ssl/worker.pem
     client-key: /etc/kubernetes/ssl/worker-key.pem
    contexts:
    - context:
     cluster: local
     user: kubelet
     name: kubelet-context
    current-context: kubelet-context
    
    #按照如下配置，修改各配置文件
    [root@node01 ssl]# grep -v ^# /etc/kubernetes/config 
    KUBE_LOGTOSTDERR="--logtostderr=true"
    KUBE_LOG_LEVEL="--v=0"
    KUBE_ALLOW_PRIV="--allow-privileged=false"
    KUBE_MASTER="--master=https://192.168.12.177:6443"
    [root@node01 ssl]# grep -v ^# /etc/kubernetes/kubelet 
    KUBELET_ADDRESS="--address=0.0.0.0"
    KUBELET_PORT="--port=10250"
    KUBELET_HOSTNAME="--hostname-override=node01.test01.com"
    KUBELET_API_SERVER="--api-servers=https://192.168.12.177:6443"
    KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
    KUBELET_ARGS="--cluster-dns=192.168.12.177 \
     --cluster-domain=test01.com \
     --tls-cert-file=/etc/kubernetes/ssl/worker.pem \
     --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \
     --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
    [root@node01 ssl]# grep -v ^# /etc/kubernetes/proxy 
    KUBE_PROXY_ARGS="--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
    
    #重启服务
    [root@node01 ssl]# systemctl restart docker kubelet kube-proxy
    
    #查看
    [root@node01 ssl]# systemctl status docker kubelet kube-proxy -l
    
    #验证证书
    [root@node01 ssl]# curl https://192.168.12.177:6443/api/v1/nodes --cert /etc/kubernetes/ssl/worker.pem --key /etc/kubernetes/ssl/worker-key.pem --cacert /etc/kubernetes/ssl/ca.pem

3.新建svc进行测试

#新建svc
    [root@master template]# cd registry/
    [root@master registry]# kubectl create -f registry-rc.yaml 
    [root@master registry]# kubectl create -f registry-svc.yaml
    
    #查看
    [root@master registry]# kubectl get pods --namespace=kube-system
    NAME READY STATUS RESTARTS AGE
    kube-registry-v0-jypg1 1/1 Running 0 38s
    [root@master registry]# kubectl get rc --namespace=kube-system
    NAME DESIRED CURRENT AGE
    kube-registry-v0 1 1 38s
    [root@master registry]# kubectl get svc --namespace=kube-system
    NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
    kube-registry 10.254.80.217 <nodes> 5000/TCP 38s
    
    #上传镜像至私有仓库
    [root@master ssl]# docker pull alpine
    [root@master ssl]# docker images
    [root@master ssl]# docker tag docker.io/alpine 192.168.12.178:30099/alpine
    [root@master ssl]# docker push 192.168.12.178:30099/alpine
    The push refers to a repository [192.168.12.178:30099/alpine]
    3fc666989c1d: Pushed 
    latest: digest: sha256:02eb5cfe4b721495135728ab4aea87418fd4edbfbf83612130a81191f0b2aae3 size: 506

4.遗留问题-dashboard部署(已经解决)

4.1目前双向TLS配置后，dashboard报错如下（已经解决）

#错误提示
    [root@master ui]# kubectl get pods --namespace=kube-system
    NAME READY STATUS RESTARTS AGE
    kube-registry-v0-jypg1 1/1 Running 2 2h
    kubernetes-dashboard-v1.1.0-wd9g3 0/1 CrashLoopBackOff 5 4m
    [root@master ui]# kubectl logs -f pods/kubernetes-dashboard-v1.1.0-wd9g3 --namespace=kube-system 
    Starting HTTP server on port 9090
    Creating API server client for https://192.168.12.177:6443
    Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service accounts configuration) or the --apiserver-host param points to a server that does not exist. Reason: Get https://192.168.12.177:6443/version: x509: failed to load system roots and no roots provided

上述问题，是因为认证问题，我们需要修改一下yaml文件。

#完整的yaml文件如下：
    [root@master ui]# vi dashboard-controller.yaml 
    apiVersion: v1
    kind: ReplicationController
    metadata:
     name: kubernetes-dashboard-v1.1.0
     namespace: kube-system
     labels:
     k8s-app: kubernetes-dashboard
     version: v1.1.0
     kubernetes.io/cluster-service: "true"
    spec:
     replicas: 1
     selector:
     k8s-app: kubernetes-dashboard
     template:
     metadata:
     labels:
     k8s-app: kubernetes-dashboard
     version: v1.1.0
     kubernetes.io/cluster-service: "true"
     spec:
     containers:
     - name: kubernetes-dashboard
     image: index.tenxcloud.com/google_containers/kubernetes-dashboard-amd64:v1.1.0
     resources:
     # keep request = limit to keep this container in guaranteed class
     limits:
     cpu: 100m
     memory: 50Mi
     requests:
     cpu: 100m
     memory: 50Mi
     ports:
     - containerPort: 9090
     livenessProbe:
     httpGet:
     path: /
     port: 9090
     initialDelaySeconds: 30
     timeoutSeconds: 30
     volumeMounts:
     - mountPath: /etc/kubernetes/ssl
     name: ssl-certs-kubernetes
     readOnly: true
     - mountPath: /etc/ssl/certs
     name: ssl-certs-host
     readOnly: true
     volumes:
     - hostPath:
     path: /etc/kubernetes/ssl
     name: ssl-certs-kubernetes
     - hostPath:
     path: /etc/pki/tls/certs
     name: ssl-certs-host
    
    #创建
    [root@master ui]# kubectl create -f dashboard-controller.yaml 
    replicationcontroller "kubernetes-dashboard-v1.1.0" created
    
    #查看
    [root@master ui]# kubectl get pods --namespace=kube-system
    NAME READY STATUS RESTARTS AGE
    kubernetes-dashboard-v1.1.0-dt16g 1/1 Running 0 2s

可以看到，相比原来yaml文件，增加了ssl认证部分。

4.2自建CA，masterIP只能https访问了，那么windows如何访问dashboard（已经解决）

默认dashboard是通过master的ip和端口进行访问的，但是在windows下因无法加载自建ca证书到时无法访问dashboard。

解决办法是修改yaml，添加nodeport。

#完整的yaml文件，如下
    [root@master ui]# vi dashboard-service.yaml 
    apiVersion: v1
    kind: Service
    metadata:
     name: kubernetes-dashboard
     namespace: kube-system
     labels:
     k8s-app: kubernetes-dashboard
     kubernetes.io/cluster-service: "true"
    spec:
     selector:
     k8s-app: kubernetes-dashboard
     type: NodePort
     ports:
     - port: 80
     nodePort: 30080
     targetPort: 9090
    
    #创建
    [root@master ui]# kubectl create -f dashboard-service.yaml
    You have exposed your service on an external port on all nodes in your
    cluster. If you want to expose this service to the external internet, you may
    need to set up firewall rules for the service port(s) (tcp:30080) to serve traffic.
    
    See http://releases.k8s.io/release-1.3/docs/user-guide/services-firewalls.md for more details.
    service "kubernetes-dashboard" created
    
    #查看
    [root@master ui]# kubectl get svc --namespace=kube-system
    NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
    kubernetes-dashboard 10.254.0.253 <nodes> 80/TCP 1m

windows下浏览器打开

http://192.168.12.178:30080/

Kubernetes双向TLS配置-Centos7![k8s-tls-001][]

本文：[Kubernetes双向TLS配置-Centos7][Kubernetes_TLS_-Centos7]

[kubernetes]: http://www.kubernetes.org.cn/
[k8s-tls-001]: http://www.pangxie.space/wp-content/uploads/2016/09/k8s-tls-001.png
[Kubernetes_TLS_-Centos7]: http://www.kubernetes.org.cn/648.html