使用kubeasz一键部署k8s，安装kubesphere，进行devops

偏执的太偏执、 2022-12-11 03:10 203阅读 0赞

### 使用kubeasz一键部署k8s ###

*  机器安排
 *  所有机器安装依赖工具和免登录
 *  master机器安装ansible
 *  master机器下载工具脚本easzup
 *  master机器下载 k8s 版本安装文件
 *  master机器加入阿里云镜像地址
 *  master机器控制端编排k8s安装
 *  给node机器加上阿里云镜像
 *  master安装helm3.0
 *  master安装nfs
 *   *  安装配置nfs
 *  安装kubesphere
 *  访问kubesphere
 *  继续安装kubesphere的插件
 *  自定义centos镜像用于编译java代码

# 机器安排 #

<table> 
 <thead> 
  <tr> 
   <th>ip</th> 
   <th>角色</th> 
   <th>CPU</th> 
   <th>内存</th> 
   <th>硬盘</th> 
   <th>软件情况</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>192.168.152.164</td> 
   <td>master</td> 
   <td>2</td> 
   <td>8G(可减少)</td> 
   <td>50G</td> 
   <td>centos7.8新机器</td> 
  </tr> 
  <tr> 
   <td>192.168.152.165</td> 
   <td>node01</td> 
   <td>2</td> 
   <td>8G(可减少)</td> 
   <td>50G</td> 
   <td>centos7.8新机器</td> 
  </tr> 
  <tr> 
   <td>192.168.152.165</td> 
   <td>node02</td> 
   <td>2</td> 
   <td>8G(可减少)</td> 
   <td>50G</td> 
   <td>centos7.8新机器</td> 
  </tr> 
 </tbody> 
</table>

# 所有机器安装依赖工具和免登录 #

# 文档中脚本默认均以root用户执行，这一步安装时间比较久
    yum update
    # 安装python
    yum install python -y
    # CentOS 7
    wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
    yum -y install python-pip
    
    
    # 更安全 Ed25519 算法
    ssh-keygen -t ed25519 -N '' -f ~/.ssh/id_ed25519
    # 或者传统 RSA 算法
    ssh-keygen -t rsa -b 2048 -N '' -f ~/.ssh/id_rsa
    ssh-copy-id $IPs #$IPs为所有节点地址包括自身，按照提示输入yes 和root密码  每台机器操作3次

注意$IPs为所有节点地址**包括自身**

# master机器安装ansible #

# pip安装ansible(国内如果安装太慢可以直接用pip阿里云加速)
    pip install pip --upgrade -i https://mirrors.aliyun.com/pypi/simple/
    pip install ansible==2.6.18 netaddr==0.7.19 -i https://mirrors.aliyun.com/pypi/simple/

# master机器下载工具脚本easzup #

export release=2.2.1
    curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${
        release}/easzup
    chmod +x ./easzup

release的参数尽量选择比较高的版本，我目前这里是2.2.1最新

# master机器下载 k8s 版本安装文件 #

我个人观点：有些版本是有坑的，比如1.18.2，我用这个1.18.2版本怎么都安装不了kubesphere。我测得1.18.6对kubesphere兼容还可行。

./easzup -D -d 19.03.5 -k v1.18.6

# master机器加入阿里云镜像地址 #

./easzup -D -d 19.03.5 -k v1.18.6这个命令可能会执行比较久，如果我们发现/etc/docker/daemon.json自动生成之后，我们可以手动添加阿里云镜像地址：

"https://b9pmyelo.mirror.aliyuncs.com"

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center]  
然后

systemctl daemon-reload&&systemctl restart docker
    #重新下载
    ./easzup -D -d 19.03.5 -k v1.18.6

# master机器控制端编排k8s安装 #

参考GitHub链接：

https://github.com/easzlab/kubeasz/blob/master/docs/setup/00-planning_and_overall_intro.md#ha-architecture

cd /etc/ansible && cp example/hosts.multi-node hosts
    vi hosts

修改hosts文件如下：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 1]

#开始安装，执行完之前需要等待几分钟
    ansible-playbook 90.setup.yml

![在这里插入图片描述][20200928004156688.png_pic_center]

# 给node机器加上阿里云镜像 #

类似master机器加入阿里云镜像地址这一节内容

"https://b9pmyelo.mirror.aliyuncs.com"

# master安装helm3.0 #

在github的releases中找个3.X的版本下载  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 2]

# 解压安装包
    tar -zxvf helm-v3.3.0-linux-amd64.tar.gz
    # 在解压后的目录中找到二进制文件，然后将其移至所需的目标位置
    mv linux-amd64/helm /usr/local/bin/helm
    # 初始化Helm存储库
    helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
    #列出可以安装的charts
    helm search repo stable

# master安装nfs #

## 安装配置nfs ##

**client端，这里为两个node节点**

yum -y install nfs-utils

**server端，master节点**

1.安装包
    yum -y install nfs-utils rpcbind
    
    2.编辑配置文件
    配置文件中的*是允许所有网段，根据自己实际情况写明网段
    cat >/etc/exports <<EOF
    /data *(insecure,rw,async,no_root_squash) 
    EOF
    
    3.创建目录并修改权限
    这里为了方便实验授予了挂载目录权限为777，请根据实际情况修改目录权限和所有者
    mkdir /data && chmod 777 /data
    
    4.启动服务
    systemctl enable nfs-server rpcbind && systemctl start nfs-server rpcbind
    
    5、192.168.152.174/175机器上验证
    showmount -e 192.168.152.171

**配置storageclass，注意修改nfs服务端IP和共享目录**

cat >storageclass.yaml <<EOF
    ---
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: nfs-provisioner
    ---
    kind: ClusterRole
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
       name: nfs-provisioner-runner
       namespace: default
    rules:
       -  apiGroups: [""]
          resources: ["persistentvolumes"]
          verbs: ["get", "list", "watch", "create", "delete"]
       -  apiGroups: [""]
          resources: ["persistentvolumeclaims"]
          verbs: ["get", "list", "watch", "update"]
       -  apiGroups: ["storage.k8s.io"]
          resources: ["storageclasses"]
          verbs: ["get", "list", "watch"]
       -  apiGroups: [""]
          resources: ["events"]
          verbs: ["watch", "create", "update", "patch"]
       -  apiGroups: [""]
          resources: ["services", "endpoints"]
          verbs: ["get","create","list", "watch","update"]
       -  apiGroups: ["extensions"]
          resources: ["podsecuritypolicies"]
          resourceNames: ["nfs-provisioner"]
          verbs: ["use"]
    ---
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: run-nfs-provisioner
    subjects:
      - kind: ServiceAccount
        name: nfs-provisioner
        namespace: default
    roleRef:
      kind: ClusterRole
      name: nfs-provisioner-runner
      apiGroup: rbac.authorization.k8s.io
    ---
    kind: Deployment
    apiVersion: apps/v1
    metadata:
      name: nfs-client-provisioner
    spec:
      selector:
        matchLabels:
          app: nfs-client-provisioner
      replicas: 1
      strategy:
        type: Recreate
      template:
        metadata:
          labels:
            app: nfs-client-provisioner
        spec:
          serviceAccount: nfs-provisioner
          containers:
            - name: nfs-client-provisioner
              image: quay.io/external_storage/nfs-client-provisioner:latest
              imagePullPolicy: IfNotPresent
              volumeMounts:
                - name: nfs-client
                  mountPath: /persistentvolumes
              env:
                - name: PROVISIONER_NAME
                  value: fuseim.pri/ifs
                - name: NFS_SERVER
                  value: 192.168.152.164 #此处修改为nfs服务器ip
                - name: NFS_PATH
                  value: /data   #这里为nfs共享目录
          volumes:
            - name: nfs-client
              nfs:
                server: 192.168.152.164 #此处修改为nfs服务器ip
                path: /data   #这里为nfs共享目录
    ---
    apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
      name: nfs-storage
    provisioner: fuseim.pri/ifs
    reclaimPolicy: Retain
    EOF

注意：quay.io/external\_storage/nfs-client-provisioner:latest有可能下载不了

**创建storageclass**

# 下载镜像
    docker pull registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner:latest
    
    docker tag registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner:latest quay.io/external_storage/nfs-client-provisioner:latest    
    
    kubectl apply -f storageclass.yaml

**设置默认strorageclass**

kubectl patch storageclass nfs-storage -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

**检查nfs-client pod状态**

#这里是在default命名空间下创建的
    kubectl get pods
    
    NAME                                      READY   STATUS    RESTARTS   AGE
    nfs-client-provisioner-7b9746695c-nrz4n   1/1     Running   0          2m38s

**检查默认存储**

#这里是在default命名空间下创建的
    kubectl get sc
    
    NAME                    PROVISIONER      AGE
    nfs-storage (default)   fuseim.pri/ifs   7m22s

# 安装kubesphere #

**下载kubesphere-installer.yaml、cluster-configuration.yaml文件：**

https://raw.githubusercontent.com/kubesphere/ks-installer/v3.0.0/deploy/kubesphere-installer.yaml
    
    https://raw.githubusercontent.com/kubesphere/ks-installer/v3.0.0/deploy/cluster-configuration.yaml

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 3]

**修改cluster-configuration.yaml的etcd地址**  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 4]  
**安装kubesphere-installer**

kubectl apply -f kubesphere-installer.yaml

等待几分钟，让它安装成功  
![在这里插入图片描述][202009280110151.png_pic_center]

**安装cluster-configuration.yaml**  
等kubesphere-installer安装成功后，执行下面命令

kubectl apply -f cluster-configuration.yaml
    #查看执行日志
    kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 5]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 6]

# 访问kubesphere #

Console: http://192.168.152.164:30880
    Account: admin
    Password: P@88w0rd

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 7]

# 继续安装kubesphere的插件 #

修改cluster-configuration.yaml  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 8]  
执行安装：

kubectl apply -f cluster-configuration.yaml

# 自定义centos镜像用于编译java代码 #

#删除所有Exited的镜像
    docker rm  $(docker ps -a | grep Exited | awk '{print $1}')
    #运行镜像
    docker run -d -i -t centos:latest /bin/bash
    #拷贝文件  427e2bbd8655是centos:latest 镜像id
    docker cp /root/centos/ 427e2bbd8655:/opt
    #进入容器
    docker exec -it 427e2bbd8655 /bin/bash
    #进入工作目录
    cd /opt/centos
    #解压文件
    rm -f apache-maven-3.6.3-bin.tar.gz
    tar -zxf jdk-11.0.8_linux-x64_bin.tar.gz
    #删除压缩文件
    rm -f apache-maven-3.6.2-bin.tar.gz
    rm -f jdk-11.0.8_linux-x64_bin.tar.gz
    
    #建立软连接
    ln -s /opt/centos/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
    ln -s /opt/centos/jdk-11.0.8/bin/java /usr/bin/java
    
    #写入path环境变量
    echo 'export JAVA_HOME=/opt/centos/jdk-11.0.8
    export M2_HOME=/opt/centos/apache-maven-3.6.3
    export JRE_HOME=${JAVA_HOME}/jre
    export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
    export PATH=${JAVA_HOME}/bin:$PATH:$M2_HOME/bin'>>/root/.bashrc
    
    mkdir -p /data/maven/repo
    chmod +x /data/maven/repo
    settings.xml /usr/local/apache-maven/conf/settings.xml
    cp settings.xml /opt/centos/apache-maven-3.6.3/conf/settings.xml
    #安装which，防止mvn启动报错
    yum install which
    
    #退出容器
    exit
    #保存容器
    docker commit 427e2bbd8655 my-centos:latest

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center]: /images/20221123/4a0da22932cc4f58a967168cb8c7998c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221123/20782e0105a14b6d9f1b85691a48c226.png
[20200928004156688.png_pic_center]: /images/20221123/2017e5481c464c09a81ae84266cc4c4b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221123/a58aa30b93b74bb28f291480b42c008c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 3]: /images/20221123/177b9d621e3b45a3b4ea1b4b1b67c744.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 4]: /images/20221123/e61f3fda67294be18fd37bd45711a68f.png
[202009280110151.png_pic_center]: /images/20221123/11ac67c123ce4b9681ba3ff4a3f789f4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 5]: /images/20221123/34f7ce93b15a4c98ae3adfd37af65dfa.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 6]: /images/20221123/fa447f143e5a4bff8f89edfffc145570.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 7]: /images/20221123/aa4cb194eecb4563a5dc7e883adb053c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L1pIT05HWkVXRUk_size_16_color_FFFFFF_t_70_pic_center 8]: /images/20221123/6e0d6a814e464239b56ae4f3169e2898.png