企业级 kubernetes SpringCloud 持续集成，从0到1

秒速五厘米 2022-12-11 09:27 167阅读 0赞

# 1 环境概述 #

192.168.38.50   master
    192.168.38.51   node1
    192.168.38.52   node2
    192.168.38.100  harbor/nfs
    192.168.38.45   gitlab/sonarqube

# 2 Kubernetes安装Jenkins #

## 2.1 PV/PVC ##

#创建命名空间
    kubectl create namespace jenkins-k8s

(1) 创建PV

vim jenkins-pv.yaml
     
    apiVersion: v1
    kind: PersistentVolume
    metadata:
      name: jenkins-pv
    spec:
      capacity:
        storage: 10Gi
      accessModes:
        - ReadWriteMany
      nfs:
        path: /usr/local/kubernetes/volumes/jenkins
        server: 192.168.38.100
    
    #应用
    kubectl apply -f jenkins-pv.yaml

![在这里插入图片描述][20200930022133573.png_pic_center]  
(2) 创建PVC

vim jenkins-pvc.yaml
    
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: jenkins-pvc
      namespace: jenkins-k8s
    spec:
      resources:
        requests:
          storage: 10Gi
      accessModes:
        - ReadWriteMany
    
    #应用
    kubectl apply -f jenkins-pvc.yaml

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center]

## 2.2 ServiceAccount ##

#创建一个sa账号
    kubectl create sa jenkins-k8s-sa -n jenkins-k8s
    #授权，kubectl create clusterrolebinding 名称、名称空间、绑定clusterrole=cluster-admin 
    kubectl create clusterrolebinding jenkins-k8s-sa-cluster -n jenkins-k8s --clusterrole=cluster-admin --serviceaccount=jenkins-k8s:jenkins-k8s-sa

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 1]

## 2.3 整合Jenkins ##

### 2.3.1 下载镜像 ###

#node1、node2
    docker pull jenkins/jenkins:latest

### 2.3.2 jenkins-deployment.yaml ###

vim jenkins-deployment.yaml
    
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: jenkins
      namespace: jenkins-k8s
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: jenkins
      template:
        metadata:
          labels:
            app: jenkins
        spec:
          serviceAccountName: jenkins-k8s-sa
          containers:
            - name: jenkins
              image: jenkins/jenkins:latest
              imagePullPolicy: IfNotPresent
              ports:
                - containerPort: 8080
                  name: web
                  protocol: TCP
                - containerPort: 50000
                  name: agent
                  protocol: TCP
              resources:
                limits:
                  cpu: 1000m
                  memory: 1Gi
                requests:
                  cpu: 500m
                  memory: 500Mi
              livenessProbe:
                httpGet:
                  port: 8080
                  path: /login
                initialDelaySeconds: 60
                timeoutSeconds: 5
                failureThreshold: 12
              readinessProbe:
                httpGet:
                  port: 8080
                  path: /login
                initialDelaySeconds: 60
              volumeMounts:
                - mountPath: /var/jenkins_home
                  name: jenkins-volume
                  subPath: jenkins-home
          volumes:
            - name: jenkins-volume
              persistentVolumeClaim:
                claimName: jenkins-pvc
    
    #应用
    kubectl apply -f jenkins-deployment.yaml
    #查看
    kubectl get pod -n jenkins-k8s
    #验证
    kubectl get pod -o wide -n jenkins-k8s
    curl 10.244.2.23:8080

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 2]

### 2.3.3 jenkins-svc.yaml ###

#创建
    vim jenkins-svc.yaml
    
    apiVersion: v1
    kind: Service
    metadata:
      name: jenkins-service
      namespace: jenkins-k8s
      labels:
        app: jenkins
    spec:
      selector:
        app: jenkins
      type: ClusterIP
      ports:
        - port: 8080
          name: web
          targetPort: web
        - name: agent
          port: 50000
          targetPort: agent
     
    #应用
    kubectl apply -f jenkins-svc.yaml
    #查看
    kubectl get svc -n jenkins-k8s

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 3]

### 2.3.4 jenkins-ingress.yaml ###

vim jenkins-ingress.yaml
    
    apiVersion: networking.k8s.io/v1beta1
    kind: Ingress
    metadata:
      name: jenkins
      namespace: jenkins-k8s
    spec:
      rules:
        - host: rosh.jenkins.com
          http:
            paths:
              - backend:
                  serviceName: jenkins-service
                  servicePort: 8080
                  
    #应用
    kubectl apply -f jenkins-ingress.yaml
    #查看
    kubectl get ingress -n jenkins-k8s

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 4]

## 2.4 Jenkins 初始化 ##

(1) 查看初始化密码

#查看挂载日志,记录初始化密码，填入初始化页面
    84f98993d6594792b7e420c74934eb89

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 5]  
（2）初始化插件  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 6]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 7]  
（3）初始化用户  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 8]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 9]

## 2.5 中文插件 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 10]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 11]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 12]

## 2.6 kubernetes插件 ##

(1) 安装插件![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 13]  
(2) 配置Kubernetes  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 14]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 15]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 16]

## 2.7 Jenkins从节点配置 ##

### 2.7.1 制作镜像 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 17]  
(1) setting.xml

<?xml version="1.0" encoding="UTF-8"?>
    <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
              xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
              xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
       
      <localRepository>/usr/local/apache-maven/repo</localRepository>
    
      <pluginGroups></pluginGroups>
      <proxies></proxies>
      <servers></servers>
      <profiles></profiles>
        
      <mirrors>    
        <mirror>     
          <id>central</id>     
          <mirrorOf>central</mirrorOf>     
          <name>aliyun maven</name>
          <url>https://maven.aliyun.com/repository/public</url>     
        </mirror>
      </mirrors>
        
       
    </settings>

(2) Dockerfile

FROM cnych/jenkins:jnlp6
    
    MAINTAINER rosh
    
    # 切换到 root 账户进行操作
    USER root
    
    # 安装 maven
    COPY apache-maven-3.6.2-bin.tar.gz .
    
    RUN tar -zxf apache-maven-3.6.2-bin.tar.gz && \
        mv apache-maven-3.6.2 /usr/local && \
        rm -f apache-maven-3.6.2-bin.tar.gz && \
        ln -s /usr/local/apache-maven-3.6.2/bin/mvn /usr/bin/mvn && \
        ln -s /usr/local/apache-maven-3.6.2 /usr/local/apache-maven && \
        mkdir -p /usr/local/apache-maven/repo
    
    COPY settings.xml /usr/local/apache-maven/conf/settings.xml

(3) 制作镜像

docker build -t jenkins-slave-maven .

![在这里插入图片描述][20200930022737630.png_pic_center]  
(4) 上传至harbor

docker tag jenkins-slave-maven:latest 192.168.38.100/library/jenkins-slave-maven:latest
    docker push 192.168.38.100/library/jenkins-slave-maven:latest

(5) 查看  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 18]

### 2.7.2 配置模板 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 19]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 20]  
master的/root/.kube目录拷贝到node1和node2节点：

scp -r /root/.kube node1:
    scp -r /root/.kube node2:

### 2.7.3 测试 ###

(1) 创建流水线  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 21]  
(2) 测试脚本

//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
      node("jenkins-slave"){ 
        stage('k8s-测试'){ 
    		sh "kubectl get node"
        }
    	
    	stage('maven-测试'){ 
    	   sh "mvn -v"
    	}
    	
    	stage('docker 测试'){ 
    	
    	  sh "docker version"
    	
    	}
    	
      }

(3) 结果  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 22]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 23]

# 3 整合Gitlab #

## 3.1 凭证 ##

jenkins添加访问凭证  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 24]

## 3.2 创建pipeline工程 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 25]

## 3.3 整合Hook ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 26]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 27]

## 3.4 Gitlab ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 28]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 29]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 30]

## 3.5 Jenkinsfile ##

(1) 流水线配置  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 31]  
(2) 生成脚本  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 32]  
(3) Jenkinsfile

//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
    node("jenkins-slave"){ 
    	stage('拉取代码'){ 
    		git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
    		echo "拉取代码成功"
    	}
    
    }

(4) 推送代码到develop分支

(5) 查看jenkins 日志  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 33]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 34]

## 3.6 Generic Webhook Trigger ##

查看Gitlab推送请求，防止频繁构建，现在限制develop分支推送代码时，触发自动构建，别的分支不触发构建。  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 35]

### 3.6.1 修改Pipeline配置 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 36]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 37]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 38]

## 3.6.2 修改Gitlab钩子 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 39]

### 3.6.3 测试 ###

master分支merge develop分支代码，推送master分支，没有触发jenkins自动化构建，查看gitlab响应信息  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 40]

# 4 制作镜像、上传Harbor #

## 4.1 Dockerfile ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 41]

#######################网关#######################
    FROM java:8
    RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
    RUN echo "Asia/Shanghai" > /etc/timezone
    RUN bash -c 'touch /app.jar'
    ARG JAR_FILE
    COPY ${JAR_FILE} app.jar
    EXPOSE 9000
    ENTRYPOINT ["java","-jar","/app.jar"]
    
    #######################student#######################
    FROM java:8
    RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
    RUN echo "Asia/Shanghai" > /etc/timezone
    RUN bash -c 'touch /app.jar'
    ARG JAR_FILE
    COPY ${JAR_FILE} app.jar
    EXPOSE 8001
    ENTRYPOINT ["java","-jar","/app.jar"]
    
    #######################teacher#######################
    FROM java:8
    RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
    RUN echo "Asia/Shanghai" > /etc/timezone
    RUN bash -c 'touch /app.jar'
    ARG JAR_FILE
    COPY ${JAR_FILE} app.jar
    EXPOSE 8002
    ENTRYPOINT ["java","-jar","/app.jar"]

## 4.2 Maven插件 ##

<plugin>
      <groupId>com.spotify</groupId>
      <artifactId>dockerfile-maven-plugin</artifactId>
      <version>1.3.6</version>
      <configuration>
          <repository>${project.artifactId}</repository>
              <buildArgs>
              	   <JAR_FILE>target/${project.build.finalName}.jar</JAR_FILE>
              </buildArgs>
      </configuration>
    </plugin>

## 4.3 集成Harbor ##

(1) 凭证  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 42]  
(2) 生成脚本  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 43]

## 4.4 nfs创建maven目录 ##

#nfs服务器
    cd /usr/local/kubernetes/volumes
    mkdir maven
    chmod 777  ./maven/
    chown nfsnobody ./maven/
    
    vim /etc/exports
    /usr/local/kubernetes/volumes/maven/  *(rw,sync,insecure,no_subtree_check,no_root_squash)
    
    #重启
    systemctl restart rpcbind && systemctl restart nfs
    #其它节点测试
    showmount -e 192.168.38.100

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 44]

## 4.5 Jenkinsfile ##

## 4.6 推送dev分支 ##

## 4.7 jenkins查看结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 45]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 46]

# 5 SonarQube #

## 5.1 Sonarqube token ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 47]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 48]

#记录token
    c06945196968cf0a3e5e326e1e4b2646fc97b135
    
    mvn sonar:sonar \
      -Dsonar.host.url=http://192.168.38.45:9000/sonar \
      -Dsonar.login=c06945196968cf0a3e5e326e1e4b2646fc97b135

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 49]

## 5.2 Jenkins整合Sonarqube ##

(1) 安装插件  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 50]  
(2) 创建凭证  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 51]  
(3) 系统配置  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 52]  
(4) 全局配置  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 53]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 54]

## 5.3 sonar-project.properties ##

#################################网关##########################################
    #SonarQube 实例名称，唯一
    sonar.projectKey=gateway-server
    #项目名称
    sonar.projectName=gateway-server
    #版本号
    sonar.projectVersion=1.0
    #指定扫描目录
    sonar.sources=.
    sonar.exclusions=**/test/**
    #jdk版本
    sonar.java.source=1.8
    sonar.java.target=1.8
    #扫描编写类的项目
    sonar.java.binaries=target/classes
    #编码格式
    sonar.sourceEncoding=UTF-8
    
    
    #################################学生##########################################
    #SonarQube 实例名称，唯一
    sonar.projectKey=student-server
    #项目名称
    sonar.projectName=student-server
    #版本号
    sonar.projectVersion=1.0
    #指定扫描目录
    sonar.sources=.
    sonar.exclusions=**/test/**
    #jdk版本
    sonar.java.source=1.8
    sonar.java.target=1.8
    #扫描编写类的项目
    sonar.java.binaries=target/classes
    #编码格式
    sonar.sourceEncoding=UTF-8
    
    
    #################################老师##########################################
    #SonarQube 实例名称，唯一
    sonar.projectKey=teacher-server
    #项目名称
    sonar.projectName=teacher-server
    #版本号
    sonar.projectVersion=1.0
    #指定扫描目录
    sonar.sources=.
    sonar.exclusions=**/test/**
    #jdk版本
    sonar.java.source=1.8
    sonar.java.target=1.8
    #扫描编写类的项目
    sonar.java.binaries=target/classes
    #编码格式
    sonar.sourceEncoding=UTF-8

## 5.4 Jenkinsfile ##

//harbor地址
    def harborUrl = "192.168.38.100"
    //harbor仓库名称
    def harborRepository = "springcloud-k8s"
    //构建版本tag
    def tag = "latest"
    //harbor 凭证
    def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
    //微服务列表
    def microServerList = ['gateway-server','student-server','teacher-server']
     
    
    //引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
    node("jenkins-slave"){ 
    	stage('拉取代码'){ 
    		git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
    		echo "拉取代码成功"
    	} 
    	
    	stage('制作镜像,上传harbor'){ 
    
            for(int i=0;i<microServerList.size();i++){ 
    
                echo "制作镜像: ${microServerList[i]}"
                //服务名称
                def serverName = "${microServerList[i]}"
                //定义镜像名称
                def imageName="${serverName}:${tag}"
                //打包，构建镜像
                sh "mvn -f ${serverName} clean package dockerfile:build"
          
    			// 打tag
    			sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
    			// 上传镜像
    			withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) { 
    				 // some block
    				 //登录
    				 sh "docker login -u ${username} -p ${password} ${harborUrl}"
    				 //上传
    				 sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
    			}
    			//删除
    			sh "docker rmi -f ${imageName}"
    			sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"          
            }
    
        }
    
         stage('代码审查'){ 
    
               //定义当前Jenkins的SonarQubeScanner工具
               def scannerHome = tool 'sonarqube-scanner'
    
               for(int i=0;i<microServerList.size();i++){ 
                      //引用当前JenkinsSonarQube环境
                      withSonarQubeEnv('sonarqube') { 
                               sh """cd ${microServerList[i]} ${scannerHome}/bin/sonar-scanner"""
                      }
               }
            }
    
    }

## 5.5 推代码 ##

## 5.6 查看结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 55]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 56]

# 6 部署 #

## 6.1创建日志PV/PVC ##

(1) gateway

vim gateway-pv-pvc.yaml
    kubectl apply -f gateway-pv-pvc.yaml 
    
    apiVersion: v1
    kind: PersistentVolume
    metadata:
      name: gateway-logs-pv
    spec:
      capacity:
        storage: 2Gi
      accessModes:
        - ReadWriteMany
      nfs:
        path: "/usr/local/kubernetes/volumes/logs/gateway"
        server: 192.168.38.100
    
    ---
    
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: gateway-logs-pvc
      namespace: jenkins-k8s
    spec:
      resources:
        requests:
          storage: 2Gi
      accessModes:
        - ReadWriteMany

(2) teacher

vim teacher-pv-pvc.yaml
    kubectl apply -f teacher-pv-pvc.yaml 
    
    
    apiVersion: v1
    kind: PersistentVolume
    metadata:
      name: teacher-logs-pv
    spec:
      capacity:
        storage: 2Gi
      accessModes:
        - ReadWriteMany
      nfs:
        path: "/usr/local/kubernetes/volumes/logs/teacher"
        server: 192.168.38.100
    
    ---
    
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: teacher-logs-pvc
      namespace: jenkins-k8s
    spec:
      resources:
        requests:
          storage: 2Gi
      accessModes:
        - ReadWriteMany

(3) student

vim student-pv-pvc.yaml
    kubectl apply -f student-pv-pvc.yaml
    
    apiVersion: v1
    kind: PersistentVolume
    metadata:
      name: student-logs-pv
    spec:
      capacity:
        storage: 2Gi
      accessModes:
        - ReadWriteMany
      nfs:
        path: "/usr/local/kubernetes/volumes/logs/student"
        server: 192.168.38.100
    
    ---
    
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: student-logs-pvc
      namespace: jenkins-k8s
    spec:
      resources:
        requests:
          storage: 2Gi
      accessModes:
        - ReadWriteMany

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 57]

## 6.2 创建权限 ##

kubectl create secret docker-registry registry-secret --namespace=jenkins-k8s \
    --docker-server=192.168.38.100 --docker-username=admin \
    --docker-password=Harbor12345
    
    
    kubectl get secret

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 58]

## 6.3 Deployment ##

(1) gateway

##############Deployment######################
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app: gateway-server
      name: gateway-server
      namespace: jenkins-k8s
    spec:
      replicas: 2
      selector:
        matchLabels:
          app: gateway-server
      template:
        metadata:
          labels:
            app: gateway-server
        spec:
          imagePullSecrets:
            - name: registry-secret
          containers:
            - name: gateway-server
              image: 192.168.38.100/springcloud-k8s/gateway-server:latest
              imagePullPolicy: Always
              volumeMounts:
                - mountPath: /gateway-server/logs
                  name: gateway-server
          volumes:
            - name: gateway-server
              persistentVolumeClaim:
                claimName: gateway-logs-pvc
    ---
    ##############Service####################
    apiVersion: v1
    kind: Service
    metadata:
      labels:
        app: gateway-server
      name: gateway-server
    spec:
      ports:
        - port: 9000
          protocol: TCP
          targetPort: 9000
      selector:
        app: gateway-server
      type: ClusterIP
    
    ---
    ##############Ingress####################
    apiVersion: networking.k8s.io/v1beta1
    kind: Ingress
    metadata:
      name: gateway-server
    spec:
      rules:
        - host: rosh.gateway-server.com
          http:
            paths:
              - backend:
                  serviceName: gateway-server
                  servicePort: 9000

(2) student

##############Deployment####################
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app: student-server
      name: student-server
      namespace: jenkins-k8s
    spec:
      replicas: 2
      selector:
        matchLabels:
          app: student-server
      template:
        metadata:
          labels:
            app: student-server
        spec:
          imagePullSecrets:
            - name: registry-secret
          containers:
            - name: student-server
              image: 192.168.38.100/springcloud-k8s/student-server:latest
              imagePullPolicy: Always
              volumeMounts:
                - mountPath: /student-server/logs
                  name: student-server
          volumes:
            - name: student-server
              persistentVolumeClaim:
                claimName: student-logs-pvc
    ---
    ##############Service####################
    apiVersion: v1
    kind: Service
    metadata:
      labels:
        app: student-server
      name: student-server
    spec:
      ports:
        - port: 8001
          protocol: TCP
          targetPort: 8001
      selector:
        app: student-server
      type: ClusterIP
    
    ---
    ##############Ingress####################
    apiVersion: networking.k8s.io/v1beta1
    kind: Ingress
    metadata:
      name: student-server
    spec:
      rules:
        - host: rosh.student-server.com
          http:
            paths:
              - backend:
                  serviceName: student-server
                  servicePort: 8001

(3) teacher

##############Deployment####################
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app: teacher-server
      name: teacher-server
      namespace: jenkins-k8s
    spec:
      replicas: 2
      selector:
        matchLabels:
          app: teacher-server
      template:
        metadata:
          labels:
            app: teacher-server
        spec:
          imagePullSecrets:
            - name: registry-secret
          containers:
            - name: teacher-server
              image: 192.168.38.100/springcloud-k8s/teacher-server:latest
              imagePullPolicy: Always
              volumeMounts:
                - mountPath: /teacher-server/logs
                  name: teacher-server
          volumes:
            - name: teacher-server
              persistentVolumeClaim:
                claimName: teacher-logs-pvc
    ---
    ##############Service####################
    apiVersion: v1
    kind: Service
    metadata:
      labels:
        app: teacher-server
      name: teacher-server
    spec:
      ports:
        - port: 8002
          protocol: TCP
          targetPort: 8002
      selector:
        app: teacher-server
      type: ClusterIP
    
    ---
    ##############Ingress####################
    apiVersion: networking.k8s.io/v1beta1
    kind: Ingress
    metadata:
      name: teacher-server
    spec:
      rules:
        - host: rosh.teacher-server.com
          http:
            paths:
              - backend:
                  serviceName: teacher-server
                  servicePort: 8002

## 6.4 Jenkinsfile ##

//harbor地址
    def harborUrl = "192.168.38.100"
    //harbor仓库名称
    def harborRepository = "springcloud-k8s"
    //构建版本tag
    def tag = "latest"
    //harbor 凭证
    def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
    //微服务列表
    def microServerList = ['gateway-server','student-server','teacher-server']
    
    
    //引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
    node("jenkins-slave"){ 
    	stage('拉取代码'){ 
    		git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
    		echo "拉取代码成功"
    	}
    
    	stage('制作镜像,上传harbor'){ 
    
            for(int i=0;i<microServerList.size();i++){ 
    
                echo "制作镜像: ${microServerList[i]}"
                //服务名称
                def serverName = "${microServerList[i]}"
                //定义镜像名称
                def imageName="${serverName}:${tag}"
                //打包，构建镜像
                sh "mvn -f ${serverName} clean package dockerfile:build"
    
    			// 打tag
    			sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
    			// 上传镜像
    			withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) { 
    				 // some block
    				 //登录
    				 sh "docker login -u ${username} -p ${password} ${harborUrl}"
    				 //上传
    				 sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
    			}
    
            }
    
        }
    
         stage('代码审查'){ 
    
               //定义当前Jenkins的SonarQubeScanner工具
               def scannerHome = tool 'sonarqube-scanner'
    
               for(int i=0;i<microServerList.size();i++){ 
                      //引用当前JenkinsSonarQube环境
                      withSonarQubeEnv('sonarqube') { 
                               sh """cd ${microServerList[i]} ${scannerHome}/bin/sonar-scanner"""
                      }
               }
        }
    
        stage('k8s 部署'){ 
            for(int i=0;i<microServerList.size();i++){ 
                sh "kubectl apply -f ${microServerList[i]}/Deployment.yaml"
            }
        }
    
        stage('删除本地镜像 '){ 
            for(int i=0;i<microServerList.size();i++){ 
                //服务名称
                 def serverName = "${microServerList[i]}"
                 //定义镜像名称
                 def imageName="${serverName}:${tag}"
                //删除
                sh "docker rmi -f ${imageName}"
                sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
            }
        }
    
    }

## 6.5 结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 59]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 60]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 61]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 62]

[20200930022133573.png_pic_center]: /images/20221123/04a1dffe1bef4202a4bdcbd59f2f778f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center]: /images/20221123/73457c58eb41404fb10df0d3957a4cfe.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221123/9b22717a701d40b68757377f3bce4e9d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221123/29c968e52a8a4b249ed9f75c9832a1fd.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 3]: /images/20221123/58a87e5662484da29980d2ae984e7bb3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 4]: /images/20221123/db57af2663ee44ffa9bd02504c09884f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 5]: /images/20221123/28bcc0a5575a4e5f9a698b3df2aa616c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 6]: /images/20221123/351263719c3f46e5a13180813e9aba63.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 7]: /images/20221123/6b649eb05883414b820153dc01e82d32.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 8]: /images/20221123/97e3fe3d11a944b79519339a64eabb8d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 9]: /images/20221123/1560bea00c2d430cb7056ea42bc4bd25.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 10]: /images/20221123/abfe97b84a844a449e15b0111d34bd0b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 11]: /images/20221123/cce4a5e112684ee582f218342385ce74.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 12]: /images/20221123/bfbc242e37104efd8f2c03ef82e20f2f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 13]: /images/20221123/dc02d68156404d2f9bec24bb1aeb288c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 14]: /images/20221123/67e7c07e33184b1c922fd8e9243c5e83.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 15]: /images/20221123/001ba622ef00485d9b83b5ad5a8f36ed.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 16]: /images/20221123/83fb032a7ef54269b769ef7125116328.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 17]: /images/20221123/c57ccaf2802d4dbda9afa93c3dbb5c11.png
[20200930022737630.png_pic_center]: /images/20221123/9254432e5f7941c7b46ae998797d9b8c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 18]: /images/20221123/1c6e5a73ad354b81bf37e94a885c537a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 19]: /images/20221123/ac0c4160e8c3486b9b27994885069dc9.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 20]: /images/20221123/0e5314efd2c74c81ab5c1eed8ddfc443.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 21]: /images/20221123/58ccbf2e3ab3495f8ff3828b5c5a33a8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 22]: /images/20221123/d95dd7bf89e046d697bea9f1c55c842d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 23]: /images/20221123/76a5161fb2704945ae3fcd0d469e3348.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 24]: /images/20221123/5f82586f9cf14efbb621cb7c1793e9e9.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 25]: /images/20221123/75da818324dd4eae970496bf0f669afa.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 26]: /images/20221123/686667b615404f9d9c4e6e27f967c0e4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 27]: /images/20221123/fcf898b861ef447f8c369770fa9bee8d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 28]: /images/20221123/5129f4c131d243f28f1f5162d8ea2e6c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 29]: /images/20221123/6f99dec766ef4e64ba9bc6ca23dcd8b3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 30]: /images/20221123/874c4c0157c748e8ad98eafe9b8c57b3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 31]: /images/20221123/c4fb25fc14dd45a7bf90e3a4aeb7fb5d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 32]: /images/20221123/0ca9c958b63640e7ba9ab0e7ccf048a3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 33]: /images/20221123/5945a4ff59714da18a3a1b635239991a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 34]: /images/20221123/ef9ac264b13b4797ad2595813ac59885.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 35]: /images/20221123/19bc5f80f33a4db89134117a80fcfc29.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 36]: /images/20221123/29a166d1dc714ae1af0f6e672a17bdbf.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 37]: /images/20221123/6201e56628c3417983633637dc5e0105.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 38]: /images/20221123/10108d08da764733889223ee69197ac4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 39]: /images/20221123/038509062a1a44b4862d28f3b6dd4995.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 40]: /images/20221123/b978900344fe479bae284cb7a4d80376.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 41]: /images/20221123/0c288fc048e548329f86945d40797d0e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 42]: /images/20221123/cf6a2282423e46ff86ad04caf30620ec.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 43]: /images/20221123/8bfe189dead64985a3b5e9d59f6299bc.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 44]: /images/20221123/4d242a5bd24949c5b7592f78931e6111.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 45]: /images/20221123/8e401ef46a6e45c4b2ce9709286118cc.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 46]: /images/20221123/8b3cd20095bc4d1d81a8f4f586a6c1ac.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 47]: /images/20221123/8bbaac065a904cc6b4527c8a3196c24f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 48]: /images/20221123/be771dcefa8a49b8ab48aa644de9df98.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 49]: /images/20221123/5ac14e9798ba4294bdf5a995dda8e1dd.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 50]: /images/20221123/61a6719187f247e89e5989d4d895734c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 51]: /images/20221123/3ab716b2dedf4e1a80f901c13c24b43f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 52]: /images/20221123/b3c5dcd7ab8640808955e918eaa89bd7.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 53]: /images/20221123/45132cc550124db79594374d14b04365.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 54]: /images/20221123/4e58976d5f4f439fbe2b89be86cafa20.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 55]: /images/20221123/75df0e2e2f7a4c95879c9d21c3892d30.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 56]: /images/20221123/2149007a1c1e466f8b9d8a428841f280.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 57]: /images/20221123/cfdc8137c29743cf945bc5545ae6cd12.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 58]: /images/20221123/cb61551bba4f4f769d58aa486f8a93f6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 59]: /images/20221123/ec6db4feacc54b0f962a937379db5876.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 60]: /images/20221123/954df931e7104887a67deaed513d8aeb.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 61]: /images/20221123/1147c59cd87b40cfad98a89cbcc9e977.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM0MTI1OTk5_size_16_color_FFFFFF_t_70_pic_center 62]: /images/20221123/9ccaf058e1374e66a2ddf0cbfcee2e40.png