27.链路追踪-skywalking-oap

痛定思痛。 2022-10-12 01:01 274阅读 0赞

### 27.链路追踪-skywalking-oap ###

我们这里手动构造资源文件部署skywalking到k8s集群中

> Helm安装参考https://github.com/apache/skywalking-kubernetes/tree/master/chart/skywalking

namespace:skywalking-ns

#### 一、skywalking-oap资源说明 ####

1.  elasticsearch-svc.yaml
2.  elasticsearch-statefulset.yaml
3.  oap-rbac.yaml
4.  oap-svc.yaml
5.  oap-es-init-job.yaml
6.  oap-deployment.yaml
7.  ui-svc.yaml
8.  ui-deployment.yaml

#### 二、skywalking-oap资源 ####

##### 1. elasticsearch #####

用于存储skywalking oap接收到的数据

*  elasticsearch-svc.yaml

kind: Service
    apiVersion: v1
    metadata:
      namespace: skywalking-ns
      name: elasticsearch-master
      labels:
        heritage: "Helm"
        release: "hzw-od-sw"
        chart: "elasticsearch"
        app: "elasticsearch-master"
      annotations:
        {
        }
    spec:
      type: ClusterIP
      selector:
        heritage: "Helm"
        release: "hzw-od-sw"
        chart: "elasticsearch"
        app: "elasticsearch-master"
      ports:
      - name: http
        protocol: TCP
        port: 9200
      - name: transport
        protocol: TCP
        port: 9300
    ---
    kind: Service
    apiVersion: v1
    metadata:
      namespace: skywalking-ns
      name: elasticsearch-master-headless
      labels:
        heritage: "Helm"
        release: "hzw-od-sw"
        chart: "elasticsearch"
        app: "elasticsearch-master"
      annotations:
        service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
    spec:
      clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve
      # Create endpoints also if the related pod isn't ready
      publishNotReadyAddresses: true
      selector:
        app: "elasticsearch-master"
      ports:
      - name: http
        port: 9200
      - name: transport
        port: 9300

*  elasticsearch-statefulset.yaml

apiVersion: apps/v1
    kind: StatefulSet
    metadata:
      namespace: skywalking-ns
      name: elasticsearch-master
      labels:
        heritage: "Helm"
        release: "hzw-od-sw"
        chart: "elasticsearch"
        app: "elasticsearch-master"
      annotations:
        esMajorVersion: "7"
    spec:
      serviceName: elasticsearch-master-headless
      selector:
        matchLabels:
          app: "elasticsearch-master"
      replicas: 2
      podManagementPolicy: Parallel
      updateStrategy:
        type: RollingUpdate
      template:
        metadata:
          name: "elasticsearch-master"
          labels:
            heritage: "Helm"
            release: "hzw-od-sw"
            chart: "elasticsearch"
            app: "elasticsearch-master"
          annotations:       
        spec:
          securityContext:
            fsGroup: 1000
            runAsUser: 1000
          affinity:
            podAntiAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
              - labelSelector:
                  matchExpressions:
                  - key: app
                    operator: In
                    values:
                    - "elasticsearch-master"
                topologyKey: kubernetes.io/hostname
          terminationGracePeriodSeconds: 120
          volumes:
          initContainers:
          - name: configure-sysctl
            securityContext:
              runAsUser: 0
              privileged: true
            image: "docker.elastic.co/elasticsearch/elasticsearch:7.5.1"
            imagePullPolicy: "IfNotPresent"
            command: ["sysctl", "-w", "vm.max_map_count=262144"]
            resources:
              {
        }
          containers:
          - name: "elasticsearch"
            securityContext:
              capabilities:
                drop:
                - ALL
              runAsNonRoot: true
              runAsUser: 1000
            image: "docker.elastic.co/elasticsearch/elasticsearch:7.5.1"
            imagePullPolicy: "IfNotPresent"
            readinessProbe:
              failureThreshold: 3
              initialDelaySeconds: 10
              periodSeconds: 10
              successThreshold: 3
              timeoutSeconds: 5
              exec:
                command:
                  - sh
                  - -c
                  - |
                    #!/usr/bin/env bash -e
                    # If the node is starting up wait for the cluster to be ready (request params: 'wait_for_status=green&timeout=1s' )
                    # Once it has started only check that the node itself is responding
                    START_FILE=/tmp/.es_start_file
                    http () {
                        local path="${1}"
                        if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then
                          BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}"
                        else
                          BASIC_AUTH=''
                        fi
                        curl -XGET -s -k --fail ${BASIC_AUTH} http://127.0.0.1:9200${path}
                    }
                    if [ -f "${START_FILE}" ]; then
                        echo 'Elasticsearch is already running, lets check the node is healthy and there are master nodes available'
                        http "/_cluster/health?timeout=0s"
                    else
                        echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )'
                        if http "/_cluster/health?wait_for_status=green&timeout=1s" ; then
                            touch ${START_FILE}
                            exit 0
                        else
                            echo 'Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )'
                            exit 1
                        fi
                    fi
            ports:
            - name: http
              containerPort: 9200
            - name: transport
              containerPort: 9300
            resources:
              limits:
                cpu: 1000m
                memory: 2Gi
              requests:
                cpu: 100m
                memory: 2Gi
            env:
              - name: node.name
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.name
              - name: cluster.initial_master_nodes
                value: "elasticsearch-master-0,elasticsearch-master-1,"    # es集群节点名，按节点数量
                # value: "elasticsearch-master-0,elasticsearch-master-1,elasticsearch-master-2,"
              - name: discovery.seed_hosts
                value: "elasticsearch-master-headless"
              - name: cluster.name
                value: "elasticsearch"
              - name: network.host
                value: "0.0.0.0"
              - name: ES_JAVA_OPTS
                value: "-Xmx1g -Xms1g"
              - name: node.data
                value: "true"
              - name: node.ingest
                value: "true"
              - name: node.master
                value: "true"
            volumeMounts:

因为elasticsearch是有状态服务，我们使用StatefulSet资源来部署elasticsearch

##### 2. oap-server #####

*  oap-rbac.yaml

apiVersion: v1
    kind: ServiceAccount
    metadata:
      labels:
        app: hzw-od-sw
        chart: skywalking-4.0.0
        component: "oap"
        heritage: Helm
        release: hzw-od-sw
      name: hzw-od-sw-skywalking-oap
      namespace: skywalking-ns
    ---
    kind: Role
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      namespace: skywalking-ns
      name: hzw-od-sw-skywalking
      labels:
        app: hzw-od-sw
        chart: "skywalking-4.0.0"
        release: "hzw-od-sw"
        heritage: "Helm"
    rules:
      - apiGroups: [""]
        resources: ["pods","configmaps"]
        verbs: ["get", "watch", "list"]
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      namespace: skywalking-ns
      name: hzw-od-sw-skywalking
      labels:
        app: hzw-od-sw
        chart: "skywalking-4.0.0"
        release: "hzw-od-sw"
        heritage: "Helm"
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: hzw-od-sw-skywalking
    subjects:
      - kind: ServiceAccount
        name: hzw-od-sw-skywalking-oap
        namespace: skywalking-ns

*  oap-svc.yaml

---
    apiVersion: v1
    kind: Service
    metadata:
      namespace: skywalking-ns
      name: hzw-od-sw-skywalking-oap
      labels:
        app: hzw-od-sw
        chart: skywalking-4.0.0
        component: "oap"
        heritage: Helm
        release: hzw-od-sw
    spec:
      type: ClusterIP
      ports:
      - port: 12800
        name: rest
      - port: 11800
        name: grpc
      selector:
        app: hzw-od-sw
        component: "oap"
        release: hzw-od-sw
    ---

*  oap-es-init-job.yaml

apiVersion: batch/v1
    kind: Job
    metadata:
      namespace: skywalking-ns
      name: "hzw-od-sw-es-init"
      labels:
        app: hzw-od-sw
        chart: skywalking-4.0.0
        component: "hzw-od-sw-skywalking-job"
        heritage: Helm
        release: hzw-od-sw
    spec:
      template:
        metadata:
          name: "hzw-od-sw-es-init"
          labels:
            app: hzw-od-sw
            component: "hzw-od-sw-skywalking-job"
            release: hzw-od-sw
        spec:
          restartPolicy: Never
          initContainers:
          - name: wait-for-elasticsearch
            image: busybox:1.30
            imagePullPolicy: IfNotPresent
            command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 elasticsearch-master 9200 && exit 0 || sleep 5; done; exit 1']
          containers:
          - name: oap
            image: apache/skywalking-oap-server:8.1.0-es7
            imagePullPolicy: IfNotPresent
            env:
            - name: JAVA_OPTS
              value: "-Xmx2g -Xms2g -Dmode=init"
            - name: SW_STORAGE
              value: elasticsearch7
            - name: SW_STORAGE_ES_CLUSTER_NODES
              value: "elasticsearch-master:9200"
    ---

在elasticsearch中初始化skywalking的数据表等资源

*  oap-deployment.yaml

---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app: hzw-od-sw
        chart: skywalking-4.0.0
        component: "oap"
        heritage: Helm
        release: hzw-od-sw
      name: hzw-od-sw-skywalking-oap
      namespace: skywalking-ns
    spec:
      replicas: 2
      selector:
        matchLabels:
          app: hzw-od-sw
          component: "oap"
          release: hzw-od-sw
      template:
        metadata:
          labels:
            app: hzw-od-sw
            component: "oap"
            release: hzw-od-sw
        spec:
          serviceAccountName: hzw-od-sw-skywalking-oap
          affinity:
            podAntiAffinity:
              preferredDuringSchedulingIgnoredDuringExecution:
              - weight: 1
                podAffinityTerm:
                  topologyKey: kubernetes.io/hostname
                  labelSelector:
                    matchLabels:
                      app: "hzw-od-sw"
                      release: "hzw-od-sw"
                      component: "oap"
          initContainers:
          - name: wait-for-elasticsearch
            image: busybox:1.30
            imagePullPolicy: IfNotPresent
            command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 elasticsearch-master 9200 && exit 0 || sleep 5; done; exit 1']
          containers:
          - name: oap
            image: apache/skywalking-oap-server:8.1.0-es7
            imagePullPolicy: IfNotPresent
            livenessProbe:
              tcpSocket:
                port: 12800
              initialDelaySeconds: 15
              periodSeconds: 20
            readinessProbe:
              tcpSocket:
                port: 12800
              initialDelaySeconds: 15
              periodSeconds: 20
            ports:
            - containerPort: 11800
              name: grpc
            - containerPort: 12800
              name: rest
            env:
            - name: JAVA_OPTS
              value: "-Dmode=no-init -Xmx2g -Xms2g"
            - name: SW_CLUSTER
              value: kubernetes
            - name: SW_CLUSTER_K8S_NAMESPACE
              value: "skywalking-ns"
            - name: SW_CLUSTER_K8S_LABEL
              value: "app=hzw-od-sw,release=hzw-od-sw,component=oap"
            - name: SKYWALKING_COLLECTOR_UID
              valueFrom:
                fieldRef:
                  fieldPath: metadata.uid
            - name: SW_STORAGE
              value: elasticsearch7
            - name: SW_STORAGE_ES_CLUSTER_NODES
              value: "elasticsearch-master:9200"
            volumeMounts:
          volumes:
    ---

##### 3. skywalking-ui #####

*  ui-svc.yaml

apiVersion: v1
    kind: Service
    metadata:
      labels:
        app: hzw-od-sw
        chart: skywalking-4.0.0
        component: "ui"
        heritage: Helm
        release: hzw-od-sw
      name: hzw-od-sw-skywalking-ui
      namespace: skywalking-ns
    spec:
      type: ClusterIP
      ports:
        - port: 80
          targetPort: 8080
          protocol: TCP
      selector:
        app: hzw-od-sw
        component: "ui"
        release: hzw-od-sw
    ---

*  ui-deployment.yaml

apiVersion: apps/v1
    kind: Deployment
    metadata:
      namespace: skywalking-ns
      name: hzw-od-sw-skywalking-ui
      labels:
        app: hzw-od-sw
        chart: skywalking-4.0.0
        component: "ui"
        heritage: Helm
        release: hzw-od-sw
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: hzw-od-sw
          component: "ui"
          release: hzw-od-sw
      template:
        metadata:
          labels:
            app: hzw-od-sw
            component: "ui"
            release: hzw-od-sw
        spec:
          containers:
          - name: ui
            image: apache/skywalking-ui:8.1.0
            imagePullPolicy: IfNotPresent
            ports:
            - containerPort: 8080
              name: page
            env:
            - name: SW_OAP_ADDRESS
              value: hzw-od-sw-skywalking-oap:12800
    ---