[kubernetes]-filebeat以sidecar模式收集pod日志

本是古典何须时尚 2022-12-01 11:58 232阅读 0赞

目的: 收集测试环境日志到kibana

定义日志收集相关配置的一个通配configmap

test-filebeat-config.yaml

#定义日志收集相关配置的一个通配configmap
    ---
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: test-filebeat-config
      namespace: test
      labels:
        config: test-filebeat-config
    data:
      filebeat.yml: |-
        filebeat.prospectors:
        - type: log
          paths:
            - /data/tomcat-haozhuo-ec/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","ec"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-ec/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","ec"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-basic/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","basic"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-basic/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","basic"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-deep/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","deep"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-deep/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","deep"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-video/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","video"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-video/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","video"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-wx/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","wx"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-wx/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","wx"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-system/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","system"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-system/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","system"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-shoppingmall/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","shoppingmall"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-shoppingmall/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","shoppingmall"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-reportContent/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","reportContent"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-reportContent/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","reportContent"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-point/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","point"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-point/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","point"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-openapi/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","openapi"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-openapi/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","openapi"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-job/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","job"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-job/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","job"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-info/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","info"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-info/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","info"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-hm/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","hm"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-hm/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","hm"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-health/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","health"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-health/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","health"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-family/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","family"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-family/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","family"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-disc/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","disc"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-disc/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","disc"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-check/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","check"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-check/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","check"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-card/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","card"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-card/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","card"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-appServer/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","appServer"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-appServer/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","appServer"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-admin/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","admin"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-admin/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","admin"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-account/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","account"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-account/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","account"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-hmp/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","hmp"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-hmp/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","hmp"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-search/json.info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","search"]
    
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-search/json.error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","search"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-service_center/info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","service_center"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-service_center/error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","service_center"]
        - type: log
          paths:
            - /data/tomcat-haozhuo-appoint_center/info.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["info","appoint_center"]
        - type: log
          enabled: true
          paths:
           - /data/tomcat-haozhuo-appoint_center/error.*.log
          json.keys_under_root: true
          json.overwrite_keys: true
          tags: ["error","appoint_center"]
        filebeat.config.modules:
          path: ${ path.config}/modules.d/*.yml
          reload.enabled: false
        setup.template.enabled: false
        setup.template.settings:
          index.number_of_shards: 3
        setup.template.name: "filebeat"
        setup.template.fields: "fields.yml"
        setup.template.overwrite: false
        setup.kibana:
        output.elasticsearch:
         hosts: ["192.168.1.165:9200"]
         index: "jsonlog-%{+yyyy.MM.dd}"

创建一个statefulset部署应用服务。filebeat以sidecar形式一起部署 StatefulSet.yaml

apiVersion: apps/v1
    kind: StatefulSet
    metadata:
    # deployment名字 和svc 和ingress绑定没关系
      name: haozhuo-account-dp
      namespace: test
    spec:
      serviceName: haozhuo-account
      replicas: 1
      # 在定义模板的时候必须定义labels,因为Deployment.spec.selector是必须字段,而他又必须和template.labels对应
      selector:
        matchLabels:
          app: haozhuo-account
      # template里面定义的内容会应用到下面所有的副本集里面(例如depolyment下的pod),在template.spec.containers里面不能定义labels标签。可以kubectl get pods --show-labels查看
      template:
        metadata:
          labels:
            app: haozhuo-account
            env: test
        spec:
          containers:
    # 20200822
    
          - image: registry.cn-shanghai.aliyuncs.com/yjk-datag/filebeat:6 # 提前下载下来到私有镜像库的镜像(官方的可能会被墙)
            name: filebeat
            args: [
              "-c", "/srv/filebeat/filebeat.yml",
              "-e",
            ]
            env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            - name: pod_name
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: POD_IP
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            - name: pod_name
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
           # securityContext:
           # runAsUser: 0
            resources:
              limits:
                memory: 200Mi
              requests:
                cpu: 200m
                memory: 200Mi
            volumeMounts:
            - name: config               # 将configmap的内容放到容器本地目录
              mountPath: /srv/filebeat/
            - name: data
              mountPath: /usr/share/filebeat/data
            - name: logdata       # 同一个pod内的两个应用共享目录logdata, 一个写一个读
              mountPath: /data/tomcat-haozhuo-account/
    
    
    
    # 20200822
    
    
    
          # containers名字 和svc 和ingress绑定没关系
          - name: haozhuo-account
            image: registry.cn-shanghai.aliyuncs.com/yjk-datag/k8s-haozhuo-account:v28
            #image: registry.cn-shanghai.aliyuncs.com/yjk-datag/haozhuo-video:vtest06
            env:
            - name: DUBBO_IP_TO_REGISTRY
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: DUBBO_PORT_TO_REGISTRY
              value: "20901"
            resources:
              limits:
                cpu: 2000m
                memory: 2048Mi
              requests:
                cpu: 1000m
                memory: 1024Mi
            ports:
            - name: jacoco-port
              containerPort: 6309
            - name: liveness-port
              containerPort: 8080
    # 使用了 hostPort 的容器只能调度到端口不冲突的 Node 上，除非有必要（比如运行一些系统级的 daemon 服务），不建议使用端口映射功能。
    # 跟 hostPort 一样的原因，避免使用 hostNetwork。
    # hostPorts:
    # - min: 8000
    # max: 8080
              hostPort: 8801
            - name: readiness-port
              containerPort: 20901
              hostPort: 20901
    # 后续需要jacoco端口也暴露一下
    
            livenessProbe:
              tcpSocket:
                port: liveness-port
              initialDelaySeconds: 20
              periodSeconds: 3
    
            readinessProbe:
              tcpSocket:
                port: readiness-port
              initialDelaySeconds: 30
              periodSeconds: 10
    
    
    # 20200822
    
            volumeMounts:
            - name: logdata       # 指定挂在目录到logdata 同一个pod内的两个应用共享目录logdata, 一个写一个读
            # 我这里要挂载应用容器haozhuo-account中/srv/tomcat/tomcat-haozhuo-account/yjklogs/里面的*.log到filebeat容器的/data/tomcat-haozhuo-account/下
              mountPath: /srv/tomcat/tomcat-haozhuo-account/yjklogs/
          volumes:
          - name: data
            emptyDir: { }
          - name: logdata         #定义logdata为EmptyDir类型挂载目录
            emptyDir: { }
          - name: config
            configMap:
              name: test-filebeat-config  #使用前面定义的configmap
              items:
              - key: filebeat.yml
                path: filebeat.yml
    
    # 20200822
    
          imagePullSecrets:
          - name: aliyun-ns-test

导入并上传镜像

docker load < filebeat6.tar.gz
    
    docker tag prima/filebeat:6 registry.cn-shanghai.aliyuncs.com/yjk-datag/filebeat:6
    
    docker push registry.cn-shanghai.aliyuncs.com/yjk-datag/filebeat:6

![format_png][]

应用yaml

kubectl apply -f StatefulSet.yaml

等到容器启动完成

进入容器需要 -c 指定容器name

# 进入应用容器
    kubectl exec -it haozhuo-account-dp-0   -c haozhuo-account  /bin/bash -n test
    # 进入filebeat容器
    kubectl exec -it haozhuo-account-dp-0   -c filebeat  /bin/bash -n test
    
    # 如果不使用-c的话会进入sidecar容器

![format_png 1][]

![format_png 2][]

手动输出error到error日志或者info日志

echo '{"@timestamp":"2020-08-22T15:59:10.014+08:00","@version":1,"message":"请求url:http://192.168.1.164:8030/dingtalk/order/saveOrderDataRecord 入参:","logger_name":"com.haozhuo.common.sys.http.HttpRequest","thread_name":"account-updateOrderBizData_Worker-1","level":"INFO","level_value":20000,"HOSTNAME":"haozhuo-account-dp-0.haozhuo-account.test.svc.cluster.local","tid":"6494f345-b18e-40ac-b4fc-baf851cab191"} ' >> /data/tomcat-haozhuo-account/json.info.2020-08-22.log

观察filebeat 发现采集成功

![format_png 3][]

kubectl logs -f  haozhuo-account-dp-0  filebeat -n test 
    kubectl logs -f  haozhuo-account-dp-0  haozhuo-account -n test

![format_png 4][]

[format_png]: /images/20221123/bb1a787fa80b475a88e63b9440537ec0.png
[format_png 1]: /images/20221123/480f027cbd6a4f029b212a9dfb392d0c.png
[format_png 2]: /images/20221123/0c0fc952d5c44a84813de2c307b3211f.png
[format_png 3]: /images/20221123/f4e6ff7121fe418382b73175c13f9f39.png
[format_png 4]: /images/20221123/3e3478129cd94ca3a6be7d0a419cb40b.png