kubernetes使用prometheus的相关配置文件

﹏ヽ暗。殇╰゛Y 2022-05-25 05:19 214阅读 0赞

本文只是简单记录一下配置文件  
prometheus.yaml

# A scrape configuration for running Prometheus on a Kubernetes cluster.
    # This uses separate scrape configs for cluster components (i.e. API server, node)
    # and services to allow each to use different authentication configs.
    #
    # Kubernetes labels will be added as Prometheus labels on metrics via the
    # `labelmap` relabeling action.
    #
    # If you are using Kubernetes 1.7.2 or earlier, please take note of the comments
    # for the kubernetes-cadvisor job; you will need to edit or remove this job.
    
    # Scrape config for API servers.
    #
    # Kubernetes exposes API servers as endpoints to the default/kubernetes
    # service so this uses `endpoints` role and uses relabelling to only keep
    # the endpoints associated with the default/kubernetes service using the
    # default named port `https`. This works for single API server deployments as
    # well as HA API server deployments.
    rule_files: 
    - /etc/prometheus_rules/*.rule
    scrape_configs:
    - job_name: 'kubernetes-apiservers'
    
      kubernetes_sd_configs:
      - role: endpoints
    
      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: https
    
      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration because discovery & scraping are two separate concerns in
      # Prometheus. The discovery auth config is automatic if Prometheus runs inside
      # the cluster. Otherwise, more config options have to be provided within the
      # <kubernetes_sd_config>.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        # If your node certificates are self-signed or use a different CA to the
        # master CA, then disable certificate verification below. Note that
        # certificate verification is an integral part of a secure infrastructure
        # so this should only be disabled in a controlled environment. You can
        # disable certificate verification by uncommenting the line below.
        #
        # insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    
      # Keep only the default/kubernetes service endpoints for the https port. This
      # will add targets for each API server which Kubernetes adds an endpoint to
      # the default/kubernetes service.
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https
    
    # Scrape config for nodes (kubelet).
    #
    # Rather than connecting directly to the node, the scrape is proxied though the
    # Kubernetes apiserver. This means it will work if Prometheus is running out of
    # cluster, or can't connect to nodes for some other reason (e.g. because of
    # firewalling).
    - job_name: 'kubernetes-nodes'
    
      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: https
    
      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration because discovery & scraping are two separate concerns in
      # Prometheus. The discovery auth config is automatic if Prometheus runs inside
      # the cluster. Otherwise, more config options have to be provided within the
      # <kubernetes_sd_config>.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    
      kubernetes_sd_configs:
      - role: node
    
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics
    
    # Scrape config for Kubelet cAdvisor.
    #
    # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
    # (those whose names begin with 'container_') have been removed from the
    # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to
    # retrieve those metrics.
    #
    # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor
    # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics"
    # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with
    # the --cadvisor-port=0 Kubelet flag).
    #
    # This job is not necessary and should be removed in Kubernetes 1.6 and
    # earlier versions, or it will cause the metrics to be scraped twice.
    - job_name: 'kubernetes-cadvisor'
    
      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: https
    
      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration because discovery & scraping are two separate concerns in
      # Prometheus. The discovery auth config is automatic if Prometheus runs inside
      # the cluster. Otherwise, more config options have to be provided within the
      # <kubernetes_sd_config>.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    
      kubernetes_sd_configs:
      - role: node
    
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
    
    # Scrape config for service endpoints.
    #
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
    # to set this to `https` & most likely set the `tls_config` of the scrape config.
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: If the metrics are exposed on a different port to the
    # service then set this appropriately.
    - job_name: 'kubernetes-service-endpoints'
    
      kubernetes_sd_configs:
      - role: endpoints
    
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name
    
    # Example scrape config for probing services via the Blackbox Exporter.
    #
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/probe`: Only probe services that have a value of `true`
    - job_name: 'kubernetes-services'
    
      metrics_path: /metrics
    
      kubernetes_sd_configs:
      - role: service
    
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox-exporter.example.com:9115
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name
    
    # Example scrape config for probing ingresses via the Blackbox Exporter.
    #
    # The relabeling allows the actual ingress scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/probe`: Only probe services that have a value of `true`
    - job_name: 'kubernetes-ingresses'
    
      metrics_path: /probe
      params:
        module: [http_2xx]
    
      kubernetes_sd_configs:
        - role: ingress
    
      relabel_configs:
        - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
          regex: (.+);(.+);(.+)
          replacement: ${1}://${2}${3}
          target_label: __param_target
        - target_label: __address__
          replacement: blackbox-exporter.example.com:9115
        - source_labels: [__param_target]
          target_label: instance
        - action: labelmap
          regex: __meta_kubernetes_ingress_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_ingress_name]
          target_label: kubernetes_name
    
    # Example scrape config for pods
    #
    # The relabeling allows the actual pod scrape endpoint to be configured via the
    # following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
    # pod's declared ports (default is a port-free target if none are declared).
    - job_name: 'kubernetes-pods'
    
      kubernetes_sd_configs:
      - role: pod
    
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name
    
    alerting:
      alertmanagers:
      - static_configs:
        - targets:
           - alertmanager:9093

规则文件

/etc/prometheus_rules $ ls -l
    total 0
    lrwxrwxrwx    1 root     root            17 Apr 12 13:50 admin.rule -> ..data/admin.rule
    lrwxrwxrwx    1 root     root            20 Apr 12 13:50 automoto.rule -> ..data/automoto.rule
    lrwxrwxrwx    1 root     root            25 Apr 12 13:50 ceres-dev.rule.yml -> ..data/ceres-dev.rule.yml
    lrwxrwxrwx    1 root     root            19 Apr 12 13:50 cicd-17.rule -> ..data/cicd-17.rule
    lrwxrwxrwx    1 root     root            19 Apr 17 10:18 default.rule -> ..data/default.rule
    lrwxrwxrwx    1 root     root            18 Apr 12 13:50 devops.rule -> ..data/devops.rule
    lrwxrwxrwx    1 root     root            20 Apr 12 13:50 dujingya.rule -> ..data/dujingya.rule
    lrwxrwxrwx    1 root     root            19 Apr 12 13:50 earth-2.rule -> ..data/earth-2.rule
    lrwxrwxrwx    1 root     root            19 Apr 12 13:50 earth-3.rule -> ..data/earth-3.rule
    lrwxrwxrwx    1 root     root            17 Apr 12 13:50 earth.rule -> ..data/earth.rule
    lrwxrwxrwx    1 root     root            20 Apr 12 13:50 guoyiqin.rule -> ..data/guoyiqin.rule
    lrwxrwxrwx    1 root     root            25 Apr 12 13:50 lijiaob-space.rule -> ..data/lijiaob-space.rule
    lrwxrwxrwx    1 root     root            19 Apr 12 13:50 newcicd.rule -> ..data/newcicd.rule
    lrwxrwxrwx    1 root     root            19 Apr 12 13:50 newteam.rule -> ..data/newteam.rule
    lrwxrwxrwx    1 root     root            18 Apr 12 13:50 qateam.rule -> ..data/qateam.rule
    lrwxrwxrwx    1 root     root            22 Apr 12 13:50 taijianxin.rule -> ..data/taijianxin.rule
    lrwxrwxrwx    1 root     root            23 Apr 12 13:50 weihongweic.rule -> ..data/weihongweic.rule
    lrwxrwxrwx    1 root     root            19 Apr 12 13:50 yaoweig.rule -> ..data/yaoweig.rule
    lrwxrwxrwx    1 root     root            23 Apr 12 13:50 zhanghongyi.rule -> ..data/zhanghongyi.rule
    lrwxrwxrwx    1 root     root            20 Apr 12 13:50 zhanghub.rule -> ..data/zhanghub.rule
    /etc/prometheus_rules $ pwd
    /etc/prometheus_rules

/etc/prometheus_rules $ cat admin.rule 
    groups:
    - name: admin.rule
      rules:
      - alert: container_cpu_usage_seconds_total_gt_9000_788a0a004db0bdc447d8f0ed31125bb7
        expr: (ceil(sum(rate(container_cpu_usage_seconds_total{
       namespace="admin",pod_name=~"^fffff-[0-9]{5,15}-[0-9a-zA-Z]{5}$"}[5m]))
          * 100 * 100)/100) > 90
        labels:
          tenxClusterID: CID-ca4135da3326
          tenxNamespace: admin
          tenxStrategyID: STRAID-L3PPd77yxNR6
          tenxStrategyName: tsfffzz
          tenxTargetName: fffff
          tenxTargetType: service
        annotations:
          condition: CPU利用率 > 90%
          createTime: 2017-11-28T15:56:40+08:00
          currentValue: CPU利用率 {
       { $value }}%
          tenxMetricType: cpu/usage_rate
          tokenMD5: 9f834d0df08163e97b1ebb8423003292
      - alert: container_cpu_usage_seconds_total_gt_9000_788a0a004db0bdc447d8f0ed31125bb7
        expr: (ceil(sum(rate(container_cpu_usage_seconds_total{
       namespace="admin",pod_name=~"^fffff-[0-9]{5,15}-[0-9a-zA-Z]{5}$"}[5m]))
          * 100 * 100)/100) > 90
        labels:
          tenxClusterID: CID-ca4135da3326
          tenxNamespace: admin
          tenxStrategyID: STRAID-L3PPd77yxNR6
          tenxStrategyName: tsfffzz
          tenxTargetName: fffff
          tenxTargetType: service
        annotations:
          condition: CPU利用率 > 90%
          createTime: 2017-11-28T15:56:12+08:00
          currentValue: CPU利用率 {
       { $value }}%
          tenxMetricType: cpu/usage_rate
          tokenMD5: 9f834d0df08163e97b1ebb8423003292

apiVersion: extensions/v1beta1
    kind: Deployment
    metadata:
      labels:
        app: prometheus
        plugin: prometheus
      name: prometheus
      namespace: kube-system
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: prometheus
          plugin: prometheus
      strategy:
        rollingUpdate:
          maxSurge: 1
          maxUnavailable: 1
        type: RollingUpdate
      template:
        metadata:
          labels:
            app: prometheus
            plugin: prometheus
        spec:
          containers:
          - args:         - --config.file=/etc/prometheus/prometheus.yaml         - --storage.tsdb.path=/prometheus         - --storage.tsdb.retention=7d         - --web.enable-lifecycle         command:
            - /bin/prometheus         image: harbor.enncloud.cn/enncloud/prometheus:2.0
            imagePullPolicy: IfNotPresent
            name: prometheus
            ports:
            - containerPort: 9090           protocol: TCP
            resources:
              limits:
                cpu: "2"
                memory: 3000Mi
              requests:
                cpu: 10m
                memory: 10Mi
            volumeMounts:
            - mountPath: /prometheus           name: data
            - mountPath: /etc/prometheus           name: config-volume
            - mountPath: /etc/prometheus_rules           name: rules-volume
          - args:         - 'while inotifywait -qq -e modify,create,delete /etc/prometheus_rules/..data/;           do sh -c  "curl -X POST http://localhost:9090/-/reload"; done; '
            image: harbor.enncloud.cn/tenx_containers/inotify:1.0
            imagePullPolicy: IfNotPresent
            name: notify
            volumeMounts:
            - mountPath: /etc/prometheus_rules           name: rules-volume
          dnsPolicy: ClusterFirst
          nodeName: test-slave-116
          restartPolicy: Always
          volumes:
          - hostPath:           path: /paas/prometheus_data
            name: data
          - configMap:           name: prometheus-config
            name: config-volume
          - configMap:           name: prometheus-rules
            name: rules-volume