kubernetes使用prometheus的相关配置文件

﹏ヽ暗。殇╰゛Y 2022-05-25 05:19 278阅读 0赞

本文只是简单记录一下配置文件
prometheus.yaml

  1. # A scrape configuration for running Prometheus on a Kubernetes cluster.
  2. # This uses separate scrape configs for cluster components (i.e. API server, node)
  3. # and services to allow each to use different authentication configs.
  4. #
  5. # Kubernetes labels will be added as Prometheus labels on metrics via the
  6. # `labelmap` relabeling action.
  7. #
  8. # If you are using Kubernetes 1.7.2 or earlier, please take note of the comments
  9. # for the kubernetes-cadvisor job; you will need to edit or remove this job.
  11. # Scrape config for API servers.
  12. #
  13. # Kubernetes exposes API servers as endpoints to the default/kubernetes
  14. # service so this uses `endpoints` role and uses relabelling to only keep
  15. # the endpoints associated with the default/kubernetes service using the
  16. # default named port `https`. This works for single API server deployments as
  17. # well as HA API server deployments.
  18. rule_files: 
  19. - /etc/prometheus_rules/*.rule
  20. scrape_configs:
  21. - job_name: 'kubernetes-apiservers'
  23.   kubernetes_sd_configs:
  24.   - role: endpoints
  26.   # Default to scraping over https. If required, just disable this or change to
  27.   # `http`.
  28.   scheme: https
  30.   # This TLS & bearer token file config is used to connect to the actual scrape
  31.   # endpoints for cluster components. This is separate to discovery auth
  32.   # configuration because discovery & scraping are two separate concerns in
  33.   # Prometheus. The discovery auth config is automatic if Prometheus runs inside
  34.   # the cluster. Otherwise, more config options have to be provided within the
  35.   # <kubernetes_sd_config>.
  36.   tls_config:
  37.     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  38.     # If your node certificates are self-signed or use a different CA to the
  39.     # master CA, then disable certificate verification below. Note that
  40.     # certificate verification is an integral part of a secure infrastructure
  41.     # so this should only be disabled in a controlled environment. You can
  42.     # disable certificate verification by uncommenting the line below.
  43.     #
  44.     # insecure_skip_verify: true
  45.   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  47.   # Keep only the default/kubernetes service endpoints for the https port. This
  48.   # will add targets for each API server which Kubernetes adds an endpoint to
  49.   # the default/kubernetes service.
  50.   relabel_configs:
  51.   - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
  52.     action: keep
  53.     regex: default;kubernetes;https
  55. # Scrape config for nodes (kubelet).
  56. #
  57. # Rather than connecting directly to the node, the scrape is proxied though the
  58. # Kubernetes apiserver. This means it will work if Prometheus is running out of
  59. # cluster, or can't connect to nodes for some other reason (e.g. because of
  60. # firewalling).
  61. - job_name: 'kubernetes-nodes'
  63.   # Default to scraping over https. If required, just disable this or change to
  64.   # `http`.
  65.   scheme: https
  67.   # This TLS & bearer token file config is used to connect to the actual scrape
  68.   # endpoints for cluster components. This is separate to discovery auth
  69.   # configuration because discovery & scraping are two separate concerns in
  70.   # Prometheus. The discovery auth config is automatic if Prometheus runs inside
  71.   # the cluster. Otherwise, more config options have to be provided within the
  72.   # <kubernetes_sd_config>.
  73.   tls_config:
  74.     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  75.   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  77.   kubernetes_sd_configs:
  78.   - role: node
  80.   relabel_configs:
  81.   - action: labelmap
  82.     regex: __meta_kubernetes_node_label_(.+)
  83.   - target_label: __address__
  84.     replacement: kubernetes.default.svc:443
  85.   - source_labels: [__meta_kubernetes_node_name]
  86.     regex: (.+)
  87.     target_label: __metrics_path__
  88.     replacement: /api/v1/nodes/${1}/proxy/metrics
  90. # Scrape config for Kubelet cAdvisor.
  91. #
  92. # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
  93. # (those whose names begin with 'container_') have been removed from the
  94. # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to
  95. # retrieve those metrics.
  96. #
  97. # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor
  98. # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics"
  99. # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with
  100. # the --cadvisor-port=0 Kubelet flag).
  101. #
  102. # This job is not necessary and should be removed in Kubernetes 1.6 and
  103. # earlier versions, or it will cause the metrics to be scraped twice.
  104. - job_name: 'kubernetes-cadvisor'
  106.   # Default to scraping over https. If required, just disable this or change to
  107.   # `http`.
  108.   scheme: https
  110.   # This TLS & bearer token file config is used to connect to the actual scrape
  111.   # endpoints for cluster components. This is separate to discovery auth
  112.   # configuration because discovery & scraping are two separate concerns in
  113.   # Prometheus. The discovery auth config is automatic if Prometheus runs inside
  114.   # the cluster. Otherwise, more config options have to be provided within the
  115.   # <kubernetes_sd_config>.
  116.   tls_config:
  117.     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  118.   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  120.   kubernetes_sd_configs:
  121.   - role: node
  123.   relabel_configs:
  124.   - action: labelmap
  125.     regex: __meta_kubernetes_node_label_(.+)
  126.   - target_label: __address__
  127.     replacement: kubernetes.default.svc:443
  128.   - source_labels: [__meta_kubernetes_node_name]
  129.     regex: (.+)
  130.     target_label: __metrics_path__
  131.     replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
  133. # Scrape config for service endpoints.
  134. #
  135. # The relabeling allows the actual service scrape endpoint to be configured
  136. # via the following annotations:
  137. #
  138. # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
  139. # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
  140. # to set this to `https` & most likely set the `tls_config` of the scrape config.
  141. # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
  142. # * `prometheus.io/port`: If the metrics are exposed on a different port to the
  143. # service then set this appropriately.
  144. - job_name: 'kubernetes-service-endpoints'
  146.   kubernetes_sd_configs:
  147.   - role: endpoints
  149.   relabel_configs:
  150.   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
  151.     action: keep
  152.     regex: true
  153.   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
  154.     action: replace
  155.     target_label: __scheme__
  156.     regex: (https?)
  157.   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
  158.     action: replace
  159.     target_label: __metrics_path__
  160.     regex: (.+)
  161.   - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
  162.     action: replace
  163.     target_label: __address__
  164.     regex: ([^:]+)(?::\d+)?;(\d+)
  165.     replacement: $1:$2
  166.   - action: labelmap
  167.     regex: __meta_kubernetes_service_label_(.+)
  168.   - source_labels: [__meta_kubernetes_namespace]
  169.     action: replace
  170.     target_label: kubernetes_namespace
  171.   - source_labels: [__meta_kubernetes_service_name]
  172.     action: replace
  173.     target_label: kubernetes_name
  175. # Example scrape config for probing services via the Blackbox Exporter.
  176. #
  177. # The relabeling allows the actual service scrape endpoint to be configured
  178. # via the following annotations:
  179. #
  180. # * `prometheus.io/probe`: Only probe services that have a value of `true`
  181. - job_name: 'kubernetes-services'
  183.   metrics_path: /metrics
  185.   kubernetes_sd_configs:
  186.   - role: service
  188.   relabel_configs:
  189.   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
  190.     action: keep
  191.     regex: true
  192.   - source_labels: [__address__]
  193.     target_label: __param_target
  194.   - target_label: __address__
  195.     replacement: blackbox-exporter.example.com:9115
  196.   - source_labels: [__param_target]
  197.     target_label: instance
  198.   - action: labelmap
  199.     regex: __meta_kubernetes_service_label_(.+)
  200.   - source_labels: [__meta_kubernetes_namespace]
  201.     target_label: kubernetes_namespace
  202.   - source_labels: [__meta_kubernetes_service_name]
  203.     target_label: kubernetes_name
  205. # Example scrape config for probing ingresses via the Blackbox Exporter.
  206. #
  207. # The relabeling allows the actual ingress scrape endpoint to be configured
  208. # via the following annotations:
  209. #
  210. # * `prometheus.io/probe`: Only probe services that have a value of `true`
  211. - job_name: 'kubernetes-ingresses'
  213.   metrics_path: /probe
  214.   params:
  215.     module: [http_2xx]
  217.   kubernetes_sd_configs:
  218.     - role: ingress
  220.   relabel_configs:
  221.     - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
  222.       action: keep
  223.       regex: true
  224.     - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
  225.       regex: (.+);(.+);(.+)
  226.       replacement: ${1}://${2}${3}
  227.       target_label: __param_target
  228.     - target_label: __address__
  229.       replacement: blackbox-exporter.example.com:9115
  230.     - source_labels: [__param_target]
  231.       target_label: instance
  232.     - action: labelmap
  233.       regex: __meta_kubernetes_ingress_label_(.+)
  234.     - source_labels: [__meta_kubernetes_namespace]
  235.       target_label: kubernetes_namespace
  236.     - source_labels: [__meta_kubernetes_ingress_name]
  237.       target_label: kubernetes_name
  239. # Example scrape config for pods
  240. #
  241. # The relabeling allows the actual pod scrape endpoint to be configured via the
  242. # following annotations:
  243. #
  244. # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
  245. # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
  246. # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
  247. # pod's declared ports (default is a port-free target if none are declared).
  248. - job_name: 'kubernetes-pods'
  250.   kubernetes_sd_configs:
  251.   - role: pod
  253.   relabel_configs:
  254.   - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
  255.     action: keep
  256.     regex: true
  257.   - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
  258.     action: replace
  259.     target_label: __metrics_path__
  260.     regex: (.+)
  261.   - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
  262.     action: replace
  263.     regex: ([^:]+)(?::\d+)?;(\d+)
  264.     replacement: $1:$2
  265.     target_label: __address__
  266.   - action: labelmap
  267.     regex: __meta_kubernetes_pod_label_(.+)
  268.   - source_labels: [__meta_kubernetes_namespace]
  269.     action: replace
  270.     target_label: kubernetes_namespace
  271.   - source_labels: [__meta_kubernetes_pod_name]
  272.     action: replace
  273.     target_label: kubernetes_pod_name
  275. alerting:
  276.   alertmanagers:
  277.   - static_configs:
  278.     - targets:
  279.        - alertmanager:9093

规则文件

  1. /etc/prometheus_rules $ ls -l
  2. total 0
  3. lrwxrwxrwx    1 root     root            17 Apr 12 13:50 admin.rule -> ..data/admin.rule
  4. lrwxrwxrwx    1 root     root            20 Apr 12 13:50 automoto.rule -> ..data/automoto.rule
  5. lrwxrwxrwx    1 root     root            25 Apr 12 13:50 ceres-dev.rule.yml -> ..data/ceres-dev.rule.yml
  6. lrwxrwxrwx    1 root     root            19 Apr 12 13:50 cicd-17.rule -> ..data/cicd-17.rule
  7. lrwxrwxrwx    1 root     root            19 Apr 17 10:18 default.rule -> ..data/default.rule
  8. lrwxrwxrwx    1 root     root            18 Apr 12 13:50 devops.rule -> ..data/devops.rule
  9. lrwxrwxrwx    1 root     root            20 Apr 12 13:50 dujingya.rule -> ..data/dujingya.rule
  10. lrwxrwxrwx    1 root     root            19 Apr 12 13:50 earth-2.rule -> ..data/earth-2.rule
  11. lrwxrwxrwx    1 root     root            19 Apr 12 13:50 earth-3.rule -> ..data/earth-3.rule
  12. lrwxrwxrwx    1 root     root            17 Apr 12 13:50 earth.rule -> ..data/earth.rule
  13. lrwxrwxrwx    1 root     root            20 Apr 12 13:50 guoyiqin.rule -> ..data/guoyiqin.rule
  14. lrwxrwxrwx    1 root     root            25 Apr 12 13:50 lijiaob-space.rule -> ..data/lijiaob-space.rule
  15. lrwxrwxrwx    1 root     root            19 Apr 12 13:50 newcicd.rule -> ..data/newcicd.rule
  16. lrwxrwxrwx    1 root     root            19 Apr 12 13:50 newteam.rule -> ..data/newteam.rule
  17. lrwxrwxrwx    1 root     root            18 Apr 12 13:50 qateam.rule -> ..data/qateam.rule
  18. lrwxrwxrwx    1 root     root            22 Apr 12 13:50 taijianxin.rule -> ..data/taijianxin.rule
  19. lrwxrwxrwx    1 root     root            23 Apr 12 13:50 weihongweic.rule -> ..data/weihongweic.rule
  20. lrwxrwxrwx    1 root     root            19 Apr 12 13:50 yaoweig.rule -> ..data/yaoweig.rule
  21. lrwxrwxrwx    1 root     root            23 Apr 12 13:50 zhanghongyi.rule -> ..data/zhanghongyi.rule
  22. lrwxrwxrwx    1 root     root            20 Apr 12 13:50 zhanghub.rule -> ..data/zhanghub.rule
  23. /etc/prometheus_rules $ pwd
  24. /etc/prometheus_rules
  26. /etc/prometheus_rules $ cat admin.rule 
  27. groups:
  28. - name: admin.rule
  29.   rules:
  30.   - alert: container_cpu_usage_seconds_total_gt_9000_788a0a004db0bdc447d8f0ed31125bb7
  31.     expr: (ceil(sum(rate(container_cpu_usage_seconds_total{
  32.    namespace="admin",pod_name=~"^fffff-[0-9]{5,15}-[0-9a-zA-Z]{5}$"}[5m]))
  33.       * 100 * 100)/100) > 90
  34.     labels:
  35.       tenxClusterID: CID-ca4135da3326
  36.       tenxNamespace: admin
  37.       tenxStrategyID: STRAID-L3PPd77yxNR6
  38.       tenxStrategyName: tsfffzz
  39.       tenxTargetName: fffff
  40.       tenxTargetType: service
  41.     annotations:
  42.       condition: CPU利用率 > 90%
  43.       createTime: 2017-11-28T15:56:40+08:00
  44.       currentValue: CPU利用率 {
  45.    { $value }}%
  46.       tenxMetricType: cpu/usage_rate
  47.       tokenMD5: 9f834d0df08163e97b1ebb8423003292
  48.   - alert: container_cpu_usage_seconds_total_gt_9000_788a0a004db0bdc447d8f0ed31125bb7
  49.     expr: (ceil(sum(rate(container_cpu_usage_seconds_total{
  50.    namespace="admin",pod_name=~"^fffff-[0-9]{5,15}-[0-9a-zA-Z]{5}$"}[5m]))
  51.       * 100 * 100)/100) > 90
  52.     labels:
  53.       tenxClusterID: CID-ca4135da3326
  54.       tenxNamespace: admin
  55.       tenxStrategyID: STRAID-L3PPd77yxNR6
  56.       tenxStrategyName: tsfffzz
  57.       tenxTargetName: fffff
  58.       tenxTargetType: service
  59.     annotations:
  60.       condition: CPU利用率 > 90%
  61.       createTime: 2017-11-28T15:56:12+08:00
  62.       currentValue: CPU利用率 {
  63.    { $value }}%
  64.       tenxMetricType: cpu/usage_rate
  65.       tokenMD5: 9f834d0df08163e97b1ebb8423003292
  67. apiVersion: extensions/v1beta1
  68. kind: Deployment
  69. metadata:
  70.   labels:
  71.     app: prometheus
  72.     plugin: prometheus
  73.   name: prometheus
  74.   namespace: kube-system
  75. spec:
  76.   replicas: 1
  77.   selector:
  78.     matchLabels:
  79.       app: prometheus
  80.       plugin: prometheus
  81.   strategy:
  82.     rollingUpdate:
  83.       maxSurge: 1
  84.       maxUnavailable: 1
  85.     type: RollingUpdate
  86.   template:
  87.     metadata:
  88.       labels:
  89.         app: prometheus
  90.         plugin: prometheus
  91.     spec:
  92.       containers:
  93.       - args:         - --config.file=/etc/prometheus/prometheus.yaml         - --storage.tsdb.path=/prometheus         - --storage.tsdb.retention=7d         - --web.enable-lifecycle         command:
  94.         - /bin/prometheus         image: harbor.enncloud.cn/enncloud/prometheus:2.0
  95.         imagePullPolicy: IfNotPresent
  96.         name: prometheus
  97.         ports:
  98.         - containerPort: 9090           protocol: TCP
  99.         resources:
  100.           limits:
  101.             cpu: "2"
  102.             memory: 3000Mi
  103.           requests:
  104.             cpu: 10m
  105.             memory: 10Mi
  106.         volumeMounts:
  107.         - mountPath: /prometheus           name: data
  108.         - mountPath: /etc/prometheus           name: config-volume
  109.         - mountPath: /etc/prometheus_rules           name: rules-volume
  110.       - args:         - 'while inotifywait -qq -e modify,create,delete /etc/prometheus_rules/..data/;           do sh -c  "curl -X POST http://localhost:9090/-/reload"; done; '
  111.         image: harbor.enncloud.cn/tenx_containers/inotify:1.0
  112.         imagePullPolicy: IfNotPresent
  113.         name: notify
  114.         volumeMounts:
  115.         - mountPath: /etc/prometheus_rules           name: rules-volume
  116.       dnsPolicy: ClusterFirst
  117.       nodeName: test-slave-116
  118.       restartPolicy: Always
  119.       volumes:
  120.       - hostPath:           path: /paas/prometheus_data
  121.         name: data
  122.       - configMap:           name: prometheus-config
  123.         name: config-volume
  124.       - configMap:           name: prometheus-rules
  125.         name: rules-volume

发表评论

表情：
评论列表 （有 0 条评论，278人围观）

还没有评论，来说两句吧...

相关阅读