一、新增 Kubernetes 集群架构监控

在 prometheus-config.yaml 一次添加如下采集数据：

1.1 kube-apiserver

需要注意的是使用https访问时，需要tls相关配置，可以指定ca证书路径或者 insecure_skip_verify: true跳过证书验证。

除此之外，还要指定 bearer_token_file，否则会提示 server returned HTTP status 400 Bad Request;

# 主要添加如下内容
    ########## prometheus 监控配置 ##########
    - job_name: prometheus
      static_configs:
      - targets: ['127.0.0.1:9090']
        labels:
          instance: prometheus

    ########## kube-apiserver 监控配置 ##########
    - job_name: kube-apiserver
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name]
        action: keep
        regex: default;kubernetes
      - source_labels: [__meta_kubernetes_endpoints_name]
        action: replace
        target_label: endpoint
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: service
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace

# 添加后完整配置如下
[root@master01 7]# cat prometheus-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitor
data:
  prometheus.yml: |
    global:
      scrape_interval:     15s
      evaluation_interval: 15s
      external_labels:
        cluster: "kubernetes"

    ############ 数据采集job ###################

    scrape_configs:
    ########## prometheus 监控配置 ##########
    - job_name: prometheus
      static_configs:
      - targets: ['127.0.0.1:9090']
        labels:
          instance: prometheus

    ########## kube-apiserver 监控配置 ##########
    - job_name: kube-apiserver
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name]
        action: keep
        regex: default;kubernetes
      - source_labels: [__meta_kubernetes_endpoints_name]
        action: replace
        target_label: endpoint
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: service
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace

    ############ 指定告警规则文件路径位置 ###################
    rule_files:
    - /etc/prometheus/rules/*.rules

# 配置生效
[root@master01 7]# kaf prometheus-config.yaml

# 手动热加载
[root@master01 7]# curl -XPOST http://prometheus.zhang-qing.com/-/reload

1.2 controller-manager

  • 查看controller-manager信息（名称可能不太一样，大家注意一下）
# kubectl describe pod -n kube-system kube-controller-manager-master01
Name:                 kube-controller-manager-master01
Namespace:            kube-system
...
Labels:               component=kube-controller-manager
                      tier=control-plane
...
Containers:
  kube-controller-manager:
    ...
    Command:
      kube-controller-manager
      --feature-gates=LegacyServiceAccountTokenNoAutoGeneration=false
      --allocate-node-cidrs=true
      --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
      --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
      --bind-address=127.0.0.1
      --client-ca-file=/etc/kubernetes/pki/ca.crt
      --cluster-cidr=172.16.0.0/12
      --cluster-name=kubernetes
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
      --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
      --controllers=*,bootstrapsigner,tokencleaner
      --kubeconfig=/etc/kubernetes/controller-manager.conf
      --leader-elect=true
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
      --root-ca-file=/etc/kubernetes/pki/ca.crt
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key
      --service-cluster-ip-range=192.168.0.0/16
      --use-service-account-credentials=true
...

由上可知，匹配pod对象，lable标签为component=kube-controller-manager即可，但需注意的是controller-manager默认只运行127.0.0.1访问，因此还需先修改controller-manager配置.

  • 所有的Master节点修改 /etc/kubernetes/manifests/kube-controller-manager.yaml
vim /etc/kubernetes/manifests/kube-controller-manager.yaml
#修改第18行内容,将bind-address的地址由原来的127.0.0.1修改为0.0.0.0
...
...
  - command:
    #端口改为0.0.0.0
    - --bind-address=0.0.0.0
    #- --port=0 # 注释0端口
...
...

#修改完成后，重新应用
kaf /etc/kubernetes/manifests/kube-controller-manager.yaml

#查看暴露的端口
[root@master01 7]# netstat -lntup | grep kube-controll
tcp6       0      0 :::10257                :::*                    LISTEN      57876/kube-controll
  • 编写prometheus配置文件，需要注意的是，他默认匹配到的是80端口，需要手动指定为10257端口;
# 添加如下配置信息
    ########## kube-controller-manager 监控配置 ##########
    - job_name: 'kube-controller-manager'
      # 使用 Kubernetes Pod 发现机制
      kubernetes_sd_configs:
        - role: pod
      # 强制使用 HTTPS 协议
      scheme: https
      # TLS 配置（测试环境跳过验证）
      tls_config:
        insecure_skip_verify: true
      # 使用 ServiceAccount 的 Token 认证
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
        # 保留标签为 component=kube-controller-manager 的 Pod
        - source_labels: [__meta_kubernetes_pod_label_component]
          regex: kube-controller-manager
          action: keep
        # 重写目标地址为 Pod IP + 10257 端口
        - source_labels: [__meta_kubernetes_pod_ip]
          regex: (.+)
          target_label: __address__
          replacement: "${1}:10257"
        # 强制使用 HTTPS 协议（冗余但明确）
        - source_labels: []
          regex: .*
          target_label: __scheme__
          replacement: https
        # 附加元数据标签
        - source_labels: [__meta_kubernetes_endpoints_name]
          action: replace
          target_label: endpoint
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: pod
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: service
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: namespace

# 添加后完整配置如下
[root@master01 7]# vim prometheus-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitor
data:
  prometheus.yml: |
    global:
      scrape_interval:     15s
      evaluation_interval: 15s
      external_labels:
        cluster: "kubernetes"

    ############ 数据采集job ###################

    scrape_configs:
    ########## prometheus 监控配置 ##########
    - job_name: prometheus
      static_configs:
      - targets: ['127.0.0.1:9090']
        labels:
          instance: prometheus

    ########## kube-apiserver 监控配置 ##########
    - job_name: kube-apiserver
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name]
        action: keep
        regex: default;kubernetes
      - source_labels: [__meta_kubernetes_endpoints_name]
        action: replace
        target_label: endpoint
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: service
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace

    ########## kube-controller-manager 监控配置 ##########
    - job_name: 'kube-controller-manager'
      # 使用 Kubernetes Pod 发现机制
      kubernetes_sd_configs:
        - role: pod
      # 强制使用 HTTPS 协议
      scheme: https
      # TLS 配置（测试环境跳过验证）
      tls_config:
        insecure_skip_verify: true
      # 使用 ServiceAccount 的 Token 认证
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
        # 保留标签为 component=kube-controller-manager 的 Pod
        - source_labels: [__meta_kubernetes_pod_label_component]
          regex: kube-controller-manager
          action: keep
        # 重写目标地址为 Pod IP + 10257 端口
        - source_labels: [__meta_kubernetes_pod_ip]
          regex: (.+)
          target_label: __address__
          replacement: "${1}:10257"
        # 强制使用 HTTPS 协议（冗余但明确）
        - source_labels: []
          regex: .*
          target_label: __scheme__
          replacement: https
        # 附加元数据标签
        - source_labels: [__meta_kubernetes_endpoints_name]
          action: replace
          target_label: endpoint
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: pod
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: service
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: namespace

    ############ 指定告警规则文件路径位置 ###################
    rule_files:
    - /etc/prometheus/rules/*.rules

# 配置生效
[root@master01 7]# kaf prometheus-config.yaml

# 手动热加载
[root@master01 7]# curl -XPOST http://prometheus.zhang-qing.com/-/reload