Kubernetes资源监控与告警：从指标到行动的完整闭环

Kubernetes资源监控与告警从指标到行动的完整闭环没有监控的集群就是黑盒没有告警的监控就是摆设。监控体系架构一个完整的K8s监控体系包含三个层次┌─────────────────────────────────────────┐ │ 应用层监控 (APM) │ │ - 业务指标、链路追踪、日志 │ ├─────────────────────────────────────────┤ │ 中间件层监控 │ │ - 数据库、缓存、消息队列 │ ├─────────────────────────────────────────┤ │ Kubernetes层监控 │ │ - Pod、Node、Control Plane │ ├─────────────────────────────────────────┤ │ 基础设施层监控 │ │ - 服务器、网络、存储 │ └─────────────────────────────────────────┘Prometheus监控栈部署1. 完整监控架构# prometheus-stack.yaml apiVersion: v1 kind: Namespace metadata: name: monitoring --- # Prometheus配置 apiVersion: v1 kind: ConfigMap metadata: name: prometheus-config namespace: monitoring data: prometheus.yml: | global: scrape_interval: 15s evaluation_interval: 15s external_labels: cluster: production replica: {{.ExternalURL}} rule_files: - /etc/prometheus/rules/*.yml alerting: alertmanagers: - static_configs: - targets: [alertmanager:9093] scrape_configs: # Kubernetes API Server - job_name: kubernetes-apiservers kubernetes_sd_configs: - role: endpoints scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] action: keep regex: default;kubernetes;https # Kubelet - job_name: kubernetes-nodes kubernetes_sd_configs: - role: node scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.) # Pod监控 - job_name: kubernetes-pods kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: ([^:])(?::\d)?;(\d) replacement: $1:$2 target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod # Service监控 - job_name: kubernetes-services kubernetes_sd_configs: - role: service metrics_path: /probe params: module: [http_2xx] relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] action: keep regex: true - source_labels: [__address__] target_label: __param_target - target_label: __address__ replacement: blackbox-exporter:9115 - source_labels: [__param_target] target_label: instance - action: labelmap regex: __meta_kubernetes_service_label_(.) - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_service_name] target_label: service # cAdvisor - job_name: kubernetes-cadvisor kubernetes_sd_configs: - role: node scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token metrics_path: /metrics/cadvisor relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.) --- # Prometheus Deployment apiVersion: apps/v1 kind: Deployment metadata: name: prometheus namespace: monitoring spec: replicas: 1 selector: matchLabels: app: prometheus template: metadata: labels: app: prometheus spec: serviceAccountName: prometheus containers: - name: prometheus image: prom/prometheus:v2.47.0 args: - --config.file/etc/prometheus/prometheus.yml - --storage.tsdb.path/prometheus - --storage.tsdb.retention.time15d - --web.console.libraries/usr/share/prometheus/console_libraries - --web.console.templates/usr/share/prometheus/consoles - --web.enable-lifecycle - --web.enable-admin-api ports: - containerPort: 9090 name: web volumeMounts: - name: config mountPath: /etc/prometheus - name: storage mountPath: /prometheus resources: requests: cpu: 500m memory: 2Gi limits: cpu: 2000m memory: 8Gi volumes: - name: config configMap: name: prometheus-config - name: storage persistentVolumeClaim: claimName: prometheus-storage2. 记录规则优化# recording-rules.yaml apiVersion: v1 kind: ConfigMap metadata: name: prometheus-recording-rules namespace: monitoring data: rules.yml: | groups: # 节点资源使用 - name: node_resources interval: 30s rules: - record: node:cpu_utilization:rate5m expr: | 100 - (avg by (instance) (irate(node_cpu_seconds_total{modeidle}[5m])) * 100) - record: node:memory_utilization:percent expr: | (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 - record: node:disk_utilization:percent expr: | (1 - (node_filesystem_avail_bytes{mountpoint/} / node_filesystem_size_bytes{mountpoint/})) * 100 - record: node:network_receive_bytes:rate5m expr: | sum by (instance) (irate(node_network_receive_bytes_total[5m])) - record: node:network_transmit_bytes:rate5m expr: | sum by (instance) (irate(node_network_transmit_bytes_total[5m])) # Pod资源使用 - name: pod_resources interval: 30s rules: - record: pod:cpu_utilization:percent expr: | (container_cpu_usage_seconds_total / kube_pod_container_resource_limits{resourcecpu}) * 100 - record: pod:memory_utilization:percent expr: | (container_memory_working_set_bytes / kube_pod_container_resource_limits{resourcememory}) * 100 - record: pod:restart_rate:5m expr: | rate(kube_pod_container_status_restarts_total[5m]) - record: pod:oom_kills:total expr: | increase(container_oom_events_total[1h]) # 集群整体指标 - name: cluster_resources interval: 60s rules: - record: cluster:cpu_allocatable:total expr: | sum(kube_node_status_allocatable{resourcecpu}) - record: cluster:cpu_request:total expr: | sum(kube_pod_container_resource_requests{resourcecpu}) - record: cluster:cpu_utilization:percent expr: | (cluster:cpu_request:total / cluster:cpu_allocatable:total) * 100 - record: cluster:pod_count:total expr: | count(kube_pod_info) - record: cluster:node_count:total expr: | count(kube_node_info)告警规则体系1. 分层告警策略# alert-rules.yaml apiVersion: v1 kind: ConfigMap metadata: name: prometheus-alert-rules namespace: monitoring data: alerts.yml: | groups: # P0 - 立即处理 - name: critical_alerts rules: - alert: KubernetesNodeNotReady expr: | kube_node_status_condition{ conditionReady, statustrue } 0 for: 5m labels: severity: critical priority: P0 annotations: summary: Kubernetes节点不可用 description: 节点{{ $labels.node }}已不可用超过5分钟 runbook_url: https://wiki/runbooks/node-not-ready - alert: KubernetesPodCrashLooping expr: | rate(kube_pod_container_status_restarts_total[15m]) 0 for: 5m labels: severity: critical priority: P0 annotations: summary: Pod反复重启 description: Pod {{ $labels.namespace }}/{{ $labels.pod }} 在过去15分钟内重启{{ $value }}次 - alert: KubernetesOutOfMemory expr: | container_memory_working_set_bytes / container_spec_memory_limit_bytes 0.95 for: 2m labels: severity: critical priority: P0 annotations: summary: Pod内存即将耗尽 description: Pod {{ $labels.pod }} 内存使用率超过95% - alert: KubernetesDiskPressure expr: | kube_node_status_condition{ conditionDiskPressure, statustrue } 1 for: 2m labels: severity: critical priority: P0 annotations: summary: 节点磁盘压力 description: 节点{{ $labels.node }}磁盘压力警告 # P1 - 1小时内处理 - name: high_priority_alerts rules: - alert: KubernetesCPUHigh expr: | 100 - (avg by (instance) (irate(node_cpu_seconds_total{modeidle}[5m])) * 100) 80 for: 10m labels: severity: warning priority: P1 annotations: summary: 节点CPU使用率过高 description: 节点{{ $labels.instance }} CPU使用率超过80% - alert: KubernetesMemoryHigh expr: | (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 85 for: 10m labels: severity: warning priority: P1 annotations: summary: 节点内存使用率过高 description: 节点{{ $labels.instance }} 内存使用率超过85% - alert: KubernetesDiskFull expr: | (1 - (node_filesystem_avail_bytes{mountpoint/} / node_filesystem_size_bytes{mountpoint/})) * 100 85 for: 5m labels: severity: warning priority: P1 annotations: summary: 节点磁盘使用率过高 description: 节点{{ $labels.instance }} 磁盘使用率超过85% - alert: KubernetesPodPending expr: | kube_pod_status_phase{phasePending} 1 for: 15m labels: severity: warning priority: P1 annotations: summary: Pod长时间Pending description: Pod {{ $labels.namespace }}/{{ $labels.pod }} 已Pending超过15分钟 # P2 - 4小时内处理 - name: medium_priority_alerts rules: - alert: KubernetesHighPodRestart expr: | increase(kube_pod_container_status_restarts_total[1h]) 3 for: 0m labels: severity: info priority: P2 annotations: summary: Pod重启次数较多 description: Pod {{ $labels.namespace }}/{{ $labels.pod }} 在过去1小时内重启超过3次 - alert: KubernetesNetworkReceiveHigh expr: | irate(node_network_receive_bytes_total[5m]) 1000000000 # 1GB/s for: 10m labels: severity: info priority: P2 annotations: summary: 网络接收流量过高 description: 节点{{ $labels.instance }} 网络接收流量超过1GB/s - alert: KubernetesResourceQuotaHigh expr: | kube_resourcequota{resourcerequests.cpu,typeused} / kube_resourcequota{resourcerequests.cpu,typehard} 0.85 for: 15m labels: severity: info priority: P2 annotations: summary: 资源配额使用率过高 description: 命名空间{{ $labels.namespace }} CPU配额使用率超过85% # P3 - 24小时内处理 - name: low_priority_alerts rules: - alert: KubernetesJobFailed expr: | kube_job_status_failed 1 for: 0m labels: severity: info priority: P3 annotations: summary: Job执行失败 description: Job {{ $labels.namespace }}/{{ $labels.job_name }} 执行失败 - alert: KubernetesCertificateExpiring expr: | (probe_ssl_earliest_cert_expiry - time()) / 86400 30 for: 0m labels: severity: info priority: P3 annotations: summary: 证书即将过期 description: 证书将在{{ $value }}天后过期2. Alertmanager配置# alertmanager-config.yaml apiVersion: v1 kind: ConfigMap metadata: name: alertmanager-config namespace: monitoring data: alertmanager.yml: | global: smtp_smarthost: smtp.example.com:587 smtp_from: alertsexample.com smtp_auth_username: alertsexample.com smtp_auth_password: password slack_api_url: https://hooks.slack.com/services/xxx pagerduty_url: https://events.pagerduty.com/v2/enqueue templates: - /etc/alertmanager/templates/*.tmpl route: receiver: default group_by: [alertname, priority, namespace] group_wait: 30s group_interval: 5m repeat_interval: 4h routes: # P0告警 - 立即电话通知 - match: priority: P0 receiver: p0-team group_wait: 0s repeat_interval: 5m continue: true # P1告警 - 短信邮件 - match: priority: P1 receiver: p1-team group_wait: 1m repeat_interval: 30m continue: true # P2告警 - 邮件Slack - match: priority: P2 receiver: p2-team group_wait: 5m repeat_interval: 2h # P3告警 - 仅邮件 - match: priority: P3 receiver: p3-team group_wait: 10m repeat_interval: 24h # 按命名空间路由 - match_re: namespace: production|core receiver: production-team routes: - match: severity: critical receiver: production-oncall inhibit_rules: # 高级别告警抑制低级别 - source_match: severity: critical target_match: severity: warning equal: [alertname, namespace] - source_match: alertname: KubernetesNodeNotReady target_match_re: alertname: KubernetesCPUHigh|KubernetesMemoryHigh equal: [instance] receivers: - name: default email_configs: - to: opsexample.com send_resolved: true - name: p0-team pagerduty_configs: - service_key: pagerduty-integration-key severity: critical description: {{ .GroupLabels.alertname }} slack_configs: - channel: #alerts-critical send_resolved: true title: P0 Alert: {{ .GroupLabels.alertname }} text: | {{ range .Alerts }} *Summary:* {{ .Annotations.summary }} *Description:* {{ .Annotations.description }} *Runbook:* {{ .Annotations.runbook_url }} {{ end }} webhook_configs: - url: http://phone-call-service:8080/call send_resolved: false - name: p1-team slack_configs: - channel: #alerts-high send_resolved: true title: ⚠️ P1 Alert: {{ .GroupLabels.alertname }} email_configs: - to: oncallexample.com send_resolved: true - name: p2-team slack_configs: - channel: #alerts-medium send_resolved: true title: P2 Alert: {{ .GroupLabels.alertname }} - name: p3-team email_configs: - to: teamexample.com send_resolved: trueGrafana仪表板1. 集群概览仪表板{ dashboard: { title: Kubernetes Cluster Overview, tags: [k8s, overview], timezone: browser, panels: [ { id: 1, title: Cluster CPU Utilization, type: stat, targets: [ { expr: cluster:cpu_utilization:percent, legendFormat: CPU Usage } ], fieldConfig: { defaults: { thresholds: { steps: [ {color: green, value: null}, {color: yellow, value: 70}, {color: red, value: 85} ] }, unit: percent } } }, { id: 2, title: Cluster Memory Utilization, type: stat, targets: [ { expr: (1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100, legendFormat: Memory Usage } ] }, { id: 3, title: Node Status, type: table, targets: [ { expr: kube_node_status_condition{condition\Ready\}, format: table, instant: true } ] }, { id: 4, title: Pod Status by Namespace, type: piechart, targets: [ { expr: count by (namespace, phase) (kube_pod_status_phase), legendFormat: {{ namespace }} - {{ phase }} } ] }, { id: 5, title: Resource Usage Trend, type: graph, targets: [ { expr: cluster:cpu_utilization:percent, legendFormat: CPU }, { expr: (1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100, legendFormat: Memory } ] } ] } }2. Pod资源详情仪表板# pod-dashboard.yaml apiVersion: v1 kind: ConfigMap metadata: name: grafana-pod-dashboard namespace: monitoring labels: grafana_dashboard: 1 data: pod-dashboard.json: | { dashboard: { title: Pod Resource Details, tags: [k8s, pod], templating: { list: [ { name: namespace, type: query, query: label_values(kube_pod_info, namespace) }, { name: pod, type: query, query: label_values(kube_pod_info{namespace~\$namespace\}, pod) } ] }, panels: [ { title: CPU Usage, type: graph, targets: [ { expr: sum(rate(container_cpu_usage_seconds_total{namespace\$namespace\, pod\$pod\, container!\\}[5m])) by (container), legendFormat: {{ container }} } ] }, { title: Memory Usage, type: graph, targets: [ { expr: container_memory_working_set_bytes{namespace\$namespace\, pod\$pod\, container!\\}, legendFormat: {{ container }} } ] }, { title: Network I/O, type: graph, targets: [ { expr: rate(container_network_receive_bytes_total{namespace\$namespace\, pod\$pod\}[5m]), legendFormat: Receive }, { expr: rate(container_network_transmit_bytes_total{namespace\$namespace\, pod\$pod\}[5m]), legendFormat: Transmit } ] }, { title: Restart Count, type: stat, targets: [ { expr: kube_pod_container_status_restarts_total{namespace\$namespace\, pod\$pod\}, legendFormat: {{ container }} } ] } ] } }自定义指标暴露1. 应用指标SDK示例# app_metrics.py from prometheus_client import Counter, Histogram, Gauge, Info, start_http_server import time import random # 定义指标 REQUEST_COUNT Counter( app_requests_total, Total requests, [method, endpoint, status] ) REQUEST_LATENCY Histogram( app_request_duration_seconds, Request latency, [method, endpoint], buckets[0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0] ) ACTIVE_CONNECTIONS Gauge( app_active_connections, Number of active connections ) QUEUE_SIZE Gauge( app_queue_size, Current queue size, [queue_name] ) APP_INFO Info( app_info, Application information ) # 设置应用信息 APP_INFO.info({ version: 2.0.0, build_time: 2026-03-28, git_commit: abc123 }) def track_request(method, endpoint, status, duration): 追踪请求 REQUEST_COUNT.labels( methodmethod, endpointendpoint, statusstatus ).inc() REQUEST_LATENCY.labels( methodmethod, endpointendpoint ).observe(duration) def update_queue_size(queue_name, size): 更新队列大小 QUEUE_SIZE.labels(queue_namequeue_name).set(size) # 启动metrics服务器 if __name__ __main__: start_http_server(9090) print(Metrics server started on port 9090) # 模拟应用运行 ACTIVE_CONNECTIONS.set(100) while True: # 模拟请求 duration random.uniform(0.01, 0.5) track_request(GET, /api/users, 200, duration) # 模拟队列变化 update_queue_size(order_queue, random.randint(0, 1000)) time.sleep(1)2. 自定义指标ServiceMonitor# custom-metrics-servicemonitor.yaml apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: custom-app-metrics namespace: monitoring labels: release: prometheus spec: namespaceSelector: matchNames: - production - staging selector: matchLabels: metrics: enabled endpoints: - port: metrics path: /metrics interval: 15s scrapeTimeout: 10s honorLabels: true metricRelabelings: - sourceLabels: [__name__] regex: app_(.*) targetLabel: component replacement: application - sourceLabels: [__name__] regex: db_(.*) targetLabel: component replacement: database日志监控集成1. Loki日志收集# loki-config.yaml apiVersion: v1 kind: ConfigMap metadata: name: loki-config namespace: monitoring data: loki.yml: | auth_enabled: false server: http_listen_port: 3100 ingester: lifecycler: address: 127.0.0.1 ring: kvstore: store: inmemory replication_factor: 1 chunk_idle_period: 5m chunk_retain_period: 30s schema_config: configs: - from: 2026-01-01 store: boltdb object_store: filesystem schema: v11 index: prefix: index_ period: 168h storage_config: boltdb: directory: /loki/index filesystem: directory: /loki/chunks limits_config: enforce_metric_name: false reject_old_samples: true reject_old_samples_max_age: 168h --- # Promtail配置 apiVersion: v1 kind: ConfigMap metadata: name: promtail-config namespace: monitoring data: promtail.yml: | server: http_listen_port: 9080 grpc_listen_port: 0 positions: filename: /tmp/positions.yaml clients: - url: http://loki:3100/loki/api/v1/push scrape_configs: - job_name: kubernetes-pods kubernetes_sd_configs: - role: pod pipeline_stages: - docker: {} relabel_configs: - source_labels: [__meta_kubernetes_pod_node_name] target_label: __host__ - action: labelmap regex: __meta_kubernetes_pod_label_(.) - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_pod_name] target_label: pod - source_labels: [__meta_kubernetes_pod_container_name] target_label: container总结完整的监控告警体系需要多层监控基础设施、K8s、应用全覆盖合理告警分层分级避免告警疲劳可视化Grafana仪表板直观展示自动化告警触发自动处理持续优化根据实际调整阈值记住监控不是目的快速发现和解决问题才是。