Kubernetes1.27容器化部署Prometheus

GitHub链接

https://github.com/prometheus-operator/kube-prometheus

根据自己的k8s版本选择对应的版本

# git设置代理
git config --global http.proxy IP地址:端口# 1.27对应的版本为release-0.13
git clone -b release-0.13 https://github.com/prometheus-operator/kube-prometheus.gitcd kube-prometheus/

修改镜像地址

cd manifests# 可以通过如下命令来查看
ls | xargs -I {} grep -iH "image:" {}# alertmanager-alertmanager.yaml
quay.io/prometheus/alertmanager:v0.26.0
swr.cn-north-4.myhuaweicloud.com/ctl456/alertmanager:v0.26.0# blackboxExporter-deployment.yaml
quay.io/prometheus/blackbox-exporter:v0.24.0
swr.cn-north-4.myhuaweicloud.com/ctl456/blackbox-exporter:v0.24.0jimmidyson/configmap-reload:v0.5.0
swr.cn-north-4.myhuaweicloud.com/ctl456/configmap-reload:v0.5.0quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2# grafana-deployment.yaml
grafana/grafana:9.5.3
swr.cn-north-4.myhuaweicloud.com/ctl456/grafana:9.5.3# kubeStateMetrics-deployment.yaml
registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.9.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-state-metrics:v2.9.2quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2# nodeExporter-daemonset.yaml
quay.io/prometheus/node-exporter:v1.6.1
swr.cn-north-4.myhuaweicloud.com/ctl456/node-exporter:v1.6.1quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2# prometheusAdapter-deployment.yaml
registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.11.1
swr.cn-north-4.myhuaweicloud.com/ctl456/prometheus-adapter:v0.11.1# prometheusOperator-deployment.yaml
quay.io/prometheus-operator/prometheus-operator:v0.67.1
swr.cn-north-4.myhuaweicloud.com/ctl456/prometheus-operator:v0.67.1quay.io/brancz/kube-rbac-proxy:v0.14.2
swr.cn-north-4.myhuaweicloud.com/ctl456/kube-rbac-proxy:v0.14.2# prometheus-prometheus.yaml
quay.io/prometheus/prometheus:v2.46.0
swr.cn-north-4.myhuaweicloud.com/ctl456/prometheus:v2.46.0

部署命令

kubectl apply --server-side -f manifests/setup
kubectl wait \--for condition=Established \--all CustomResourceDefinition \--namespace=monitoring
kubectl apply -f manifests/

# 查看是否全部运行
kubectl get svc,pod -n monitoring# 修改type为NodePort
kubectl edit svc grafana -n monitoring# 删除规则
kubectl -n monitoring delete networkpolicy --all# grafana默认账号密码
admin
admin

对Etcd集群进行监控（云原生监控）

Etcd的Metrics接口需要https协议访问

# 先获取证书文件目录
grep -E "key-file|cert-file" /etc/kubernetes/manifests/etcd.yamlcurl -s --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://IP地址:2379/metrics -k | tail -1

创建Etcd Service

vim etcd-svc.yaml

apiVersion: v1
kind: Service
metadata:labels:app: etcd-promname: etcd-promnamespace: kube-system
spec:ports:- name: https-metricsport: 2379protocol: TCPtargetPort: 2379selector: # 添加标签选择器component: etcd # 容器etcd的标签type: ClusterIP
---
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:labels:app: etcd-promkubernetes.io/service-name: etcd-promname: etcd-promnamespace: kube-system
addressType: IPv4
ports:- name: https-metricsport: 2379protocol: TCP
endpoints:- addresses:- "YOUR_ETCD_IP01" # Etcd集群的IP地址- "YOUR_ETCD_IP02"- "YOUR_ETCD_IP03"

kubectl apply -f etcd-svc.yaml
kubectl get svc -n kube-system etcd-prom#通过curl测试访问
curl -s --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://CLUSTER-IP:2379/metrics -k | tail -1

创建Etcd证书的Secret

# 创建Etcd证书的Secret
kubectl create secret generic etcd-ssl --from-file=/etc/kubernetes/pki/etcd/ca.crt --from-file=/etc/kubernetes/pki/etcd/server.crt --from-file=/etc/kubernetes/pki/etcd/server.key -n monitoring# 将证书挂载到容器里
kubectl edit prometheus k8s -n monitoring

# 检查是否挂载成功
kubectl exec -n monitoring prometheus-k8s-0 -c prometheus -- ls /etc/prometheus/secrets/etcd-ssl/

创建Etcd ServiceMonitor

vim servicemonitor.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:name: etcdnamespace: monitoringlabels:app: etcd
spec:jobLabel: k8s-appendpoints:- interval: 30sport: https-metrics  # 这个port对应 Service.spec.ports.namescheme: httpstlsConfig:caFile: /etc/prometheus/secrets/etcd-ssl/ca.crt #证书路径certFile: /etc/prometheus/secrets/etcd-ssl/server.crtkeyFile: /etc/prometheus/secrets/etcd-ssl/server.keyinsecureSkipVerify: true  # 关闭证书校验selector:matchLabels:app: etcd-prom  # 跟Service的lables保持一致namespaceSelector:matchNames:- kube-system

kubectl create -f servicemonitor.yaml

grafana导入模板

成功截图

对MySQL进行监控（非云原生监控）

部署mysql

apiVersion: v1
kind: Service
metadata:name: wordpress-mysqllabels:app: wordpress
spec:ports:- protocol: TCPport: 3306targetPort: 3306selector:app: wordpresstier: mysqltype: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:name: wordpress-mysqllabels:app: wordpresstier: mysql
spec:selector:matchLabels:app: wordpresstier: mysqlstrategy:type: Recreatetemplate:metadata:labels:app: wordpresstier: mysqlspec:containers:- image: swr.cn-north-4.myhuaweicloud.com/ctl456/mysql:5.6name: mysqlenv:- name: MYSQL_ROOT_PASSWORDvalue: changemeports:- containerPort: 3306name: mysqlvolumeMounts:- name: tz-configmountPath: /etc/localtimevolumes:- name: tz-confighostPath:path: /usr/share/zoneinfo/Asia/Shanghaitype: File

# 查看服务是否启动
kubectl get svc,pod

创建用户和权限

kubectl exec -it mysqlpod名称 -- bash

CREATE USER 'exporter'@'%' IDENTIFIED BY 'exporter';
GRANT PROCESS,REPLICATION CLIENT,SELECT ON *.* TO 'exporter'@'%';

配置Exporter

vim mysql-exporter.yaml

---
apiVersion: apps/v1
kind: Deployment
metadata:name: mysql-exporternamespace: monitoring
spec:replicas: 1selector:matchLabels:k8s-app: mysql-exportertemplate:metadata:labels:k8s-app: mysql-exporterspec:containers:- name: mysql-exporterimage: registry.cn-beijing.aliyuncs.com/dotbalo/mysqld-exporter env:- name: DATA_SOURCE_NAMEvalue: "exporter:exporter@(wordpress-mysql:3306)/" # 用户名:密码@(数据库地址:端口)imagePullPolicy: IfNotPresentports:- containerPort: 9104
---
apiVersion: v1
kind: Service
metadata:name: mysql-exporternamespace: monitoringlabels:k8s-app: mysql-exporter
spec:type: ClusterIPselector:k8s-app: mysql-exporterports:- name: apiport: 9104protocol: TCP

kubectl apply -f mysql-exporter.yaml

配置ServiceMonitor

vim mysql-sm.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:name: mysql-exporternamespace: monitoringlabels:k8s-app: mysql-exporternamespace: monitoring
spec:jobLabel: k8s-appendpoints:- port: apiinterval: 30sscheme: httpselector:matchLabels:k8s-app: mysql-exporternamespaceSelector:matchNames:- monitoring

grafana导入模板

6239

成功截图

静态配置

创建一个空文件

touch prometheus-additional.yaml

创建secret

kubectl create secret generic additional-config --from-file=prometheus-additional.yaml -n monitoring

编辑Prometheus的配置

kubectl edit prometheus -n monitoring

additionalScrapeConfigs:key: prometheus-additional.yamlname: additional-configoptional: true

编辑prometheus-additional.yaml

vim prometheus-additional.yaml

- job_name: "blackbox"metrics_path: /probeparams:module: [http_2xx]static_configs:- targets:- https://www.baidu.comrelabel_configs:- source_labels: [__address__]target_label: __param_target- source_labels: [__param_target]target_label: instance- target_label: __address__replacement: blackbox-exporter:19115

# 更新配置
kubectl create secret generic additional-config --from-file=prometheus-additional.yaml --dry-run=client -o yaml | kubectl replace -f - -n monitoring

grafana导入模板

13659

成功截图

监控Windows（外部）主机

下载Windows_exporter并安装

GitHub链接：https://github.com/prometheus-community/windows_exporter/releases

编辑prometheus-additional.yaml

vim prometheus-additional.yaml

- job_name: "WindowsServerMonitor"static_configs:- targets:- "192.168.1.5:9182" # windowsIP地址labels:server_type: "windows"relabel_configs:- source_labels: [__address__]target_label: instance

# 更新配置
kubectl create secret generic additional-config --from-file=prometheus-additional.yaml --dry-run=client -o yaml | kubectl replace -f - -n monitoring

grafana导入模板

12566

成功截图

Alertmanager告警

Alertmanager邮件通告

cd kube-prometheus/manifests/

vim alertmanager-secret.yaml

# global配置
smtp_from: "[e-mail]" # 163邮箱地址
smtp_smarthost: "smtp.163.com:465" # 163邮箱SMTP服务器地址
smtp_hello: "163.com"
smtp_auth_username: "[e-mail]" # 163邮箱地址
smtp_auth_password: "xxxxxx" # 授权码
smtp_require_tls: false# receivers配置
"email_configs":
- to: "xxxxx" # 收件人邮箱地址send_resolved: true

# 使配置生效
kubectl replace -f alertmanager-secret.yaml