k8s1.30.0高可用集群部署

负载均衡

nginx负载均衡

两台nginx负载均衡
vim /etc/nginx/nginx.conf

stream {upstream kube-apiserver {server 192.168.0.11:6443     max_fails=3 fail_timeout=30s;#server 192.168.0.12:6443     max_fails=3 fail_timeout=30s;#server 192.168.0.13:6443     max_fails=3 fail_timeout=30s;}server {listen 6443;proxy_connect_timeout 2s;proxy_timeout 900s;proxy_pass kube-apiserver;}
}

keepalived

nginx检测脚本
vim /data/shell/check_nginx_status.sh

#!/bin/bash
nginx_status=$(ps -ef | grep nginx | grep -v grep | grep -v check | wc -l)if [ $nginx_status -eq 0 ];thensystemctl stop keepalived.service
fi

master节点keepalived配置（不抢占机制）
vim /etc/keepalived/keepalived.conf

global_defs {router_id real-server1}vrrp_script chk_nginx {script "/data/shell/check_nginx_status.sh"interval 2
}vrrp_instance VI_1 {state BACKUPinterface ens32virtual_router_id 151priority 100nopreemptadvert_int 5authentication {auth_type  PASSauth_pass  1111}virtual_ipaddress {192.168.0.10/24}track_script {                                                                                  chk_nginx}}

backup节点keepalived配置

vim /etc/keepalived/keepalived.conf

global_defs {router_id real-server2}vrrp_script chk_nginx {script "/data/shell/check_nginx_status.sh"interval 2
}vrrp_instance VI_1 {state BACKUPinterface ens32virtual_router_id 151priority 50nopreemptadvert_int 5authentication {auth_type  PASSauth_pass  1111}virtual_ipaddress {192.168.0.10/24}track_script {                                                                                  chk_nginx}}

k8s节点系统设置

vim /etc/modules-load.d/containerd.conf

overlay
br_netfilter

modprobe overlay
modprobe br_netfilter

vim /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720

sysctl --system

ipvsadm

yum install ipset ipvsadm

modprobe br_netfilter
modprobe overlay
modprobe ip_conntrack
modprobe  ip_vs
modprobe  ip_vs_rr
modprobe  ip_vs_wrr
modprobe  ip_vs_sh
modprobe  nf_conntrack
lsmod | grep conntrack
lsmod | grep br_netfilt
lsmod | grep overlay
lsmod |egrep  "ip_vs|nf_conntrack"

cat > /etc/modules-load.d/kubernetes.conf << EOF
# /etc/modules-load.d/kubernetes.conf
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
ip_tables
EOF

chmod a+x /etc/modules-load.d/kubernetes.conf

containerd

wget https://github.com/containerd/containerd/releases/download/v1.7.23/cri-containerd-cni-1.7.23-linux-amd64.tar.gztar xvf cri-containerd-cni-1.7.23-linux-amd64.tar.gz -C /wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
cp runc.amd64 /usr/local/sbin/runc

mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

vim /etc/containerd/config.toml

...
SystemdCgroup = true
...
sandbox_image = "registry.cn-beijing.aliyuncs.com/wuxingge123/pause:3.9"
...[plugins."io.containerd.grpc.v1.cri".registry.mirrors][plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]endpoint = ["https://p4oudlho.mirror.aliyuncs.com"]

配置私有仓库

[plugins."io.containerd.grpc.v1.cri".registry][plugins."io.containerd.grpc.v1.cri".registry.mirrors][plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]endpoint = ["https://bqr1dr1n.mirror.aliyuncs.com"][plugins."io.containerd.grpc.v1.cri".registry.configs][plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".tls]insecure_skip_verify = true[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".auth]username = "admin"password = "Harbor12345"

启动containerd

systemctl start containerd.service
systemctl enable containerd.service

k8s

安装

https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/x86_64/

vim /etc/yum.repos.d/k8s.repo

[kubernetes]
name=kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=0

在线安装

yum install kubelet-1.30.0 kubectl-1.30.0 kubeadm-1.30.0

离线安装

yum localinstall kubernetes-cni-1.4.0-150500.1.1.x86_64.rpm 
yum localinstall cri-tools-1.30.0-150500.1.1.x86_64.rpm
yum localinstall kubeadm-1.30.0-150500.1.1.x86_64.rpm kubectl-1.30.0-150500.1.1.x86_64.rpm kubelet-1.30.0-150500.1.1.x86_64.rpm

配置kubelet

vim /etc/sysconfig/kubelet

KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

systemctl enable kubelet

下载镜像

kubeadm config images list --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123kubeadm config images pull --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123

calico准备

wget https://docs.projectcalico.org/v3.25/manifests/calico.yaml --no-check-certificate

vim calico.yaml

            - name: CALICO_IPV4POOL_CIDRvalue: "10.224.0.0/16"

下载calico镜像

ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/cni:v3.25.0ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/node:v3.25.0ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/kube-controllers:v3.25.0

k8s init

生成初始化配置文件

kubeadm config print init-defaults > kubeadm-config.yaml

vim kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 24h0m0susages:- signing- authentication
kind: InitConfiguration
localAPIEndpoint:advertiseAddress: 192.168.0.11bindPort: 6443
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresenttaints: null
---
apiServer:timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.0.10:6443
controllerManager: {}
dns: {}
etcd:local:dataDir: /var/lib/etcd
imageRepository: swr.cn-jl-1.manageone.cloud.cnpc/gsms-project
kind: ClusterConfiguration
kubernetesVersion: 1.30.0
networking:dnsDomain: cluster.localpodSubnet: 10.224.0.0/16serviceSubnet: 10.96.0.0/12
scheduler: {}

初始化集群

kubeadm init --config kubeadm-config.yaml --upload-certs

初始化成功显示以下内容

Your Kubernetes control-plane has initialized successfully!To start using your cluster, you need to run the following as a regular user:mkdir -p $HOME/.kubesudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/configsudo chown $(id -u):$(id -g) $HOME/.kube/configAlternatively, if you are the root user, you can run:export KUBECONFIG=/etc/kubernetes/admin.confYou should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:https://kubernetes.io/docs/concepts/cluster-administration/addons/You can now join any number of the control-plane node running the following command on each as root:kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \--control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.Then you can join any number of worker nodes by running the following on each as root:kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464

配置kubectl客户端

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

设置kubectl命令补全

echo "source <(kubectl completion bash)" >> /etc/profile

部署calico

kubectl apply -f calico.yaml

添加master节点

  kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \--control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826

添加node节点

kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464

ingress-nginx v1.11.2

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.2/deploy/static/provider/cloud/deploy.yaml

metrics-server

wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml