ubuntu k8s 1.31

ubuntu 系统 设置

更新源

apt-get upgradeapt upgradeapt update
apt-get update

释放root

sudo passwd root密码su -
密码

设置root可以登录

cd /etc/ssh/sshd_config.d && vi ssh.conf

PermitRootLogin yes
PasswordAuthentication yes

:wq 保存退出

systemctl restart ssh

系统初始化设置

关闭防火墙

ufw disablesystemctl disable --now ufw

修改 hostname

# 修改 hostname 
hostnamectl set-hostname k8s-01
# 查看修改结果
hostnamectl status
# 设置 hostname 解析
echo "127.0.0.1   $(hostname)" >> /etc/hosts

如果是多个集群，设置多次

echo "192.168.1.64 k8s-01" >> /etc/hosts
echo "192.168.1.65 k8s-02" >> /etc/hosts
echo "192.168.1.66 k8s-03" >> /etc/hosts

关闭 swap

# 禁用交换分区(在旧版的 k8s 中 kubelet 都要求关闭 swapoff ，但最新版的 kubelet 其实已经支持 swap ，因此这一步其实可以不做。)
swapoff -a
# 永久禁用，打开/etc/fstab注释掉swap那一行。  
sed -ri 's/.*swap.*/#&/' /etc/fstab

关闭 selinux

有selinux 可以关闭一下

修改limit

ulimit -SHn 65535

vi /etc/security/limits.conf

末尾添加如下内容

* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited

:wq 保存退出

时区设置

timedatectl set-timezone Asia/Shanghaisystemctl restart systemd-timesyncd.service

检查设置

timedatectl status
# 或者
date

开启流量转发

设置所需的 sysctl 参数，参数在重新启动后保持不变

修改内核参数(首先确认你的系统已经加载了 br_netfilter 模块，默认是没有该模块的，需要你先安装 bridge-utils)
apt-get install -y bridge-utils
modprobe br_netfilter
lsmod | grep br_netfilter
如果报错找不到包，需要先更新 apt-get update -y

cat > /etc/modules-load.d/k8s.conf <<EOF
overlay
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

应用 sysctl 参数而不重新启动

sysctl --systemsysctl net.ipv4.ip_forward

lsmod | grep br_netfilter
lsmod | grep overlay

ipvs设置

安装ipvs

apt install -y  ipset ipvsadm 

cat > /etc/modules-load.d/ipvs.conf << EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

sudo modprobe ip_vslsmod | grep ip_vslsmod | grep -e ip_vs -e nf_conntrack

安装容器运行时

安装基础环境

apt -y install apt-transport-https ca-certificates curl software-properties-commoncurl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg

安装 containerd

https://www.ghproxy.cn/https://github.com/containerd/containerd/releases/download/v1.7.24/containerd-1.7.24-linux-amd64.tar.gztar Cxzvf /usr/local containerd-1.7.24-linux-amd64.tar.gzhttps://gh.llkk.cc/https://github.com/containerd/containerd/releases/download/v1.7.24/cri-containerd-1.7.24-linux-amd64.tar.gzsudo tar -C / -xzf cri-containerd-1.7.24-linux-amd64.tar.gz
export PATH=$PATH:/usr/local/bin:/usr/local/sbin
source ~/.bashrc
mkdir /etc/containerd

配置 containerd

#1.重新初始化containerd的配置文件
containerd config default | tee /etc/containerd/config.toml 

修改SystemdCgroup

grep SystemdCgroup /etc/containerd/config.toml#2 修改Cgroup的管理者为systemd组件
sed -ri 's#(SystemdCgroup = )false#\1true#' /etc/containerd/config.toml grep SystemdCgroup /etc/containerd/config.toml

修改修改pause的基础镜像版本

grep sandbox_image /etc/containerd/config.toml#3 修改pause的基础镜像名称
sed -i 's#registry.k8s.io/pause:3.8#registry.aliyuncs.com/google_containers/pause:3.10#' /etc/containerd/config.tomlgrep sandbox_image /etc/containerd/config.toml

启动containerd

systemctl daemon-reload
systemctl enable --now containerd
systemctl status containerd
systemctl restart containerd

安装kubelet kubeadm kubectl

添加阿里云k8s源

sudo apt-get updatesudo apt-get install -y apt-transport-https ca-certificates curl gpgcurl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/deb/Release.key |gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpgecho "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.listapt-get update#查看kubectl可用版本
apt-cache madison kubectl

高可用集群【可选】

## 安装keepalived haproxy
apt install -y keepalived haproxy

配置keepalived配置文件

配置keepalived配置文件

 vim /etc/keepalived/keepalived.conf

示例配置文件

MASTER

! /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {router_id LVS_DEVEL
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh"interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {# 状态，主节点为MASTER，从节点为BACKUPstate MASTER# 修改为你自己网卡的名字interface eth0virtual_router_id 51# MASTER当中使用101，BACKUP当中使用100priority 101authentication {auth_type PASS# 设置好你的密码，keepalived集群当中需要保证这个值的一致auth_pass k8s_test}virtual_ipaddress {# 设置你的虚拟IP地址192.168.1.67/24}track_script {check_apiserver}
}

BACKUP

! /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {router_id LVS_DEVEL
}
vrrp_script check_apiserver {script "/etc/keepalived/check_apiserver.sh"interval 3weight -2fall 10rise 2
}vrrp_instance VI_1 {# 状态，主节点为MASTER，从节点为BACKUPstate MASTER# 修改为你自己网卡的名字interface eth0virtual_router_id 51# MASTER当中使用101，BACKUP当中使用100priority 100authentication {auth_type PASS# 设置好你的密码，keepalived集群当中需要保证这个值的一致auth_pass k8s_test}virtual_ipaddress {# 设置你的虚拟IP地址192.168.1.67/24}track_script {check_apiserver}
}

配置keepalived配置文件 脚本

编辑keepalived配置文件 脚本

vim /etc/keepalived/check_apiserver.sh

示例配置文件

#!/bin/sherrorExit() {echo "*** $*" 1>&2exit 1
}curl -sfk --max-time 2 https://localhost:7443/healthz -o /dev/null || errorExit "Error GET https://localhost:7443/healthz"

给脚本执行权限

 chmod +x /etc/keepalived/check_apiserver.sh

编辑haproxy配置文件

cp    /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg_bakvim    /etc/haproxy/haproxy.cfg

haproxy示例文件

# /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
globallog /dev/log local0log /dev/log local1 noticedaemon#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaultsmode                    httplog                     globaloption                  httplogoption                  dontlognulloption http-server-closeoption forwardfor       except 127.0.0.0/24option                  redispatchretries                 30timeout http-request    10stimeout queue           20stimeout connect         5stimeout client          35stimeout server          35stimeout http-keep-alive 10stimeout check           10s#---------------------------------------------------------------------
# apiserver frontend which proxys to the control plane nodes
#---------------------------------------------------------------------
frontend apiserver# 注意负载均衡的端口要与keepalived里面的配置保持一致bind *:7443mode tcpoption tcplogdefault_backend apiserver#---------------------------------------------------------------------
# round robin balancing for apiserver
#---------------------------------------------------------------------
backend apiserveroption httpchk GET /healthzhttp-check expect status 200mode tcpoption ssl-hello-chkbalance     roundrobinserver k8s-master-01 192.168.1.64:6443 checkserver k8s-master-02 192.168.1.65:6443 checkserver k8s-master-03 192.168.1.66:6443 check

启动keepalived haproxy

 systemctl enable haproxy --now
systemctl enable keepalived --nowsystemctl start haproxy
systemctl start keepalived systemctl status haproxy
systemctl status keepalived 

如果配置文件更改 ，快速重启keepalived,haproxy

systemctl restart keepalived systemctl restart haproxy

安装kubeadm kubectl kubelet

安装kubeadm kubectl kubelet

sudo apt update && \
sudo apt install -y kubelet kubectl kubeadm && \
sudo apt-mark hold kubelet kubeadm kubectl

查看kubeadm版本

kubeadm version

开机启动kubelet

systemctl enable kubelet && systemctl restart kubelet

初始化k8s

输出配置文件

kubeadm config print init-defaults  > kubeadm-config.yaml

修改配置文件

apiVersion: kubeadm.k8s.io/v1beta4
bootstrapTokens:
- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 24h0m0susages:- signing- authentication
kind: InitConfiguration
localAPIEndpoint:# apisServer地址修改advertiseAddress: 192.168.1.81# 端口不变bindPort: 6443
nodeRegistration:criSocket: unix:///var/run/containerd/containerd.sockimagePullPolicy: IfNotPresentimagePullSerial: true#节点名称修改name: k8s-01taints: null
timeouts:controlPlaneComponentHealthCheck: 4m0sdiscovery: 5m0setcdAPICall: 2m0skubeletHealthCheck: 4m0skubernetesAPICall: 1m0stlsBootstrap: 5m0supgradeManifests: 5m0s
---
apiServer: {}
apiVersion: kubeadm.k8s.io/v1beta4
##ca证书和颁发证书时间修改
caCertificateValidityPeriod: 876000h0m0s
certificateValidityPeriod: 876000h0m0s
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
# 新增控制平台地址 单节点不需要这个 controlPlaneEndpoint
controlPlaneEndpoint: "192.168.1.67:7443"
controllerManager: {}
dns: {}
encryptionAlgorithm: RSA-2048
etcd:local:dataDir: /var/lib/etcd
#修改镜像拉取地址
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.32.0
networking:dnsDomain: cluster.local# 设置service网络地址范围serviceSubnet: 10.96.0.0/12# 设置pod网络地址范围podSubnet: 10.60.0.0/16 
proxy: {}
scheduler: {}
---
#https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
#开启ipvs模式
mode: ipvs 

拉取镜像 初始化集群

#拉取镜像
kubeadm config images pull --config kubeadm-config.yaml#尝试运行 如果无错误，执行下面的初始化
kubeadm init --config kubeadm-config.yaml --dry-run#初始化k8s集群 单节点不需要--upload-certs
kubeadm init --config kubeadm-config.yaml --upload-certs

初始化成功执行

  mkdir -p $HOME/.kubesudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/configsudo chown $(id -u):$(id -g) $HOME/.kube/configexport KUBECONFIG=/etc/kubernetes/admin.conf

其他master节点加入集群

 kubeadm join 192.168.1.67:7443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:006f34ef31937c42d822f3a635368d4861ff5bd8211538d629e0e3c282461349 \--control-plane --certificate-key 62eeb97fff1258c0f3eb79e46d68050ee6da66f6256eb74c5e41e98aa55aa6bb

work节点加入集群

kubeadm join 192.168.146.200:6443 --token abcdef.0123456789abcdef \--discovery-token-ca-cert-hash sha256:9d783c63bf9a20ce9634788b96905f369668d9f439e444e271907561455b8779 

安装网络插件

curl https://raw.githubusercontent.com/projectcalico/calico/v3.29.1/manifests/calico-typha.yaml -o calico.yamlwget https://github.com/projectcalico/calico/releases/download/v3.29.1/release-v3.29.1.tgztar -xvf release-v3.29.1.tgz
cd release-v3.29.1/images
ctr -n k8s.io images import calico-cni.tar 
ctr -n k8s.io images import calico-flannel-migration-controller.tar 
ctr -n k8s.io images import calico-node.tar 
ctr -n k8s.io images import calico-typha.tar 
ctr -n k8s.io images import calico-dikastes.tar 
ctr -n k8s.io images import calico-kube-controllers.tar 
ctr -n k8s.io images import calico-pod2daemon.tar #修改其中的 pod网络范围  CALICO_IPV4POOL_CIDR  默认值应该是 192.168.0.0/16kubectl apply -f  calico-typha.yaml

初始化失败处理

重置 kubeadm ，执行这个后需要敲 y 回车

kubeadm reset -f
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X

删除上次 init 生成的文件

sudo rm -rf /etc/cni/net.d 
sudo rm -rf /var/lib/etcd 
ipvsadm --clear
rm -rf $HOME/.kube/config
rm -rf /etc/kubernetes

证书续期

证书有效期查看

kubeadm  certs check-expiration   

证书续期

# 备份证书
cp -r /etc/kubernetes /etc/kubernetes.bak
# 重新生成证书
kubeadm certs renew all# 删除服务让证书生效
# kubectl delete pod etcd-master -n kube-system
# kubectl delete pod kube-apiserver-master -n kube-system
# kubectl delete pod kube-controller-manager-master -n kube-system
# kubectl delete pod kube-scheduler-master -n kube-system# 删除服务让证书生效
crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | xargs crictl rmp -f cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

去掉污点

# 查看污点
kubectl describe nodes k8s-uat |grep Taints
# 去掉污点 示例
kubectl taint node k8s-01 node.kubernetes.io/not-ready:NoSchedule-
# 查看污点 示例
kubectl taint node k8s-01 node-role.kubernetes.io/control-plane:NoSchedule-