使用Ubuntu虚拟机离线部署RKE2高可用集群

环境说明

宿主机和虚拟机的OS与内核相同，如下

$ cat /etc/issue
Ubuntu 22.04.3 LTS \n \l$ uname -sr
Linux 6.2.0-34-generic

虚拟化软件版本

$ kvm --version
QEMU emulator version 6.2.0 (Debian 1:6.2+dfsg-2ubuntu6.14)
Copyright (c) 2003-2021 Fabrice Bellard and the QEMU Project developers
$ virt-manager --version
4.0.0

安装前准备

参考《基于ubuntu22.04安装KVM虚拟机》安装5台虚拟机

来自 https://blog.csdn.net/u010438035/article/details/129260034
参考下面修改hosts文件，并配置每台虚拟机的ip和hostname，其中vip 设置为：192.168.122.10

cat /etc/hosts
192.168.122.10 rke2-vip
192.168.122.11 rke2-master-01
192.168.122.12 rke2-master-02
192.168.122.13 rke2-master-03
192.168.122.14 rke2-node-01
192.168.122.15 rke2-node-02

配置5台虚拟机免密ssh登陆

ssh-keygen -t rsa
ssh-copy-id -i .ssh/id_rsa.pub rke2-master-01
ssh-copy-id -i .ssh/id_rsa.pub rke2-master-02
ssh-copy-id -i .ssh/id_rsa.pub rke2-master-03
ssh-copy-id -i .ssh/id_rsa.pub rke2-node-01
ssh-copy-id -i .ssh/id_rsa.pub rke2-node-01

配置5台虚拟机sudo 免输入密码

注意下面的username是使用者的实际名字，需要修改

cat > /etc/sudoers.d/passwordless  << EOF
# add in user AFTER the sudo and admin group
username ALL=(ALL) NOPASSWD: ALL
EOF

在宿主机创建一个串行脚本 allnodes

allnodes只需要放置在宿主机上，使用普通用户执行========
下面的username是使用者的实际名字，需要修改

cat > allnodes  << "EOF"
#!/bin/bash
nodes=(192.168.122.11 192.168.122.12 192.168.122.13 192.168.122.14 192.168.122.15)
for i in ${nodes[*]}; do ssh username@$i "hostname;$1"; done
EOFchmod +x allnodes

配置5台虚拟机基础环境

1.升级Linux内核到最新版
./allnodes 'sudo apt-get -y update'
./allnodes 'sudo apt-get -y dist-upgrade'2.设置时间同步
kubernetes要求集群中的节点时间必须精确一致，您可以同时运行一下date命令，检查一下几台机器的时间是否正常。如果正常，则可以跳过此步。
# 借助于chronyd服务（程序包名称chrony）设定各节点时间精确同步
./allnodes "sudo apt-get install -y chrony"
./allnodes "sudo chronyc sources -v"# 设置成东八区时区
./allnodes "sudo timedatectl set-timezone Asia/Shanghai"3.禁用swap分区
./allnodes "sudo swapoff -a"
./allnodes "sudo sed -ri 's/.*swap.*/#&/' /etc/fstab"4.配置linux的内核参数
# 配置加载模块
./allnodes 'sudo sh -c "cat > /etc/modules-load.d/k8s.conf <<EOF
overlay
br_netfilter
EOF"'
# 加载 br_netfilter 模块
./allnodes 'sudo modprobe overlay && sudo modprobe br_netfilter'
# 添加网桥过滤及地址转发
./allnodes 'sudo sh -c "cat > /etc/sysctl.d/k8s.conf  <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF"'./allnodes 'sudo sysctl --system'# 检查是否加载
./allnodes 'sudo lsmod | grep br_netfilter'
./allnodes 'sudo lsmod | grep overlay'5.配置ipvs功能
./allnodes 'sudo apt install -y ipset ipvsadm'./allnodes 'sudo sh -c "cat > /etc/modules-load.d/ipvs.conf << EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF"'# 临时加载
./allnodes 'sudo modprobe -- ip_vs && sudo modprobe -- ip_vs_rr && sudo modprobe -- ip_vs_wrr && sudo modprobe -- ip_vs_sh && sudo modprobe -- nf_conntrack'# 开机加载配置，添加需要加载的模块
./allnodes 'sudo sh -c "cat >> /etc/modules <<EOF
ip_vs_sh
ip_vs_wrr
ip_vs_rr
ip_vs
nf_conntrack
EOF"'# 检查是否加载
./allnodes 'sudo lsmod | grep -e -ip_vs -e nf_conntrack'6.重启OS
上面步骤完成之后，需要重新启动linux系统： 
./allnodes 'sudo reboot'

在3台master节点配置 haproxy 与 keepalived

串行脚本masters相比allnodes去掉了node节点的ip

1) 配置 Haproxy=====

./masters 'sudo sh -c "cat >> /etc/haproxy/haproxy.cfg << EOF
#---------------------------------------------------------------------
# kubernetes apiserver frontend which proxys to the backends
#---------------------------------------------------------------------
frontend rke2-apiservermode                 tcpbind                 *:19345  # 监听端口option               tcplogdefault_backend      rke2-apiserver
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend rke2-apiservermode        tcp        # 模式 TCPoption      tcplogoption      tcp-checkbalance     roundrobin # 采用轮询的负载算法  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100server      rke2-master-01   192.168.122.11:9345 check server      rke2-master-02   192.168.122.12:9345 checkserver      rke2-master-03   192.168.122.13:9345 check
#---------------------------------------------------------------------
# collection haproxy statistics message
#---------------------------------------------------------------------
listen statsbind                 *:1080stats auth           admin:awesomePasswordstats refresh        5sstats realm          HAProxy\ Statisticsstats uri            /admin?stats
EOF"'	

=2) keepalived 配置=

./masters 'sudo sh -c "cat >> /etc/keepalived/keepalived.conf <<EOF
global_defs {script_user rootenable_script_security
}
vrrp_script chk_apiserver {script "/etc/keepalived/check_apiserver.sh" # 定义脚本路径和名称interval 5 # 每 5 秒执行一次检测，注意设置间隔太小会有问题weight -15 # 权重变化fall 2     # 检测连续2次失败才算失败rise 1     # 检测1次成功就算成功
}
vrrp_instance VI_1 {state BACKUP # backup节点设为BACKUP, <看情况调整>interface enp1s0  # 服务器网卡接口virtual_router_id 51  # 这个值只要在 keepalived 集群中保持一致即可，默认值是 51priority 50 #如：master设为 100，备份服务 50，比备份服务器上高就行了，如：master设为 100，备份服务 50advert_int 1     authentication {auth_type PASSauth_pass K8SHA_KA_AUTH  #这个值只要在keepalived集群中保持一致即可}virtual_ipaddress {192.168.122.10   # VIP 地址，<看情况调整>}track_script {chk_apiserver}
}
EOF"'

上述文件需要根据实际情况再修改
state {MASTER|BACKUP}# 主节点值为 MASTER，备节点值为 BACKUP，这里配置 rke2-master-01 为 MASTER，rke2-master-02、rke2-master-03 为 BACKUP
设置 priority 主节点的优先级大于备节点，如: 主节点rke2-master-01 设为 100，备份节点rke2-master-02、rke2-master-03设为 50

./masters 'sudo sh -c "cat >> /etc/keepalived/check_apiserver.sh <<EOF
#!/bin/bash 
err=0
for k in $(seq 1 3)
docheck_code=$(pgrep haproxy)if [[ $check_code == "" ]]; thenerr=$(expr $err + 1)sleep 1continueelseerr=0breakfi
doneif [[ $err != "0" ]]; thenecho "systemctl stop keepalived"/usr/bin/systemctl stop keepalivedexit 1
elseexit 0
fi
EOF"'

=配置开机启动========

./masters 'sudo systemctl enable haproxy '
./masters 'sudo systemctl restart haproxy'
./masters 'sudo systemctl enable keepalived '
./masters 'sudo systemctl restart keepalived'

离线部署rke2

下载如下文件并复制到5台虚拟机
curl -OLs https://github.com/rancher/rke2/releases/download/v1.28.2%2Brke2r1/rke2-images.linux-amd64.tar.zst
curl -OLs https://github.com/rancher/rke2/releases/download/v1.28.2%2Brke2r1/rke2.linux-amd64.tar.gz
curl -OLs https://github.com/rancher/rke2/releases/download/v1.28.2%2Brke2r1/sha256sum-amd64.txt
curl -sfL https://rancher-mirror.rancher.cn/rke2/install.sh --output install.sh
在虚拟机中给install.sh增加可执行权限

安装第一个master节点

这里假设上传安装包到 /root/rke2-artifacts/ 目录了

INSTALL_RKE2_VERSION=v1.28.2+rke2r1 INSTALL_RKE2_ARTIFACT_PATH=/root/rke2-artifacts sh /root/install.sh
mkdir -p /etc/rancher/rke2
cat >> /etc/rancher/rke2/config.yaml << EOF
#server: "https://192.168.122.10:19345" # 等三台都起来后把这个配置取消注释，重启服务
write-kubeconfig: "/root/.kube/config"
write-kubeconfig-mode: 0644
## 自定义一个 token 标识
token: "RKE2@Cluster"
## tls-san 填写LB的统一入口ip地址或域名
tls-san:- "192.168.122.10"## 打上污点，不让用户工作负载调度到该节点上
node-taint:- "CriticalAddonsOnly=true:NoExecute"
kube-proxy-arg: # 不指定的话，默认是 iptables 模式- "proxy-mode=ipvs"
EOF
systemctl enable rke2-server && sudo systemctl start rke2-server# 验证
ln -s /var/lib/rancher/rke2/agent/etc/crictl.yaml /etc/crictl.yaml
ln -s /var/lib/rancher/rke2/bin/kubectl /usr/bin/kubectl
ln -s /var/lib/rancher/rke2/bin/crictl /usr/bin/crictlkubectl get node
crictl ps
crictl images

将其他 master 节点加入集群

与第一个master节点不同的是/etc/rancher/rke2/config.yaml中取消了server参数注释，第一个master节点的注释要等所有其他master节点起来后才取消注释

INSTALL_RKE2_VERSION=v1.28.2+rke2r1 INSTALL_RKE2_ARTIFACT_PATH=/root/rke2-artifacts sh /root/install.sh
mkdir -p /etc/rancher/rke2
cat >> /etc/rancher/rke2/config.yaml << EOF
server: "https://192.168.122.10:19345"
write-kubeconfig: "/root/.kube/config"
write-kubeconfig-mode: 0644
## 自定义一个 token 标识
token: "RKE2@Cluster"
## tls-san 填写LB的统一入口ip地址或域名
tls-san:- "192.168.122.10"## 打上污点，不让用户工作负载调度到该节点上
node-taint:- "CriticalAddonsOnly=true:NoExecute"
kube-proxy-arg: # 不指定的话，默认是 iptables 模式- "proxy-mode=ipvs"
EOF
systemctl enable rke2-server && sudo systemctl start rke2-server# 验证
ln -s /var/lib/rancher/rke2/agent/etc/crictl.yaml /etc/crictl.yaml
ln -s /var/lib/rancher/rke2/bin/kubectl /usr/bin/kubectl
ln -s /var/lib/rancher/rke2/bin/crictl /usr/bin/crictlkubectl get node
crictl ps
crictl images

将 node 节点加入集群

INSTALL_RKE2_TYPE="agent" INSTALL_RKE2_VERSION=v1.28.2+rke2r1 INSTALL_RKE2_ARTIFACT_PATH=/root/rke2-artifacts sh /root/install.sh
mkdir -p /etc/rancher/rke2
cat > /etc/rancher/rke2/config.yaml << EOF
server: "https://192.168.122.10:19345"
token: "RKE2@Cluster"kube-proxy-arg:- "proxy-mode=ipvs"
EOFsystemctl enable rke2-agent && sudo systemctl start rke2-agent

发布一个pod进行验证

# cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:name: busyboxnamespace: default
spec:containers:- name: busyboximage: docker.io/library/busybox:1.28command:- sleep- "3600"imagePullPolicy: IfNotPresentrestartPolicy: Always
EOF# kubectl get pod -A -owide| grep busybox
default       busybox                                                 1/1     Running     0               54s   10.42.9.4        rke2-node-02     <none>           <none>

删除集群节点

## 登录到要删除的节点，执行以下命令
rke2-killall.sh
rke2-uninstall.sh
## 在master节点执行删除操作
kubectl delete node {NODE-NAME}
kubectl -n kube-system delete secrets {NODE-NAME}.node-password.rke2

参考《RKE2 高可用部署》
来自 https://blog.csdn.net/cl18707602767/article/details/132641657