Ansible自动化部署kubernetes集群

机器环境介绍

1.1. 机器信息介绍

IP	hostname	application	CPU	Memory
192.168.204.129	k8s-master01	etcd，kube-apiserver，kube-controller-manager，kube-scheduler,kubelet,kube-proxy,containerd	2C	4G
192.168.204.130	k8s-worker01	etcd，kubelet，kube-proxy，containerd	2C	4G
192.168.204.131	k8s-worker02	etcd，kubelet，kube-proxy，containerd	2C	4G

1.2. 规划IP地址介绍

在Kubernetes中CNI网络插件采用Calico,划分三个网段

网段信息	配置
Pod网段	172.16.0.0/16
Service网段	10.96.0.0/16

安装的kubernetets版本为1.28.5，Calico版本为3.26.4，容器运行环境为containerd

如果需要其他版本kuberneres，需要修改下面的脚本

• 修改kubernetes源里面的版本
• 修改安装master和worker节点里面定义的版本变量值

如下需要使用其他版本的CNI插件或者不同版本的calico插件，需要对网络插件部分脚本进行修改

安装配置ansible

2.1. ansible软件部署

• 安装ansible软件

apt update && apt install ansible -y

• 配置ansible配置

mkdir /etc/ansible/ && touch /etc/ansible/hosts

• 配置/etc/ansible/hosts文件

[master]
192.168.204.129[worker]
192.168.204.130
192.168.204.131

• 配置免密登录, 此过程中不要输入密码

ssh-keygen -t rsa

• 分发免密登录

ssh-copy-id root@192.168.204.129
ssh-copy-id root@192.168.204.130
ssh-copy-id root@192.168.204.131

• 配置hosts

cat >> /etc/hosts <<EOF

2.2. 测试ansible连接性

• 编写测试脚本

cat >test_nodes.yml <<EOF
---
- name: test nodeshosts: masterworkertasks:- name: Ping nodesping:
EOF

• 执行ansible测试
• 
  ansible-playbook test_node.yml

配置kubernetes脚本

3.1. 编写的kubernetes 脚本

• 编写的install-kubernetes.yml文件内容如下

---
- name: Performance Basic Confighosts: masterworkerbecome: yestasks:- name: Check if fstab contains swapshell: grep -q "swap" /etc/fstabregister: fstab_contains_swap- name: Temp Disable swapcommand: swapoff -awhen: fstab_contains_swap.rc == 0- name: Permanent Disable swapshell: sed -i 's/.*swap.*/#&/g' /etc/fstabwhen: fstab_contains_swap.rc == 0- name: Disable Swap unit-filesshell: |swap_units=$(systemctl list-unit-files | grep swap | awk '{print $1}')for unit in $swap_units; dosystemctl mask $unitdone- name: Stop UFW serviceservice:name: ufwstate: stopped- name: Disable UFW at bootservice:name: ufwenabled: no- name: Set timezoneshell: TZ='Asia/Shanghai'; export TZ- name: Set timezone permanentlyshell: |cat >> /etc/profile << EOFTZ='Asia/Shanghai'; export TZEOF- name: Create .hushlogin file in $HOMEfile:path: "{{ ansible_env.HOME }}/.hushlogin"state: touch- name: Install required packagesapt:name: "{{ packages }}"state: presentvars:packages:- apt-transport-https- ca-certificates- curl- gnupg- lsb-release- name: Add Aliyun Docker GPG keyshell: curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add- name: Add Aliyun Docker repositoryshell: echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker-ce.list- name: Add Aliyun Kubernetes GPG keyshell: curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg- name: Add Aliyun Kubernetes repositoryshell: echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list- name: Set apt sources to use USTC mirrorsshell: sed -i 's#cn.archive.ubuntu.com#mirrors.aliyun.com#g' /etc/apt/sources.list- name: Update apt cacheapt:update_cache: yes- name: Load br_netfilter on startshell: echo "modprobe br_netfilter" >> /etc/profile- name: Load br_netfiltershell: modprobe br_netfilter- name: Update sysctl settingssysctl:name: "{{ item.name }}"value: "{{ item.value }}"state: presentreload: yeswith_items:- { name: "net.bridge.bridge-nf-call-iptables", value: "1" }- { name: "net.bridge.bridge-nf-call-ip6tables", value: "1" }- { name: "net.ipv4.ip_forward", value: "1" }- name: Install IPVSapt:name: "{{ packages }}"state: presentvars:packages:- ipset- ipvsadm- name: Create ipvs modulesfile:name: /etc/modules-load.d/ipvs.modulesmode: 0755state: touch- name: Write ipvs.modules filelineinfile:dest: /etc/modules-load.d/ipvs.modulesline: "#!/bin/bash\nmodprobe -- ip_vs\nmodprobe -- ip_vs_rr\nmodprobe -- ip_vs_wrr\nmodprobe -- ip_vs_sh\nmodprobe -- nf_conntrack\nmodprobe -- overlay\nmodprobe -- br_netfilter"- name: Execute ipvs.modules scriptshell: sh /etc/modules-load.d/ipvs.modules- name: Install Containerdapt:name: "{{ packages }}"state: presentvars:packages:- containerd.io- name: Generate default containerd fileshell: containerd config default > /etc/containerd/config.toml- name: Config sandbox imageshell: sed -i 's#sandbox_image = "registry.k8s.io/pause:3.6"#sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"#g' /etc/containerd/config.toml- name: Modify Systemd Cgroupshell: sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml- name: Restart Containerdshell: systemctl restart containerd- name: Systemctl enable containerdshell: systemctl enable containerd- name: Install Kubernetes Masterhosts: masterbecome: yesvars:kubernetes_version: "1.28.5"pod_network_cidr: "172.16.0.0/16"service_cidr: "10.96.0.0/16"image_repository: "registry.aliyuncs.com/google_containers"calico_version: "v3.26.4"tasks:- name: Install Master kubernetes packagesapt:name: "{{ packages }}"state: presentvars:packages:- kubelet={{ kubernetes_version }}-1.1- kubeadm={{ kubernetes_version }}-1.1- kubectl={{ kubernetes_version }}-1.1- name: Initialize Kubernetes Mastercommand: kubeadm init --kubernetes-version={{ kubernetes_version }} --pod-network-cidr={{ pod_network_cidr }} --service-cidr={{ service_cidr }} --image-repository={{ image_repository }}register: kubeadm_outputchanged_when: "'kubeadm join' in kubeadm_output.stdout"- name: Save join commandcopy:content: |{{ kubeadm_output.stdout_lines [-2] }}{{ kubeadm_output.stdout_lines [-1] }}dest: /root/kubeadm_join_master.shwhen: kubeadm_output.changed- name: cope join master scriptshell: sed -i 's/"//g' /root/kubeadm_join_master.sh- name: copy kubernetes configshell: mkdir -p {{ ansible_env.HOME }}/.kube && cp -i /etc/kubernetes/admin.conf {{ ansible_env.HOME }}/.kube/config- name: enable kubectlcommand: systemctl enable kubelet- name: Create calico directoryfile:path: "{{ ansible_env.HOME }}/calico/{{ calico_version }}"state: directory- name: download calico tigera-operator.yamlcommand: wget https://ghproxy.net/https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/tigera-operator.yaml -O {{ ansible_env.HOME }}/calico/{{ calico_version }}/tigera-operator.yaml- name: download calico custom-resources.yamlcommand: wget https://ghproxy.net/https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/custom-resources.yaml -O {{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml- name: set calico netwok rangereplace:path: "{{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml"regexp: "blockSize: 26"replace: "blockSize: 24"- name: set calico ip poolsreplace:path: "{{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml"regexp: "cidr: 192.168.0.0/16"replace: "cidr: {{ pod_network_cidr }}"- name: apply calico tigera-operator.yamlcommand: kubectl create -f {{ ansible_env.HOME }}/calico/{{ calico_version }}/tigera-operator.yaml- name: apply calico custom-resources.yamlcommand: kubectl create -f {{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml- name: set crictl configcommand: crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock- name: Install Kubernetes workerhosts: workerbecome: yesvars:kubernetes_version: "1.28.5"tasks:- name: Install worker kubernetes packagesapt:name: "{{ packages }}"state: presentvars:packages:- kubelet={{ kubernetes_version }}-1.1- kubeadm={{ kubernetes_version }}-1.1- name: copy kubeadm join script to workerscopy:src: /root/kubeadm_join_master.shdest: /root/kubeadm_join_master.shmode: 0755- name: worker join to clustercommand: sh /root/kubeadm_join_master.sh- name: set crictl configcommand: crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock- name: enable kubectlcommand: systemctl enable kubelet

执行kubernetes脚本 

 ansible-playbook install-kubernetes.yml

• 集群状态

kubectl get node -o wide

• 集群pod状态

kubectl get pod -A