离线安装 K3S

一、前言

简要记录一下离线环境下 K3S 的搭建，版本为 v1.23.17+k3s1，使用外部数据库 MySQL
作元数据存储，禁用默认组件（coredns、servicelb、traefik、local-storage、metrics-server）并使用 Helm
单独安装（coredns、metrics-server、traefik、longhorn）。

需要一台联网主机（虚拟机），和多台未联网主机（服务器）。

二、联网虚拟机

2.1 快速引导一个单节点集群

curl -fsSL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 bash -s - server \--data-dir /data/k3s/var/lib/rancher/k3s \--cluster-cidr 10.8.0.0/16 \--service-cidr 10.16.0.0/16 \--cluster-dns 10.16.0.10 \--service-node-port-range 1-65535 \--kube-proxy-arg proxy-mode=ipvs \--disable coredns \--disable servicelb \--disable traefik \--disable local-storage \--disable metrics-server

2.2 快速安装 Longhorn 的依赖

Installation Requirements

# yum
yum install iscsi-initiator-utils nfs-utils# ubuntu
apt install open-iscsi nfs-common# 启动
systemctl enable iscsid --now

2.3 快速安装应用（通过 Helm Controller）

### coredns
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: corednsnamespace: kube-systemlabels:app: coredns
spec:repo: https://coredns.github.io/helmchart: corednstargetNamespace: kube-systembootstrap: truevaluesContent: |-fullnameOverride: corednsserviceType: ClusterIPservice:clusterIP: 10.16.0.10name: corednsservers:- zones:- zone: .port: 53plugins:- name: errors- name: healthconfigBlock: |-lameduck 5s- name: ready- name: kubernetesparameters: cluster.local in-addr.arpa ip6.arpaconfigBlock: |-pods insecurefallthrough in-addr.arpa ip6.arpattl 30- name: prometheusparameters: 0.0.0.0:9153- name: forwardparameters: . /etc/resolv.conf- name: cacheparameters: 30- name: loop- name: reload- name: loadbalance###  metrics-server
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: metrics-servernamespace: kube-systemlabels:app: metrics-server
spec:repo: https://charts.bitnami.com/bitnamichart: metrics-servertargetNamespace: kube-systembootstrap: truevaluesContent: |apiService:create: trueextraArgs:- --kubelet-insecure-tls- --kubelet-use-node-status-port- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname- --metric-resolution=15s### traefik
---
apiVersion: v1
kind: Namespace
metadata:name: traefik-system---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: traefiknamespace: traefik-systemlabels:app: traefik
spec:repo: https://traefik.github.io/chartschart: traefiktargetNamespace: traefik-systembootstrap: truevaluesContent: |-deployment:kind: DeploymentingressClass:enabled: trueisDefaultClass: trueproviders:kubernetesCRD:enabled: trueallowCrossNamespace: trueallowExternalNameServices: trueallowEmptyServices: truekubernetesIngress:enabled: trueallowExternalNameServices: trueallowEmptyServices: truepublishedService:enabled: trueports:traefik:port: 9000protocol: TCPexpose: falseexposedPort: 9000metrics:port: 9100protocol: TCPexpose: falseexposedPort: 9100web:port: 80protocol: TCPexpose: trueexposedPort: 80nodePort: 30080websecure:port: 443protocol: TCPexpose: trueexposedPort: 443ndoePort: 30443tls:enabled: trueservice:type: NodePortsecurityContext:capabilities:drop: []add: [ALL]readOnlyRootFilesystem: falsepodSecurityContext:runAsGroup: 0runAsNonRoot: falserunAsUser: 0### longhorn
---
apiVersion: v1
kind: Namespace
metadata:name: longhorn-system---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: longhornnamespace: longhorn-systemlabels:app: longhorn
spec:repo: https://charts.longhorn.iochart: longhorntargetNamespace: longhorn-systembootstrap: truevaluesContent: |-persistence:defaultClassReplicaCount: 1csi:attacherReplicaCount: 1provisionerReplicaCount: 1resizerReplicaCount: 1snapshotterReplicaCount: 1defaultSettings:defaultDataPath: /data/longhorndefaultReplicaCount: 1deletingConfirmationFlag: truelonghornUI:replicas: 1longhornConversionWebhook:replicas: 1longhornAdmissionWebhook:replicas: 1longhornRecoveryBackend:replicas: 1ingress:enabled: truehost: longhorn.example.org

kubectl apply -f charts.yaml

三、资源准备

3.1 下载 Longhorn 依赖

1. 查看服务器 glibc 版本

   ldd --version
   

   os	os version	glibc version
   centos	7.9	2.17
   centos	8.4	2.28
   ubuntu	18.04	2.27
   ubuntu	20.04	2.31
   ubuntu	22.04	2.35

2. 创建对应 glibc 版本的容器

   # centos 7
   kubectl run centos --image=centos:7.9.2009 --command -- /bin/sleep infinity
   kubectl exec -it pod/centos -- /bin/bash# ubuntu 22
   kubectl run ubuntu --image=ubuntu:22.04 --command -- /bin/sleep infinity
   kubectl exec -it pod/ubuntu -- /bin/bash
   

3. 下载依赖

   # yum
   yum install iscsi-initiator-utils nfs-utils --downloadonly --downloaddir=rpm -y
   tar -czvf ./rpm.tar.gz ./rpm# apt
   apt update && apt install open-iscsi nfs-common --download-only -y && mkdir -p deb && cp /var/cache/apt/archives/*.deb deb
   tar -czvf ./deb.tar.gz ./deb
   

4. 复制出依赖

   # yum
   kubectl cp centos:/rpm.tar.gz ./rpm.tar.gz# apt
   kubectl cp ubuntu:/deb.tar.gz ./deb.tar.gz
   

3.2 下载 K3S 资源

参考文档：离线安装

wget https://github.com/k3s-io/k3s/releases/download/v1.23.17+k3s1/k3s-airgap-images-amd64.tar.gz
wget https://github.com/k3s-io/k3s/releases/download/v1.23.17+k3s1/k3s
wget https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz
wget https://get.k3s.io -O install.sh

3.3 下载 HelmChart 和导出镜像

# 下载 helm chart 包
helm repo add coredns  https://coredns.github.io/helm     && helm pull coredns/coredns        --version 1.26.0
helm repo add bitnami  https://charts.bitnami.com/bitnami && helm pull bitnami/metrics-server --version 6.5.2
helm repo add traefik  https://traefik.github.io/charts   && helm pull traefik/traefik        --version 24.0.0
helm repo add longhorn https://charts.longhorn.io         && helm pull longhorn/longhorn      --version 1.5.1# 导出镜像
k3s ctr image ls -q | grep -v 'sha256' | sort -u | xargs k3s ctr image export image.tar

四、未联网服务器

4.1 准备 K3S 资源

# 准备 k3s 镜像
mkdir -p /data/k3s/var/lib/rancher/k3s/agent/images
cp ./k3s-airgap-images-amd64.tar.gz /data/k3s/var/lib/rancher/k3s/agent/images# 准备 k3s 二进制可执行文件
install ./k3s /usr/local/bin# 准备 helm 二进制可执行文件
tar -zxvf ./helm-v3.12.2-linux-amd64.tar.gz
install ./linux-amd64/helm /usr/local/bin# 准备 k3s 安装脚本
chmod +x ./install.sh

4.2 引导第一个 Server 节点启动

# 引导 Server
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \--data-dir /data/k3s/var/lib/rancher/k3s \--cluster-cidr 10.8.0.0/16 \--service-cidr 10.16.0.0/16 \--cluster-dns 10.16.0.10 \--service-node-port-range 1-65535 \--kube-proxy-arg proxy-mode=ipvs \--disable coredns \--disable servicelb \--disable traefik \--disable local-storage \--disable metrics-server \--datastore-endpoint="mysql://<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DATABASE>"# 查看 Token
cat /data/k3s/var/lib/rancher/k3s/server/token

4.3 引导其它 Server 节点加入

配置标识在所有 Server 节点必须是相同的。

INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \--data-dir /data/k3s/var/lib/rancher/k3s \--cluster-cidr 10.8.0.0/16 \--service-cidr 10.16.0.0/16 \--cluster-dns 10.16.0.10 \--service-node-port-range 1-65535 \--kube-proxy-arg proxy-mode=ipvs \--disable coredns \--disable servicelb \--disable traefik \--disable local-storage \--disable metrics-server \--datastore-endpoint="mysql://<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DATABASE>" \--token <TOKEN>

4.4 引导 Agent 节点加入

INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \--data-dir /data/k3s/var/lib/rancher/k3s \--datastore-endpoint="mysql://<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DATABASE>" \--token <TOKEN>

4.5 安装 Longhorn 依赖

# yum
tar -zxvf rpm.tar.gz
rpm -ivh ./rpm/*.rpm# apt
tar -zxvf deb.tar.gz
apt install ./deb/*.deb

4.4 导入镜像和安装应用

# 导出镜像
k3s ctr image import ./image.tar# coredns
helm install coredns coredns-1.26.0.tgz --namespace kube-system --values <VALUES_YAML_FILE>