k8s-部署对象存储minio

环境信息

minio版本 :最新

k8s 版本1.22

使用nfs作为共享存储

一.单节点安装包部署

脚本部署，一键部署，单节点应用于数据量小，一些缓存存储，比如gitlab-runner的产物数据，maven的打包依赖数据

#!/bin/bash# 步骤一：创建目录
sudo mkdir -p /opt/minio/{bin,conf,data}# 步骤二：下载服务
cd /opt/minio/bin
sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio
sudo chmod +x minio# 步骤三：创建配置文件
sudo tee /opt/minio/conf/minio.conf > /dev/null <<EOF
MINIO_VOLUMES="/opt/minio/data"
#--address指定S3 API端口，--console-address指定web控制台端口.
MINIO_OPTS="--address :39000 --console-address :39001" 
# Root user for the server.
MINIO_ROOT_USER=admin
# Root secret for the server.
MINIO_ROOT_PASSWORD=minio123
EOF# 步骤四：创建启动命令
sudo tee /etc/systemd/system/minio.service > /dev/null <<EOF
[Unit]
Description=MinIO
Documentation=https://www.minio.org.cn/
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/opt/minio/bin/minio[Service]
User=root
Group=rootEnvironmentFile=/opt/minio/conf/minio.conf
ExecStart=/opt/minio/bin/minio server \$MINIO_OPTS \$MINIO_VOLUMES
Restart=alwaysLimitNOFILE=65536
TimeoutStopSec=infinity
SendSIGKILL=no[Install]
WantedBy=multi-user.target
EOF# 步骤五：启动服务
sudo systemctl daemon-reload
sudo systemctl enable minio.service
sudo systemctl start minio.service
sudo systemctl status minio.service

二.单节点k8s部署

1.minio-deploy.yaml部署

apiVersion: apps/v1
kind: Deployment
metadata:name: minio
spec:replicas: 1selector:matchLabels:app: miniotemplate:metadata:labels:app: miniospec:containers:- name: minioenv:- name: MINIO_ROOT_USERvalue:  'root'- name: MINIO_ROOT_PASSWORDvalue:  'root123456'        image: minio/minio:latestimagePullPolicy: IfNotPresentcommand:- /bin/sh- -c- minio server /data --console-address ":5000"ports:- name: datacontainerPort: 9000protocol: "TCP"- name: consolecontainerPort: 5000protocol: "TCP"volumeMounts:- name: minio-persistent-datamountPath: /datavolumes:- name: minio-persistent-datanfs:server: 192.168.110.190 path: /data/data/minio

执行资源文件

kubectl -f apply  minio-deploy.yaml -n minio

2.minio-service.yaml 创建

apiVersion: v1
kind: Service
metadata:name: minio
spec:type: NodePortports:- name: dataport: 9000targetPort: 9000protocol: TCPnodePort: 30069- name: consoleport: 5000targetPort: 5000protocol: TCPnodePort: 30070selector:app: minio

执行资源文件

kubectl -f apply  minio-service.yaml -n minio

安装好后浏览器访问ui,登录密码root和root123456

http://集群内部任意ip:30070 

三.k8s集群部署

1.创建挂载多磁盘目录

mkdir -p /data/data/minio/pv{1..4}

2.创建sc.yaml

cat > sc.yaml << 'EOF'
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: minio-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
EOF

3.创建pv.yaml

cat > pv.yaml << 'EOF'
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-nfs-pv-1
spec:capacity:storage: 10GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagenfs:server: 192.168.110.190path: /data/data/minio/pv1
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-nfs-pv-2
spec:capacity:storage: 10GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagenfs:server: 192.168.110.190path: /data/data/minio/pv2
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-nfs-pv-3
spec:capacity:storage: 10GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagenfs:server: 192.168.110.190path: /data/data/minio/pv3
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-nfs-pv-4
spec:capacity:storage: 10GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagenfs:server: 192.168.110.190path: /data/data/minio/pv4
EOF

4.创建 SVC

cat > svc.yaml << 'EOF'
apiVersion: v1
kind: Service
metadata:name: minio-hsnamespace: gitlablabels:app: minio
spec:clusterIP: Noneports:- port: 9000name: data- port: 5000name: consoleselector:app: minio
---
apiVersion: v1
kind: Service
metadata:name: minio-scnamespace: gitlablabels:app: minio
spec:type: NodePortports:- name: dataport: 9000targetPort: 9000protocol: TCPnodePort: 30090- name: consoleport: 5000targetPort: 5000protocol: TCPnodePort: 30050selector:app: minio
EOF

5.创建 STS

cat > sts.yaml << 'EOF'
apiVersion: apps/v1
kind: StatefulSet
metadata:name: minionamespace: gitlab
spec:serviceName: "minio-hs"replicas: 4selector:matchLabels:app: miniotemplate:metadata:labels:app: miniospec:containers:- name: minioenv:- name: MINIO_ROOT_USERvalue: "admin"- name: MINIO_ROOT_PASSWORDvalue: "admin12345"image: minio/minio:RELEASE.2023-08-09T23-30-22ZimagePullPolicy: IfNotPresentcommand:- /bin/sh- -c- minio server --console-address ":5000" http://minio-{0...3}.minio-hs.minio.svc.cluster.local/dataports:- name: datacontainerPort: 9000protocol: "TCP"- name: consolecontainerPort: 5000protocol: "TCP"volumeMounts:- name: datamountPath: /datavolumeClaimTemplates:- metadata:name: dataspec:accessModes: [ "ReadWriteOnce" ]resources:requests:storage: 10GistorageClassName: "minio-storage"
EOF

6.卸载服务

kubectl delete pod minio-mc -n gitlab

kubectl delete -f sts.yaml
kubectl get pod -n gitlab -o wide

kubectl delete -f svc.yaml
kubectl get svc -n gitlab

kubectl delete namespace minio
kubectl get namespace

kubectl get pvc -n gitlab
kubectl delete pvc -n migitlabnio --all

kubectl delete -f pv.yaml -n gitlab
kubectl get pv

kubectl delete -f sc.yaml -n gitlab
kubectl get sc

rm -rf /data/data/minio
mkdir -p /data/data/minio/pv{1..4}
tree /data/data/minio

四.注意事项

分部署部署时，注意使用的minio镜像，我是试了好多版本，才可以的，因为大多数存在minio-dns访问问题，各个几点之间通信异常