k8s部署Kafka集群超详细讲解

准备部署环境

Kubernetes集群信息

NAME	VERSION
k8s-master	v1.29.2
k8s-node01	v1.29.2
k8s-node02	v1.29.2

Kafka：3.7.1版本，apche版本
Zookeeper：3.6.3版本

准备StorageClass

# kubectl get sc 
NAME                PROVISIONER      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-csi (default)   nfs.csi.k8s.io   Delete          Immediate           false                  51d

创建namespace

Kubectl create ns kafka

部署zookeeper集群

编写zookeeper集群部署zk.yaml文件

apiVersion: v1
kind: Service
metadata:name: zk-hsnamespace: kafkalabels:app: zk
spec:ports:- port: 2888name: server- port: 3888name: leader-election- port: 2181name: clientclusterIP: Noneselector:app: zk
---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: zknamespace: kafka
spec:selector:matchLabels:app: zkserviceName: zk-hsreplicas: 3updateStrategy:type: RollingUpdatepodManagementPolicy: Paralleltemplate:metadata:labels:app: zkspec:containers:- name: zookeeperimagePullPolicy: Alawysimage: registry.cn-hangzhou.aliyuncs.com/aliyun_0612/zookeeper:3.6.3resources:requests:memory: "200Mi"cpu: "0.1"ports:- containerPort: 2181name: client- containerPort: 2888name: server- containerPort: 3888name: leader-electioncommand:- sh- -c- "start-zookeeper \--servers=3 \--data_dir=/var/lib/zookeeper/data \--data_log_dir=/var/lib/zookeeper/data/log \--conf_dir=/opt/zookeeper/conf \--client_port=2181 \--election_port=3888 \--server_port=2888 \--tick_time=2000 \--init_limit=10 \--sync_limit=5 \--heap=512M \--max_client_cnxns=60 \--snap_retain_count=3 \--purge_interval=12 \--max_session_timeout=40000 \--min_session_timeout=4000 \--log_level=INFO"readinessProbe:exec:command:- sh- -c- "zookeeper-ready 2181"initialDelaySeconds: 10timeoutSeconds: 5livenessProbe:exec:command:- sh- -c- "zookeeper-ready 2181"initialDelaySeconds: 10timeoutSeconds: 5volumeMounts:- name: datadirmountPath: /var/lib/zookeepersecurityContext:runAsUser: 0fsGroup: 0volumeClaimTemplates:- metadata:name: datadirspec:accessModes: [ "ReadWriteMany" ]storageClassName: nfs-csiresources:requests:storage: 2Gi

kafka集群部署kafka.yaml文件

---
apiVersion: v1
kind: Service
metadata:name: kafka-svcnamespace: kafkalabels:app: kafka
spec:ports:- port: 9092name: serverclusterIP: Noneselector:app: kafka
---
apiVersion: apps/v1
kind: StatefulSet
metadata:name: kafkanamespace: kafka
spec:selector:matchLabels:app: kafkaserviceName: kafka-svcreplicas: 3template:metadata:labels:app: kafkaspec:containers:- name: k8s-kafkaimagePullPolicy: Always image: registry.cn-hangzhou.aliyuncs.com/aliyun_0612/kafka:3.7.1resources:requests:memory: "600Mi"cpu: 500mports:- containerPort: 9092name: servercommand:- sh- -c- "exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties --override broker.id=${HOSTNAME##*-} \--override listeners=PLAINTEXT://:9092 \--override zookeeper.connect=zk-hs.kafka.svc.cluster.local:2181 \--override log.dir=/var/lib/kafka \--override auto.create.topics.enable=true \--override auto.leader.rebalance.enable=true \--override background.threads=10 \--override compression.type=producer \--override delete.topic.enable=false \--override leader.imbalance.check.interval.seconds=300 \--override leader.imbalance.per.broker.percentage=10 \--override log.flush.interval.messages=9223372036854775807 \--override log.flush.offset.checkpoint.interval.ms=60000 \--override log.flush.scheduler.interval.ms=9223372036854775807 \--override log.retention.bytes=-1 \--override log.retention.hours=168 \--override log.roll.hours=168 \--override log.roll.jitter.hours=0 \--override log.segment.bytes=1073741824 \--override log.segment.delete.delay.ms=60000 \--override message.max.bytes=1000012 \--override min.insync.replicas=1 \--override num.io.threads=8 \--override num.network.threads=3 \--override num.recovery.threads.per.data.dir=1 \--override num.replica.fetchers=1 \--override offset.metadata.max.bytes=4096 \--override offsets.commit.required.acks=-1 \--override offsets.commit.timeout.ms=5000 \--override offsets.load.buffer.size=5242880 \--override offsets.retention.check.interval.ms=600000 \--override offsets.retention.minutes=1440 \--override offsets.topic.compression.codec=0 \--override offsets.topic.num.partitions=50 \--override offsets.topic.replication.factor=3 \--override offsets.topic.segment.bytes=104857600 \--override queued.max.requests=500 \--override quota.consumer.default=9223372036854775807 \--override quota.producer.default=9223372036854775807 \--override replica.fetch.min.bytes=1 \--override replica.fetch.wait.max.ms=500 \--override replica.high.watermark.checkpoint.interval.ms=5000 \--override replica.lag.time.max.ms=10000 \--override replica.socket.receive.buffer.bytes=65536 \--override replica.socket.timeout.ms=30000 \--override request.timeout.ms=30000 \--override socket.receive.buffer.bytes=102400 \--override socket.request.max.bytes=104857600 \--override socket.send.buffer.bytes=102400 \--override unclean.leader.election.enable=true \--override zookeeper.session.timeout.ms=6000 \--override zookeeper.set.acl=false \--override broker.id.generation.enable=true \--override connections.max.idle.ms=600000 \--override controlled.shutdown.enable=true \--override controlled.shutdown.max.retries=3 \--override controlled.shutdown.retry.backoff.ms=5000 \--override controller.socket.timeout.ms=30000 \--override default.replication.factor=1 \--override fetch.purgatory.purge.interval.requests=1000 \--override group.max.session.timeout.ms=300000 \--override group.min.session.timeout.ms=6000 \--override inter.broker.protocol.version=2.2.0 \--override log.cleaner.backoff.ms=15000 \--override log.cleaner.dedupe.buffer.size=134217728 \--override log.cleaner.delete.retention.ms=86400000 \--override log.cleaner.enable=true \--override log.cleaner.io.buffer.load.factor=0.9 \--override log.cleaner.io.buffer.size=524288 \--override log.cleaner.io.max.bytes.per.second=1.7976931348623157E308 \--override log.cleaner.min.cleanable.ratio=0.5 \--override log.cleaner.min.compaction.lag.ms=0 \--override log.cleaner.threads=1 \--override log.cleanup.policy=delete \--override log.index.interval.bytes=4096 \--override log.index.size.max.bytes=10485760 \--override log.message.timestamp.difference.max.ms=9223372036854775807 \--override log.message.timestamp.type=CreateTime \--override log.preallocate=false \--override log.retention.check.interval.ms=300000 \--override max.connections.per.ip=2147483647 \--override num.partitions=3 \--override producer.purgatory.purge.interval.requests=1000 \--override replica.fetch.backoff.ms=1000 \--override replica.fetch.max.bytes=1048576 \--override replica.fetch.response.max.bytes=10485760 \--override reserved.broker.max.id=1000 "env:- name: KAFKA_HEAP_OPTSvalue : "-Xmx512M -Xms512M"- name: KAFKA_OPTSvalue: "-Dlogging.level=INFO"volumeMounts:- name: datadirmountPath: /var/lib/kafkareadinessProbe:tcpSocket:port: 9092timeoutSeconds: 1initialDelaySeconds: 5securityContext:runAsUser: 1000fsGroup: 1000volumeClaimTemplates:- metadata:name: datadirspec:accessModes: [ "ReadWriteMany" ]storageClassName: nfs-csiresources:requests:storage: 10Gi

部署集群

部署集群

[root@k8s-master ~]# kubectl apply -f zk.yaml
[root@k8s-master ~]# kubectl apply -f kafka.yaml

检查状态

出现启动失败情况，解决不了可以下方评论，专业大神会提供解决方案

验证zk集群

进入zk pod中，查看zk集群是否正常

[root@k8s-master test]# kubectl exec -it -nkafka zk-0 bash

使用

[root@k8s-master test]# zkCli.sh

查看kafka brokers

通过验证，zk集群正常

验证Kafka集群

进入kafka pod节点中

[root@k8s-master ~]# kubectl exec -it -nkafka kafka-0 bash

创建测试topic

/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic mytopic --replication-factor 3 --partitions 3

这里要使用绝对路径

使用describe 查看topic分区数以及副本数正常

/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --describe --topic mytopic

这里要使用绝对路径

至此，kafka集群在kubernetes上部署已经完成。
有什么问题可以下方评论。