docker compose部署flink集群

本次部署2个jobmanager和3个taskmanager

一、部署zookeeper集群

flink使用zookeeper用作高可用

部署集群参考：docker compose部署zookeeper集群-CSDN博客

二、创建目录及配置文件

创建timezone文件，内容填写Asia/Shanghai

手动创建目录：data、conf、tmp、log

conf文件夹内容如下：

需要为每个节点单独准备配置文件，log4j和logback配置可以共用

这些配置文件，可以从官网下载flink包解压后获取，或启动flink容器后docker cp复制出来。

逐个修改flink-conf-xxx.yaml配置

以flink-conf-jobmanager1.yaml举例：配置如下

其他配置文件，根据各配置项的备注描述进行对应修改

三、创建docker compose文件

vim docker-compose.yaml

networks:
  net:
    external: true

services:
  jobmanager1:
    restart: always
    image: apache/flink:1.16.3
    container_name: jobmanager1
    hostname: jobmanager1
    ports:
     - '8081:8081'
    volumes:
      - /etc/localtime:/etc/localtime
      - /home/sumengnan/apache/flink/timezone:/etc/timezone
      - /home/sumengnan/apache/flink/conf/flink-conf-jobmanager1.yaml:/opt/flink/conf/flink-conf.yaml
      - /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
      - /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
      - /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
      - /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
      - /home/sumengnan/apache/flink/data:/opt/flink/data
      - /home/sumengnan/apache/flink/log:/opt/flink/log
      - /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
    networks:
      - net
  jobmanager2:
    restart: always
    image: apache/flink:1.16.3
    container_name: jobmanager2
    hostname: jobmanager2
    ports:
     - '8082:8081'
    volumes:
      - /etc/localtime:/etc/localtime
      - /home/sumengnan/apache/flink/timezone:/etc/timezone
      - /home/sumengnan/apache/flink/conf/flink-conf-jobmanager2.yaml:/opt/flink/conf/flink-conf.yaml
      - /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
      - /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
      - /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
      - /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
      - /home/sumengnan/apache/flink/data:/opt/flink/data
      - /home/sumengnan/apache/flink/log:/opt/flink/log
      - /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
    networks:
      - net
    depenes_on:
      - jobmanager1

  taskmanager1:
    restart: always
    image: apache/flink:1.16.3
    container_name: taskmanager1
    hostname: taskmanager1
    command: taskmanager
    volumes:
      - /etc/localtime:/etc/localtime
      - /home/sumengnan/apache/flink/timezone:/etc/timezone
      - /home/sumengnan/apache/flink/conf/flink-conf-taskmanager1.yaml:/opt/flink/conf/flink-conf.yaml
      - /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
      - /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
      - /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
      - /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
      - /home/sumengnan/apache/flink/data:/opt/flink/data
      - /home/sumengnan/apache/flink/log:/opt/flink/log
      - /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
    networks:
      - net
    depenes_on:
      - jobmanager1
      - jobmanager2
  taskmanager2:
    restart: always
    image: apache/flink:1.16.3
    container_name: taskmanager2
    hostname: taskmanager2
    command: taskmanager
    volumes:
      - /etc/localtime:/etc/localtime
      - /home/sumengnan/apache/flink/timezone:/etc/timezone
      - /home/sumengnan/apache/flink/conf/flink-conf-taskmanager2.yaml:/opt/flink/conf/flink-conf.yaml
      - /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
      - /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
      - /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
      - /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
      - /home/sumengnan/apache/flink/data:/opt/flink/data
      - /home/sumengnan/apache/flink/log:/opt/flink/log
      - /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
    networks:
      - net
    depenes_on:
      - jobmanager1
      - jobmanager2
  taskmanager3:
    restart: always
    image: apache/flink:1.16.3
    container_name: taskmanager3
    hostname: taskmanager3
    command: taskmanager
    volumes:
      - /etc/localtime:/etc/localtime
      - /home/sumengnan/apache/flink/timezone:/etc/timezone
      - /home/sumengnan/apache/flink/conf/flink-conf-taskmanager3.yaml:/opt/flink/conf/flink-conf.yaml
      - /home/sumengnan/apache/flink/conf/log4j.properties:/opt/flink/conf/log4j.properties
      - /home/sumengnan/apache/flink/conf/logback.xml:/opt/flink/conf/logback.xml
      - /home/sumengnan/apache/flink/conf/log4j-console.properties:/opt/flink/conf/log4j-console.properties
      - /home/sumengnan/apache/flink/conf/logback-console.xml:/opt/flink/conf/logback-console.xml
      - /home/sumengnan/apache/flink/data:/opt/flink/data
      - /home/sumengnan/apache/flink/log:/opt/flink/log
      - /home/sumengnan/apache/flink/tmp:/opt/flink/tmp
    networks:
      - net
    depenes_on:
      - jobmanager1
      - jobmanager2

 

 四、启动

五、访问

访问8081或8082jobmanager

 可以看出jobmanager与3个taskmanager工作正常