[Docker][Flink] Added Stack File and Dependencies for Flink Deployment

Docker-Swarm-deployment/analytics/flink/Dockerfile

@@ -0,0 +1,25 @@
+FROM maven:latest as deps
+
+FROM flink:1.20
+
+ENV TZ=Africa/Accra
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN mkdir /opt/flink/plugins/s3-fs-presto \
+    && curl https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-presto/1.20.0/flink-s3-fs-presto-1.20.0.jar --output /opt/flink/plugins/s3-fs-presto/flink-s3-fs-presto-1.20.0.jar
+
+RUN mkdir /opt/flink/plugins/s3-fs-hadoop \
+    && curl https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.20.0/flink-s3-fs-hadoop-1.20.0.jar --output /opt/flink/plugins/s3-fs-hadoop/flink-s3-fs-hadoop-1.20.0.jar
+
+RUN mkdir /opt/flink/plugins/rmq \
+    && curl https://repo1.maven.org/maven2/org/apache/flink/flink-connector-rabbitmq/1.16.1/flink-connector-rabbitmq-1.16.1.jar --output /opt/flink/plugins/rmq/flink-connector-rabbitmq-1.16.1.jar
+
+RUN mkdir /opt/flink/plugins/fasterxml \
+    && curl https://repo1.maven.org/maven2/com/fasterxml/uuid/java-uuid-generator/4.1.0/java-uuid-generator-4.1.0.jar --output /opt/flink/plugins/fasterxml/java-uuid-generator-4.1.0.jar
+
+RUN mkdir /opt/flink/plugins/json \
+    && curl https://repo1.maven.org/maven2/org/json/json/2020518/json-20230227.jar --output /opt/flink/plugins/json/json-20230227.jar
+
+RUN mkdir /opt/flink/plugins/http \
+    && curl https://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.14/httpclient-4.5.14.jar --output /opt/flink/plugins/http/httpclient-4.5.14.jar

Docker-Swarm-deployment/analytics/flink/README.md

@@ -0,0 +1,38 @@
+# Introduction
+Docker swarm stack for Flink deployment
+
+## Flink Installation
+
+## Assign node labels
+ The flink container is constrained to run on specifc node by adding node labels to only one of the nodes, refer [here](https://docs.docker.com/engine/swarm/services/#placement-constraints) for more info. This ensures the container is placed always to same node on restart and able to mount the same local docker volume.
+```sh
+docker node update --label-add flink-node=true <node_name>
+```
+## Copy the secrets 
+```sh
+cp -r example-secrets/ .
+```
+## Replace the minio secrets in 
+```sh
+secrets/flink-conf.yaml
+```
+
+## Define Appropriate values of resources
+
+Define Appropriate values of resources -
+- CPU 
+- RAM 
+- PID limit 
+in `flink-stack.resources.yaml`  for flink as shown in sample resource-values file for [here](flink-stack.resources.yaml)
+
+## Deploy
+Deploy flink stack:
+```sh
+docker stack deploy -c flink-stack.yaml -c flink-stack.resources.yaml flink
+```
+
+# Note
+1. If you need to expose the flink management ui on a port or have custom stack configuration ( see [here](example-flink-stack.custom.yaml) for example configuration of 'flink-stack.custom.yaml' file)  and bring up like as follows
+```sh
+docker stack deploy -c flink-stack.yaml -c flink-stack.resources.yaml -c flink-stack.custom.yaml flink
+```

Docker-Swarm-deployment/analytics/flink/example-flink-stack.custom.yaml

@@ -0,0 +1,9 @@
+version: "3.9"
+
+services:
+  jobmanager:
+    ports: 
+      - target: 8081  
+        published: 8081
+        mode: host 
+

Docker-Swarm-deployment/analytics/flink/example-flink-stack.resources.yaml

@@ -0,0 +1,21 @@
+version: '3.9'
+
+services:
+  jobmanager:
+    deploy:
+      resources: 
+        limits:
+          cpus: '1.50'
+          memory: 6G
+        reservations:
+          cpus: '0.5'   
+          memory: 3G
+  taskmanager:
+    deploy:
+      resources: 
+        limits:
+          cpus: '1.50'
+          memory: 4G
+        reservations:
+          cpus: '0.5'   
+          memory: 2G

Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-conf.yaml

@@ -0,0 +1,39 @@
+jobmanager.rpc.address: tasks.jobmanager
+taskmanager.numberOfTaskSlots: 3
+# web.upload.dir: /jars
+state.backend: rocksdb
+state.checkpoints.dir: s3://state/checkpoints
+state.savepoints.dir: s3://state/savepoints
+s3.endpoint: http://tasks.minio:10000
+s3.path.style.access: true
+s3.access-key: <minio access key>
+s3.secret-key: <minio secret key>
+
+presto.s3.access-key: <minio access key>
+presto.s3.secret-key: <minio secret key>
+presto.s3.endpoint: http://tasks.minio:10000
+presto.s3.path-style-access: true
+
+high-availability: NONE
+high-availability.storageDir: s3://recovery/
+high-availability.zookeeper.quorum: zook:2181
+
+metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+metrics.reporter.prom.port: 9251
+blob.server.port: 6124
+query.server.port: 6125
+
+jobmanager.memory.process.size: 6g
+jobmanager.memory.heap.size: 4g
+jobmanager.memory.off-heap.size: 1g
+jobmanager.memory.jvm-overhead.min: 512m
+jobmanager.memory.jvm-overhead.max: 1g
+
+taskmanager.memory.process.size: 5g
+taskmanager.memory.flink.size: 4g 
+taskmanager.memory.jvm-metaspace.size: 512m
+taskmanager.memory.framework.off-heap.size: 512m
+taskmanager.memory.jvm-overhead.min: 512m
+taskmanager.memory.jvm-overhead.max: 512m
+taskmanager.memory.task.heap.size: 2g
+taskmanager.memory.managed.size: 1200m

Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-jobmanager/log4j-console.properties

@@ -0,0 +1,69 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds.
+monitorInterval=30
+
+# This affects logging for both user code and Flink
+rootLogger.level = INFO
+rootLogger.appenderRef.console.ref = ConsoleAppender
+rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+# Uncomment this if you want to _only_ change Flink's logging
+logger.flink.name = org.apache.flink.api.common.functions
+logger.flink.level = INFO
+
+# The following lines keep the log level of common libraries/connectors on
+# log level DEBUG. The root logger does not override this. You have to manually
+# change the log levels here.
+# logger.akka.name = akka
+# logger.akka.level = INFO
+# logger.kafka.name= org.apache.kafka
+# logger.kafka.level = INFO
+# logger.hadoop.name = org.apache.hadoop
+# logger.hadoop.level = INFO
+# logger.zookeeper.name = org.apache.zookeeper
+# logger.zookeeper.level = INFO
+# logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3
+# logger.shaded_zookeeper.level = INFO
+
+# Log all infos to the console
+appender.console.name = ConsoleAppender
+appender.console.type = CONSOLE
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{MM-ddHH:mm:ss} [%p] %c -> %m %n
+
+# Log all infos in the given rolling file
+appender.rolling.name = RollingFileAppender
+appender.rolling.type = RollingFile
+appender.rolling.append = true
+appender.rolling.fileName = ${sys:log.file}
+appender.rolling.filePattern = ${sys:log.file}.%i
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = %d{MM-ddHH:mm:ss} [%p] %c -> %m %n
+appender.rolling.policies.type = Policies
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=100MB
+appender.rolling.policies.startup.type = OnStartupTriggeringPolicy
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.max = ${env:MAX_LOG_FILE_NUMBER:-10}
+
+# Suppress the irrelevant (wrong) warnings from the Netty channel handler
+logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+logger.netty.level = OFF
+

Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-taskmanager/log4j-console.properties

@@ -0,0 +1,68 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds.
+monitorInterval=30
+
+# This affects logging for both user code and Flink
+rootLogger.level = INFO
+rootLogger.appenderRef.console.ref = ConsoleAppender
+rootLogger.appenderRef.rolling.ref = RollingFileAppender
+
+# Uncomment this if you want to _only_ change Flink's logging
+logger.flink.name = org.apache.flink.api.common.functions
+logger.flink.level = INFO
+
+# The following lines keep the log level of common libraries/connectors on
+# log level DEBUG. The root logger does not override this. You have to manually
+# change the log levels here.
+#logger.akka.name = akka
+#logger.akka.level = INFO
+#logger.kafka.name= org.apache.kafka
+#logger.kafka.level = INFO
+#logger.hadoop.name = org.apache.hadoop
+#logger.hadoop.level = INFO
+#logger.zookeeper.name = org.apache.zookeeper
+#logger.zookeeper.level = INFO
+#logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3
+#logger.shaded_zookeeper.level = INFO
+
+# Log all infos to the console
+appender.console.name = ConsoleAppender
+appender.console.type = CONSOLE
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} [%p] %c %m %xEx{full,separator(|)} %n
+
+# Log all infos in the given rolling file
+appender.rolling.name = RollingFileAppender
+appender.rolling.type = RollingFile
+appender.rolling.append = true
+appender.rolling.fileName = ${sys:log.file}
+appender.rolling.filePattern = ${sys:log.file}.%i
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} [%p] %c %m %xEx{full,separator(|)} %n
+appender.rolling.policies.type = Policies
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=100MB
+appender.rolling.policies.startup.type = OnStartupTriggeringPolicy
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.max = ${env:MAX_LOG_FILE_NUMBER:-10}
+
+# Suppress the irrelevant (wrong) warnings from the Netty channel handler
+logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+logger.netty.level = OFF

Docker-Swarm-deployment/analytics/flink/flink-stack.yaml

@@ -0,0 +1,77 @@
+version: '3.9'
+
+networks:
+  overlay-net:
+    external: true
+    driver: overlay
+
+services:
+  jobmanager:
+    image: ghcr.io/datakaveri/flink:1.20-2
+    hostname: jobmanager
+    configs:
+      - source: config
+        target: /opt/flink/conf/flink-conf.yaml
+      - source: jm-console-properties
+        target: /opt/flink/conf/log4j-console.properties
+    restart: on-failure
+    networks:
+      - overlay-net
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: any
+        max_attempts: 5
+      placement:
+        constraints:
+          - "node.labels.flink-node==true"
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "5"
+        max-size: "10m"
+        tag: "{\"name\":\"{{.Name}}\",\"id\":\"{{.ID}}\"}"
+    ports:
+    command: "jobmanager"
+
+
+
+  taskmanager:
+    image: ghcr.io/datakaveri/flink:1.20-2
+    hostname: taskmanager
+    configs:
+      - source: config
+        target: /opt/flink/conf/flink-conf.yaml
+      - source: tm-console-properties
+        target: /opt/flink/conf/log4j-console.properties:rw
+    restart: on-failure
+    networks:
+      - overlay-net
+    deploy:
+      replicas: 2
+      restart_policy:
+        condition: any
+        max_attempts: 5
+      placement:
+        constraints:
+          - "node.labels.flink-node==true"
+    logging:
+      driver: "json-file"
+      options:
+        max-file: "5"
+        max-size: "10m"
+        tag: "{\"name\":\"{{.Name}}\",\"id\":\"{{.ID}}\"}"
+    command: "taskmanager"
+
+
+volumes:
+  jars:
+
+configs:
+  jm-console-properties:
+    file: ./secrets/flink-jobmanager/log4j-console.properties
+  tm-console-properties:
+    file: ./secrets/flink-taskmanager/log4j-console.properties
+  config:
+    file: ./secrets/flink-conf.yaml
+