From be6ab11846f965d8aed1cc6bedf26cf01838b3d1 Mon Sep 17 00:00:00 2001 From: Sridhar Rao Kukkala Date: Wed, 9 Oct 2024 14:12:26 +0530 Subject: [PATCH 1/3] Created flink stack file and its dependencies --- .../analytics/flink/Dockerfile | 25 ++++++ .../analytics/flink/README.md | 38 +++++++++ .../flink-jobmanager/log4j-console.properties | 69 +++++++++++++++++ .../analytics/flink/flink-stack.yaml | 77 +++++++++++++++++++ .../log4j-console.properties | 68 ++++++++++++++++ 5 files changed, 277 insertions(+) create mode 100644 Docker-Swarm-deployment/analytics/flink/Dockerfile create mode 100644 Docker-Swarm-deployment/analytics/flink/README.md create mode 100644 Docker-Swarm-deployment/analytics/flink/flink-jobmanager/log4j-console.properties create mode 100644 Docker-Swarm-deployment/analytics/flink/flink-stack.yaml create mode 100644 Docker-Swarm-deployment/analytics/flink/flink-taskmanager/log4j-console.properties diff --git a/Docker-Swarm-deployment/analytics/flink/Dockerfile b/Docker-Swarm-deployment/analytics/flink/Dockerfile new file mode 100644 index 000000000..6c00f7396 --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/Dockerfile @@ -0,0 +1,25 @@ +FROM maven:latest as deps + +FROM flink:1.20 + +ENV TZ=Africa/Accra + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN mkdir /opt/flink/plugins/s3-fs-presto \ + && curl https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-presto/1.20.0/flink-s3-fs-presto-1.20.0.jar --output /opt/flink/plugins/s3-fs-presto/flink-s3-fs-presto-1.20.0.jar + +RUN mkdir /opt/flink/plugins/s3-fs-hadoop \ + && curl https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.20.0/flink-s3-fs-hadoop-1.20.0.jar --output /opt/flink/plugins/s3-fs-hadoop/flink-s3-fs-hadoop-1.20.0.jar + +RUN mkdir /opt/flink/plugins/rmq \ + && curl https://repo1.maven.org/maven2/org/apache/flink/flink-connector-rabbitmq/1.16.1/flink-connector-rabbitmq-1.16.1.jar --output /opt/flink/plugins/rmq/flink-connector-rabbitmq-1.16.1.jar + +RUN mkdir /opt/flink/plugins/fasterxml \ + && curl https://repo1.maven.org/maven2/com/fasterxml/uuid/java-uuid-generator/4.1.0/java-uuid-generator-4.1.0.jar --output /opt/flink/plugins/fasterxml/java-uuid-generator-4.1.0.jar + +RUN mkdir /opt/flink/plugins/json \ + && curl https://repo1.maven.org/maven2/org/json/json/2020518/json-20230227.jar --output /opt/flink/plugins/json/json-20230227.jar + +RUN mkdir /opt/flink/plugins/http \ + && curl https://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.14/httpclient-4.5.14.jar --output /opt/flink/plugins/http/httpclient-4.5.14.jar diff --git a/Docker-Swarm-deployment/analytics/flink/README.md b/Docker-Swarm-deployment/analytics/flink/README.md new file mode 100644 index 000000000..16a7834fa --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/README.md @@ -0,0 +1,38 @@ +# Introduction +Docker swarm stack for Flink deployment + +## Flink Installation + +## Assign node labels + The flink container is constrained to run on specifc node by adding node labels to only one of the nodes, refer [here](https://docs.docker.com/engine/swarm/services/#placement-constraints) for more info. This ensures the container is placed always to same node on restart and able to mount the same local docker volume. +```sh +docker node update --label-add flink-node=true +``` +## Copy the secrets +```sh +cp -r example-secrets/ . +``` +## Replace the minio secrets in +```sh +secrets/flink-conf.yaml +``` + +## Define Appropriate values of resources + +Define Appropriate values of resources - +- CPU +- RAM +- PID limit +in `flink-stack.resources.yaml` for flink as shown in sample resource-values file for [here](flink-stack.resources.yaml) + +## Deploy +Deploy flink stack: +```sh +docker stack deploy -c flink-stack.yaml -c flink-stack.resources.yaml flink +``` + +# Note +1. If you need to expose the flink management ui on a port or have custom stack configuration ( see [here](example-flink-stack.custom.yaml) for example configuration of 'flink-stack.custom.yaml' file) and bring up like as follows +```sh +docker stack deploy -c flink-stack.yaml -c flink-stack.resources.yaml -c flink-stack.custom.yaml flink +``` \ No newline at end of file diff --git a/Docker-Swarm-deployment/analytics/flink/flink-jobmanager/log4j-console.properties b/Docker-Swarm-deployment/analytics/flink/flink-jobmanager/log4j-console.properties new file mode 100644 index 000000000..eed449e8e --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/flink-jobmanager/log4j-console.properties @@ -0,0 +1,69 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds. +monitorInterval=30 + +# This affects logging for both user code and Flink +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender +rootLogger.appenderRef.rolling.ref = RollingFileAppender + +# Uncomment this if you want to _only_ change Flink's logging +logger.flink.name = org.apache.flink.api.common.functions +logger.flink.level = INFO + +# The following lines keep the log level of common libraries/connectors on +# log level DEBUG. The root logger does not override this. You have to manually +# change the log levels here. +# logger.akka.name = akka +# logger.akka.level = INFO +# logger.kafka.name= org.apache.kafka +# logger.kafka.level = INFO +# logger.hadoop.name = org.apache.hadoop +# logger.hadoop.level = INFO +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = INFO +# logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3 +# logger.shaded_zookeeper.level = INFO + +# Log all infos to the console +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{MM-ddHH:mm:ss} [%p] %c -> %m %n + +# Log all infos in the given rolling file +appender.rolling.name = RollingFileAppender +appender.rolling.type = RollingFile +appender.rolling.append = true +appender.rolling.fileName = ${sys:log.file} +appender.rolling.filePattern = ${sys:log.file}.%i +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = %d{MM-ddHH:mm:ss} [%p] %c -> %m %n +appender.rolling.policies.type = Policies +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=100MB +appender.rolling.policies.startup.type = OnStartupTriggeringPolicy +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = ${env:MAX_LOG_FILE_NUMBER:-10} + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline +logger.netty.level = OFF + diff --git a/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml b/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml new file mode 100644 index 000000000..a534fc34f --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml @@ -0,0 +1,77 @@ +version: '3.9' + +networks: + overlay-net: + external: true + driver: overlay + +services: + jobmanager: + image: ghcr.io/datakaveri/flink:1.20-2 + hostname: jobmanager + configs: + - source: config + target: /opt/flink/conf/flink-conf.yaml + - source: jm-console-properties + target: /opt/flink/conf/log4j-console.properties + restart: on-failure + networks: + - overlay-net + deploy: + replicas: 1 + restart_policy: + condition: any + max_attempts: 5 + placement: + constraints: + - "node.labels.flink-node==true" + logging: + driver: "json-file" + options: + max-file: "5" + max-size: "10m" + tag: "{\"name\":\"{{.Name}}\",\"id\":\"{{.ID}}\"}" + ports: + command: "jobmanager" + + + + taskmanager: + image: ghcr.io/datakaveri/flink:1.20-2 + hostname: taskmanager + configs: + - source: config + target: /opt/flink/conf/flink-conf.yaml + - source: tm-console-properties + target: /opt/flink/conf/log4j-console.properties:rw + restart: on-failure + networks: + - overlay-net + deploy: + replicas: 2 + restart_policy: + condition: any + max_attempts: 5 + placement: + constraints: + - "node.labels.flink-node==true" + logging: + driver: "json-file" + options: + max-file: "5" + max-size: "10m" + tag: "{\"name\":\"{{.Name}}\",\"id\":\"{{.ID}}\"}" + command: "taskmanager" + + +volumes: + jars: + +configs: + jm-console-properties: + file: ./flink-jobmanager/log4j-console.properties + tm-console-properties: + file: ./flink-taskmanager/log4j-console.properties + config: + file: ./flink-conf.yaml + diff --git a/Docker-Swarm-deployment/analytics/flink/flink-taskmanager/log4j-console.properties b/Docker-Swarm-deployment/analytics/flink/flink-taskmanager/log4j-console.properties new file mode 100644 index 000000000..212db1017 --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/flink-taskmanager/log4j-console.properties @@ -0,0 +1,68 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds. +monitorInterval=30 + +# This affects logging for both user code and Flink +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender +rootLogger.appenderRef.rolling.ref = RollingFileAppender + +# Uncomment this if you want to _only_ change Flink's logging +logger.flink.name = org.apache.flink.api.common.functions +logger.flink.level = INFO + +# The following lines keep the log level of common libraries/connectors on +# log level DEBUG. The root logger does not override this. You have to manually +# change the log levels here. +#logger.akka.name = akka +#logger.akka.level = INFO +#logger.kafka.name= org.apache.kafka +#logger.kafka.level = INFO +#logger.hadoop.name = org.apache.hadoop +#logger.hadoop.level = INFO +#logger.zookeeper.name = org.apache.zookeeper +#logger.zookeeper.level = INFO +#logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3 +#logger.shaded_zookeeper.level = INFO + +# Log all infos to the console +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} [%p] %c %m %xEx{full,separator(|)} %n + +# Log all infos in the given rolling file +appender.rolling.name = RollingFileAppender +appender.rolling.type = RollingFile +appender.rolling.append = true +appender.rolling.fileName = ${sys:log.file} +appender.rolling.filePattern = ${sys:log.file}.%i +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} [%p] %c %m %xEx{full,separator(|)} %n +appender.rolling.policies.type = Policies +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=100MB +appender.rolling.policies.startup.type = OnStartupTriggeringPolicy +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = ${env:MAX_LOG_FILE_NUMBER:-10} + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline +logger.netty.level = OFF From 2f4a6a96e7b97e2dc1293bdb9da227cb8ab456b4 Mon Sep 17 00:00:00 2001 From: Sridhar Rao Kukkala Date: Wed, 9 Oct 2024 14:19:07 +0530 Subject: [PATCH 2/3] Added resources and custom files --- .../flink/example-flink-stack.custom.yaml | 9 ++++++++ .../flink/example-flink-stack.resources.yaml | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 Docker-Swarm-deployment/analytics/flink/example-flink-stack.custom.yaml create mode 100644 Docker-Swarm-deployment/analytics/flink/example-flink-stack.resources.yaml diff --git a/Docker-Swarm-deployment/analytics/flink/example-flink-stack.custom.yaml b/Docker-Swarm-deployment/analytics/flink/example-flink-stack.custom.yaml new file mode 100644 index 000000000..7d58189f6 --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/example-flink-stack.custom.yaml @@ -0,0 +1,9 @@ +version: "3.9" + +services: + jobmanager: + ports: + - target: 8081 + published: 8081 + mode: host + diff --git a/Docker-Swarm-deployment/analytics/flink/example-flink-stack.resources.yaml b/Docker-Swarm-deployment/analytics/flink/example-flink-stack.resources.yaml new file mode 100644 index 000000000..41128018b --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/example-flink-stack.resources.yaml @@ -0,0 +1,21 @@ +version: '3.9' + +services: + jobmanager: + deploy: + resources: + limits: + cpus: '1.50' + memory: 6G + reservations: + cpus: '0.5' + memory: 3G + taskmanager: + deploy: + resources: + limits: + cpus: '1.50' + memory: 4G + reservations: + cpus: '0.5' + memory: 2G \ No newline at end of file From b37a9915d873a0e6c1ef435dd5b928396e27a195 Mon Sep 17 00:00:00 2001 From: Sridhar Rao Kukkala Date: Fri, 18 Oct 2024 11:34:33 +0530 Subject: [PATCH 3/3] Created example-secrets dir and moved configurations to example-secrets/secrets --- .../example-secrets/secrets/flink-conf.yaml | 39 +++++++++++++++++++ .../flink-jobmanager/log4j-console.properties | 0 .../log4j-console.properties | 0 .../analytics/flink/flink-stack.yaml | 6 +-- 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-conf.yaml rename Docker-Swarm-deployment/analytics/flink/{ => example-secrets/secrets}/flink-jobmanager/log4j-console.properties (100%) rename Docker-Swarm-deployment/analytics/flink/{ => example-secrets/secrets}/flink-taskmanager/log4j-console.properties (100%) diff --git a/Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-conf.yaml b/Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-conf.yaml new file mode 100644 index 000000000..07e44edda --- /dev/null +++ b/Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-conf.yaml @@ -0,0 +1,39 @@ +jobmanager.rpc.address: tasks.jobmanager +taskmanager.numberOfTaskSlots: 3 +# web.upload.dir: /jars +state.backend: rocksdb +state.checkpoints.dir: s3://state/checkpoints +state.savepoints.dir: s3://state/savepoints +s3.endpoint: http://tasks.minio:10000 +s3.path.style.access: true +s3.access-key: +s3.secret-key: + +presto.s3.access-key: +presto.s3.secret-key: +presto.s3.endpoint: http://tasks.minio:10000 +presto.s3.path-style-access: true + +high-availability: NONE +high-availability.storageDir: s3://recovery/ +high-availability.zookeeper.quorum: zook:2181 + +metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter +metrics.reporter.prom.port: 9251 +blob.server.port: 6124 +query.server.port: 6125 + +jobmanager.memory.process.size: 6g +jobmanager.memory.heap.size: 4g +jobmanager.memory.off-heap.size: 1g +jobmanager.memory.jvm-overhead.min: 512m +jobmanager.memory.jvm-overhead.max: 1g + +taskmanager.memory.process.size: 5g +taskmanager.memory.flink.size: 4g +taskmanager.memory.jvm-metaspace.size: 512m +taskmanager.memory.framework.off-heap.size: 512m +taskmanager.memory.jvm-overhead.min: 512m +taskmanager.memory.jvm-overhead.max: 512m +taskmanager.memory.task.heap.size: 2g +taskmanager.memory.managed.size: 1200m diff --git a/Docker-Swarm-deployment/analytics/flink/flink-jobmanager/log4j-console.properties b/Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-jobmanager/log4j-console.properties similarity index 100% rename from Docker-Swarm-deployment/analytics/flink/flink-jobmanager/log4j-console.properties rename to Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-jobmanager/log4j-console.properties diff --git a/Docker-Swarm-deployment/analytics/flink/flink-taskmanager/log4j-console.properties b/Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-taskmanager/log4j-console.properties similarity index 100% rename from Docker-Swarm-deployment/analytics/flink/flink-taskmanager/log4j-console.properties rename to Docker-Swarm-deployment/analytics/flink/example-secrets/secrets/flink-taskmanager/log4j-console.properties diff --git a/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml b/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml index a534fc34f..fd919c251 100644 --- a/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml +++ b/Docker-Swarm-deployment/analytics/flink/flink-stack.yaml @@ -69,9 +69,9 @@ volumes: configs: jm-console-properties: - file: ./flink-jobmanager/log4j-console.properties + file: ./secrets/flink-jobmanager/log4j-console.properties tm-console-properties: - file: ./flink-taskmanager/log4j-console.properties + file: ./secrets/flink-taskmanager/log4j-console.properties config: - file: ./flink-conf.yaml + file: ./secrets/flink-conf.yaml