linux-docker-compose.yml

services:
  # Apache Flink Job Manager
  jobmanager:
    build:
      context: .
      dockerfile: linux-Dockerfile
      args:
        AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
        AWS_REGION: ${AWS_REGION}
        AWS_S3_BUCKET: ${AWS_S3_BUCKET}
        AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
    ports:
      - 8081:8081
      - 9092:9092
      - 8443:443
      - 8501:8501
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        rest.flamegraph.enabled: true
        web.backpressure.refresh-interval: 10000
        classloader.check-leaked-classloader: false
        python.executable: /opt/flink/python_apps/.venv/bin/python
      - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
      - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
      - AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN
      - AWS_REGION=$AWS_REGION
      - AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
      - AWS_S3_BUCKET=${AWS_S3_BUCKET}
  # Apache Flink Task Manager
  taskmanager:
    build:
      context: .
      dockerfile: linux-Dockerfile
      args:
        AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
        AWS_REGION: ${AWS_REGION}
        AWS_S3_BUCKET: ${AWS_S3_BUCKET}
    depends_on:
      - jobmanager
    command: taskmanager
    scale: 3
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 2
        taskmanager.memory.task.off-heap.size: 256m
        taskmanager.memory.process.size: 2048m
        taskmanager.memory.network.fraction: 0.2
        python.fn-execution.memory.limit: 512mb
        classloader.check-leaked-classloader: false
        python.executable: /opt/flink/python_apps/.venv/bin/python
      - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
      - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
      - AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN
      - AWS_REGION=$AWS_REGION
      - AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
      - AWS_S3_BUCKET=${AWS_S3_BUCKET}