docker-compose.yml

version: "3"

volumes:
  postgres_data: {}
  postgres_data_backups: {}

services:
  spark:
    build:
      context: .
      dockerfile: ./container/Dockerfile
    env_file:
      - ./.envs/.postgres
      - ./.envs/.env_vars
    volumes:
      - ./src:/opt/work-dir
    command: sleep infinity

  postgres:
    image: postgres:12.3
    container_name: postgres
    ports: 
      - 5432:5432
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - postgres_data_backups:/backups
    env_file:
      - ./.envs/.postgres
    restart: unless-stopped

  zookeeper:
    image: confluentinc/cp-zookeeper:5.5.0
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
  kafka:
    image: confluentinc/cp-kafka:5.5.0
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      # "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
      # An important note about accessing Kafka from clients on other machines:
      # -----------------------------------------------------------------------
      #
      # The config used here exposes port 9092 for _external_ connections to the broker
      # i.e. those from _outside_ the docker network. This could be from the host machine
      # running docker, or maybe further afield if you've got a more complicated setup.
      # If the latter is true, you will need to change the value 'localhost' in
      # KAFKA_ADVERTISED_LISTENERS to one that is resolvable to the docker host from those
      # remote clients
      #
      # For connections _internal_ to the docker network, such as from other services
      # and components, use kafka:29092.
      #
      # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
      # "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
      #
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://${KAFKA_HOSTNAME_COMMAND}:9092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
  kafkacat:
    image: confluentinc/cp-kafkacat
    depends_on:
      - kafka
    command: "bash -c 'sleep infinity'"

  kafka-connect:
    image: confluentinc/cp-kafka-connect-base:latest
    container_name: kafka-connect
    depends_on:
      - zookeeper
      - kafka
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/'

    command:
      - bash
      - -c
      - |
        echo "Installing connector plugins"
        confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:5.5.0
        confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.3.0
        # -----------

        # -----------
        # Launch the Kafka Connect worker
        /etc/confluent/docker/run &
        #
        # Don't exit
        sleep infinity