docker-compose.yml

version: '3'

x-connect-image: &connect-image cricketeerone/apache-kafka-connect:3.8.0

x-connect: &connect-vars
  CONNECT_BOOTSTRAP_SERVERS: kafka:29092

  CONNECT_GROUP_ID: cg_connect-jib
  CONNECT_CONFIG_STORAGE_TOPIC: connect-jib_config
  CONNECT_OFFSET_STORAGE_TOPIC: connect-jib_offsets
  CONNECT_STATUS_STORAGE_TOPIC: connect-jib_status
  # Cannot be higher than the number of brokers in the Kafka cluster
  CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
  CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
  CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
  # Defaults for all connectors
  CONNECT_KEY_CONVERTER: org.apache.kafka.connect.converters.ByteArrayConverter
  CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.converters.ByteArrayConverter
  # Where Jib places classes
  CONNECT_PLUGIN_PATH: /app/libs

  # Connect client overrides
  CONNECT_TASK_SHUTDOWN_GRACEFUL_TIMEOUT_MS: 30000
  CONNECT_OFFSET_FLUSH_INTERVAL_MS: 900000
  # Connect consumer overrides
  CONNECT_CONSUMER_MAX_POLL_RECORDS: 500

services:
  kafka:
    image: bitnami/kafka:3.8.0
    restart: unless-stopped
    ports:
      - '9092:9092'
    volumes:
      - 'kafka_data:/bitnami/kafka'
      - $PWD/lipsum.txt:/data/lipsum.txt:ro  # Some data to produce
    environment:
      BITNAMI_DEBUG: yes
      ALLOW_PLAINTEXT_LISTENER: yes
      # BEGIN: Kraft
      KAFKA_ENABLE_KRAFT: yes
      KAFKA_CFG_PROCESS_ROLES: controller,broker
      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_CFG_NODE_ID: 1
      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      # END: Kraft
      KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
      KAFKA_CFG_LOG_RETENTION_HOURS: 48  # 2 days of retention for demo purposes
      # https://rmoff.net/2018/08/02/kafka-listeners-explained/
      KAFKA_CFG_LISTENERS: INTERNAL://:29092,CONTROLLER://:9093,EXTERNAL://0.0.0.0:9092
      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://127.0.0.1:9092
      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT

  # Jib app
  connect-jib-1:
    image: *connect-image
    hostname: connect-jib-1
    depends_on:
      - kafka
    ports:
      - '8083:8083'
    environment:
      <<: *connect-vars
      CONNECT_REST_ADVERTISED_HOST_NAME: connect-jib-1

volumes:
  kafka_data:
    driver: local