docker-compose.yml

version: "3"
services:
  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:latest
    ports:
      - 8080:8080
    depends_on:
      - kafka0
      - kafka1
      - schemaregistry0
      - schemaregistry1
      - kafka-connect0
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092
      KAFKA_CLUSTERS_0_METRICS_PORT: 9997
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083
      KAFKA_CLUSTERS_1_NAME: secondLocal
      KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS: kafka1:29092
      KAFKA_CLUSTERS_1_METRICS_PORT: 9998
      KAFKA_CLUSTERS_1_SCHEMAREGISTRY: http://schemaregistry1:8085
      DYNAMIC_CONFIG_ENABLED: "true"
    networks:
      - go-kafka

  kafka0:
    image: confluentinc/cp-kafka:7.2.1
    hostname: kafka0
    container_name: kafka0
    ports:
      - "9092:9092"
      - "9997:9997"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9997
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka0:29093"
      KAFKA_LISTENERS: "PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs"
    volumes:
      - ./scripts/update_run.sh:/tmp/update_run.sh
    command: 'bash -c ''if [ ! -f /tmp/update_run.sh ]; then echo "ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'''
    networks:
      - go-kafka

  kafka1:
    image: confluentinc/cp-kafka:7.2.1
    hostname: kafka1
    container_name: kafka1
    ports:
      - "9093:9092"
      - "9998:9998"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9998
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9998
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:29093"
      KAFKA_LISTENERS: "PLAINTEXT://kafka1:29092,CONTROLLER://kafka1:29093,PLAINTEXT_HOST://0.0.0.0:9092"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs"
    volumes:
      - ./scripts/update_run.sh:/tmp/update_run.sh
    command: 'bash -c ''if [ ! -f /tmp/update_run.sh ]; then echo "ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'''
    networks:
      - go-kafka

  schemaregistry0:
    image: confluentinc/cp-schema-registry:7.2.1
    ports:
      - 8085:8085
    depends_on:
      - kafka0
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092
      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
      SCHEMA_REGISTRY_HOST_NAME: schemaregistry0
      SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085
      SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http"
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
    networks:
      - go-kafka

  schemaregistry1:
    image: confluentinc/cp-schema-registry:7.2.1
    ports:
      - 18085:8085
    depends_on:
      - kafka1
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:29092
      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
      SCHEMA_REGISTRY_HOST_NAME: schemaregistry1
      SCHEMA_REGISTRY_LISTENERS: http://schemaregistry1:8085
      SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http"
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
    networks:
      - go-kafka

  kafka-connect0:
    image: confluentinc/cp-kafka-connect:7.2.1
    ports:
      - 8083:8083
    depends_on:
      - kafka0
      - schemaregistry0
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka0:29092
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: _connect_status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
    networks:
      - go-kafka

  kafka-init-topics:
    image: confluentinc/cp-kafka:7.2.1
    volumes:
      - ./data/message.json:/data/message.json
    depends_on:
      - kafka1
    command: "bash -c 'echo Waiting for Kafka to be ready... && \
      cub kafka-ready -b kafka0:29092 1 30 && \
      cub kafka-ready -b kafka1:29092 1 30 && \
      kafka-topics --create --topic eventStore_Task --partitions 10 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \
      kafka-topics --create --topic TASK_CREATED_V1 --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka1:29092'"
    networks:
      - go-kafka

  # postgres:
  #   container_name: postgres
  #   image: postgres
  #   volumes:
  #     - pg-data:/var/lib/postgresql/data
  #   environment:
  #     POSTGRES_USER: "user"
  #     POSTGRES_PASSWORD: "pass"
  #     POSTGRES_DB: "postgres"
  #   ports:
  #     - 5432:5432

  # mysql:
  #   image: mysql
  #   container_name: mysql
  #   restart: always
  #   volumes:
  #     - ./mysql-data:/var/lib/mysql
  #   environment:
  #     MYSQL_ROOT_PASSWORD: password
  #     MYSQL_DATABASE: local
  #     MYSQL_USER: mysql
  #     MYSQL_PASSWORD: password
  #   ports:
  #     - "3306:3306"

  elastic:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
    container_name: elasticsearch
    restart: always
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      discovery.type: single-node
    volumes:
      - ./esdata:/usr/share/elasticsearch/data
    networks:
      - go-kafka

  logstash_0:
    image: logstash:8.9.2
    container_name: logstash_0
    command: bash -c "cd bin && logstash-plugin install logstash-output-elasticsearch && logstash -f /usr/share/logstash/config/logstash.conf"
    volumes:
      - ./logstash_0.conf:/usr/share/logstash/config/logstash.conf
    depends_on:
      - elastic
      - kafka0
    networks:
      - go-kafka

  logstash_1:
    image: logstash:8.9.2
    container_name: logstash_1
    command: bash -c "cd bin && logstash-plugin install logstash-output-mongodb && logstash -f /usr/share/logstash/config/logstash.conf"
    volumes:
      - ./logstash_1.conf:/usr/share/logstash/config/logstash.conf
    depends_on:
      - kafka1
      - mongodb
    networks:
      - go-kafka

  mongodb:
    image: mongo
    container_name: mongo
    environment:
      - MONGO_INITDB_DATABASE=users
      - MONGO_INITDB_ROOT_USERNAME=admin
      - MONGO_INITDB_ROOT_PASSWORD=password
    ports:
      - 27017:27017
    volumes:
      - ./mongodb-data:/data/db
    networks:
      - go-kafka

  kibana:
    image: docker.elastic.co/kibana/kibana:7.5.2
    environment:
      SERVER_NAME: kibana.example.org
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    ports:
      - 5601:5601
    networks:
      - go-kafka

  # app:
  #   build: .
  #   container_name: app
  #   image: app
  #   environment:
  #     PG_URL: "postgres://user:pass@postgres:5432/postgres"
  #   ports:
  #     - 8080:8080
  #   depends_on:
  #     - postgres

  # integration:
  #   build:
  #     context: .
  #     dockerfile: integration-test/Dockerfile
  #   container_name: integration
  #   image: integration
  #   depends_on:
  #     - app
# volumes:
#   pg-data:

networks:
  go-kafka: