docker-compose.yaml

version: "3.3"

networks:
  cogniplant:
    driver: overlay

services:
  
  zookeeper: 
    hostname: zookeeper
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_MY_ID: 1
      ZOOKEEPER_CLIENT_PORT: ${ZOOKEEPER_CLIENT_PORT}
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVERS: ${ZOOKEEPER_SERVERS}
    volumes:
      - ./cogniplant/zookeeper/data:/var/lib/zookeeper/data
      - ./cogniplant/zookeeper/log:/var/lib/zookeeper/log
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.30'
          memory: 4024M
    networks:
      - cogniplant

  kafka:
    image: confluentinc/cp-kafka:latest
    ports:
      - 19092:19092
      - 9092:9092
    depends_on:
      - zookeeper
    environment:
      KAFKA_LISTENERS: 'INTERNAL://:${KAFKA_INTERNAL},EXTERNAL://:${KAFKA_EXTERNAL}'
      KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka:${KAFKA_INTERNAL},EXTERNAL://${HOST}:${KAFKA_EXTERNAL}'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" #later use SSL
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:${ZOOKEEPER_CLIENT_PORT}'
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    volumes: 
      - ./cogniplant/kafka/data:/var/lib/kafka/data
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.30'
          memory: 4096M
    networks:
      - cogniplant
  
  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:latest
    depends_on:
      - zookeeper
      - kafka
      - schema-registry
    ports:
      - 8022:8022
    environment:
      KAFKA_REST_LISTENERS: 'http://127.0.0.1:8022'
      KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:${SCHEMA_REGISTRY_PORT}'
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka:${KAFKA_INTERNAL}'
    volumes:
      - ./cogniplant/kafka-rest-proxy/log:/datalog
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.10'
          memory: 1096M
    networks:
      - cogniplant

  schema-registry:
    image: confluentinc/cp-schema-registry:latest
    hostname: schema-registry
    ports:
      - 8081:8081
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:${KAFKA_INTERNAL}'
    volumes:
      - ./cogniplant/schema-registry/data:/data
      - ./cogniplant/schema-registry/log:/datalog
    depends_on:
      - zookeeper
      - kafka
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.10'
          memory: 512M
    networks:
      - cogniplant

  kafka-connect:
    image: confluentinc/cp-kafka-connect:latest
    hostname: kafka-connect
    ports:
      - "8083:8083"
    environment:
      CONNECT_REST_PORT: ${CONNECT_REST_PORT}
      CONNECT_BOOTSTRAP_SERVERS: 'kafka:${KAFKA_INTERNAL}'
      # Required. A unique string that identifies the Connect cluster group this worker belongs to.
      CONNECT_GROUP_ID: compose-connect-group
      # Connect will actually use Kafka topics as a datastore for configuration and other data. #meta
      # Required. The name of the topic where connector and task configuration data are stored.
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      # Required. The name of the topic where connector and task configuration offsets are stored.
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      # Required. The name of the topic where connector and task configuration status updates are stored.
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      # Required. Converter class for key Connect data. This controls the format of the
      # data that will be written to Kafka for source connectors or read from Kafka for sink connectors.
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      # Allows connect to leverage the power of schema registry. Here we define it for key schemas.
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:${SCHEMA_REGISTRY_PORT}'
      # Required. Converter class for value Connect data. This controls the format of the
      # data that will be written to Kafka for source connectors or read from Kafka for sink connectors.
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      # Allows connect to leverage the power of schema registry. Here we define it for value schemas.
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:${SCHEMA_REGISTRY_PORT}'
      # Required. Converter class for internal key Connect data that implements the Converter interface.
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      # Required. Converter class for offset value Connect data that implements the Converter interface.
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      # Required. The hostname that will be given out to other workers to connect to.
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      # The next three are required when running in a single-node cluster, as we are.
      # We would be able to take the default (of 3) if we had three or more nodes in the cluster.
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars'
    volumes:
      - ./cogniplant/kafka-connect/connectors:/etc/kafka-connect/jars/
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.50'
          memory: 3048M
    networks:
      - cogniplant
    depends_on:
      - zookeeper
      - kafka

  kafka-connect-ui:
    image: landoop/kafka-connect-ui
    hostname: kafka-connect-ui
    ports:
      - "${CONNECT_UI_PORT}:9004"
    environment:
      CONNECT_URL: 'http://kafka-connect:${CONNECT_REST_PORT}/'
      PORT: 9004
    deploy:
      placement:
        constraints:
          - node.labels.role==master
    depends_on:
      - kafka-connect
    networks:
      - cogniplant
  
  kafdrop:
    image: obsidiandynamics/kafdrop
    ports:
      - '${KAFDROP_UI_PORT}:9000'
    environment:
      KAFKA_BROKERCONNECT: 'kafka:${KAFKA_INTERNAL}'
      JVM_OPTS: "-Xms16M -Xmx48M"
    deploy:
      placement:
        constraints:
          - node.labels.role==master
    depends_on:
      - kafka
    networks:
      - cogniplant

  # Create a service called spark-master for stream processing
  spark-master-stream:
    image: "bde2020/spark-master:latest"
    ports:
      - '${SPARK_MASTER_STREAM_UI_PORT}:8080'
      - "7078:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark
    volumes:
      - ./cogniplant/spark/applications:/opt/spark-apps
      - ./cogniplant/spark/data:/opt/spark-data
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.2'
          memory: 2096M
      replicas: 1
    networks:
      - cogniplant
  

      #SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
  # Create a spark-worker for stream processing
  spark-worker-stream:
    image: "bde2020/spark-worker:latest"
    ports:
      - "18081:8081"
    environment:
      - SPARK_MASTER=spark://spark-master-stream:7077
      - SPARK_WORKER_CORES=${SPARK_WORKER_STREAM_CORES}
      - SPARK_WORKER_MEMORY=${SPARK_WORKER_STREAM_MEMORY}
    volumes:
      - ./cogniplant/spark/applications:/opt/spark-apps
      - ./cogniplant/spark/data:/opt/spark-data
    depends_on:
      - spark-master-stream
    deploy:
      resources:
        limits:
          cpus: '0.2'
          memory: 2096M
      placement:
        constraints:
          - node.labels.role==master
      replicas: 1
    networks:
      - cogniplant
    
  #Create a service called spark-master for batch processing
  spark-master-batch:
    image: "bde2020/spark-master:latest"
    ports:
      - "${SPARK_MASTER_BATCH_UI_PORT}:8080"
      - "7079:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark
    volumes:
      - ./cogniplant/spark/applications:/opt/spark-apps
      - ./cogniplant/spark/data:/opt/spark-data
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.2'
          memory: 2096M
      replicas: 1
    networks:
      - cogniplant
  
  # Create a spark-worker for batch processing
  spark-worker-batch:
    image: "bde2020/spark-worker:latest"
    ports:
      - "28081:28081"
    environment:
      - SPARK_MASTER=spark://spark-master-batch:7077
      - SPARK_WORKER_CORES=${SPARK_WORKER_BATCH_CORES}
      - SPARK_WORKER_MEMORY=${SPARK_WORKER_BATCH_MEMORY}
    volumes:
      - ./cogniplant/spark/applications:/opt/spark-apps
      - ./cogniplant/spark/data:/opt/spark-data
    depends_on:
      - spark-master-batch
    deploy:
      resources:
        limits:
          cpus: '0.2'
          memory: 2096M
      placement:
        constraints:
          - node.labels.role==master
      replicas: 1
    networks:
      - cogniplant

  influxdb:
    image: influxdb:latest
    ports:
      - '8086:8086'
    volumes:
      - ./cogniplant/influxdb/data:/var/lib/influxdb
    environment:
      - INFLUXDB_DB=${INFLUXDB_DB_NAME}
      - INFLUXDB_ADMIN_USER=${INFLUXDB_ADMIN_USER}
      - INFLUXDB_ADMIN_PASSWORD=${INFLUXDB_ADMIN_PASSWORD}
    deploy:
      resources:
        limits:
          cpus: '0.3'
          memory: 3096M
      placement:
        constraints:
          - node.labels.role==master
      replicas: 1
    networks:
      - cogniplant
  
  chronograf:
    image: chronograf
    ports:
      - '${CHRONOGRAF_UI_PORT}:8888'
    depends_on:
      - influxdb
    environment:
      - INFLUXDB_URL=${INFLUXDB_URL}
      - INFLUXDB_USERNAME=${CHRONOGRAF_USERNAME}
      - INFLUXDB_PASSWORD=${CHRONOGRAF_PWD}
    volumes:  
      - ./cogniplant/chronograf/data:/var/lib/chronograf
    deploy:
      resources:
        limits:
          cpus: '0.1'
          memory: 1096M
      placement:
        constraints:
          - node.labels.role==master
    networks: 
      - cogniplant

  grafana:
    image: grafana/grafana:latest
    ports:
      - '${GRAFANA_UI_PORT}:3000'
    volumes:
      - ./cogniplant/grafana/data:/var/lib/grafana
      - ./cogniplant/grafana/log:/var/log/grafana
      - ./cogniplant/grafana/provisioning:/etc/grafana/provisioning
    depends_on:
      - influxdb
    environment:
      - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME}
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PWD}
    deploy:
      resources:
        limits:
          cpus: '0.1'
          memory: 1096M
      placement:
        constraints:
          - node.labels.role==master
    networks:
      - cogniplant

  portainer:
    image: portainer/portainer:latest
    hostname: portainer
    command: -H unix:///var/run/docker.sock
    ports:
      - '${PORTAINER_UI_PORT}:9000'
      - 8000:8000
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./cogniplant/portainer/data:/data
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.10'
          memory: 1024M
    networks:
      - cogniplant

  schema-registry-ui:
    image: landoop/schema-registry-ui:latest
    hostname: schema-registry-ui
    ports:
      - '${SCHEMA_REGISTRY_UI_PORT}:8000'
    environment:
      SCHEMAREGISTRY_URL: http://schema-registry:${SCHEMA_REGISTRY_PORT}/
      PROXY: "true"
    volumes:
      - ./cogniplant/schema-registry-ui/data:/data
    depends_on:
      - schema-registry
    deploy:
      placement:
        constraints:
          - node.labels.role==master
      resources:
        limits:
          cpus: '0.10'
          memory: 1024M
    networks:
      - cogniplant    

  mlflow_db:
    image: postgres
    hostname: mlflow_db
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PWD}
    ports:
      - 5432:5432
    volumes:
      - ./cogniplant/postgres/data:/var/lib/postgres
    deploy:
      resources:
        limits:
          cpus: '0.1'
          memory: 1024M
      placement:
        constraints:
          - node.labels.role==master
    networks:
      - cogniplant