docker-compose.yml

# https://itnext.io/creating-a-big-data-development-platform-using-docker-compose-892f7f4da738

version: '3' 
services:

  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
    container_name: namenode
    volumes:
      - /tmp/hdfs/namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop.env
    ports:
      - "50070:50070"
    networks:
      net_pet:
        ipv4_address: 172.27.1.5
  
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
    container_name: datanode
    volumes:
      - /tmp/hdfs/datanode:/hadoop/dfs/data
      - ./bank:/bank
    env_file:
      - ./hadoop.env
    environment:
      SERVICE_PRECONDITION: "namenode:50070"
    depends_on:
      - namenode
    ports:
      - "50075:50075"
    networks:
      net_pet:
        ipv4_address: 172.27.1.6

  spark-master:
    image: bde2020/spark-master:2.4.0-hadoop2.7
    container_name: spark-master
    ports:
      - '8080:8080'
      - '7077:7077'
    volumes:
      - ./jars_dir:/opt/bitnami/spark/ivy:z
    networks:
      net_pet:
        ipv4_address: 172.27.1.10
     
  spark-worker_1:
    image: bde2020/spark-worker:2.4.0-hadoop2.7
    container_name: spark-worker-1
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore
    depends_on:
      - spark-master
    ports:
      - 8081:8081
    env_file:
      - ./hadoop.env
    networks:
      net_pet:
        ipv4_address: 172.27.1.11

  zookeeper:
    image: wurstmeister/zookeeper:3.4.6
    ports:
      - "2181:2181"
    networks:
      net_pet:
        ipv4_address: 172.27.1.15

  kafka:
    image: wurstmeister/kafka:2.12-2.3.0
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 172.27.1.16
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    networks:
      net_pet:
        ipv4_address: 172.27.1.16 

  mongo:
    image: mongo:4.4
    container_name: mongo-db
    restart: unless-stopped
    ports:
      - 26000:27017
    volumes:
      - ./db/mongo/data:/data/db/
      - ./log/mongo/logs:/var/log/mongodb/
      - ./initdb.d:/docker-entrypoint-initdb.d/
    networks:
      net_pet:
        ipv4_address: 172.27.1.17



networks:
  net_pet:
    ipam:
      driver: default
      config:
        - subnet: 172.27.0.0/16