docker-compose.yaml

version: '2'
services:
  namenode:
    image: hadoop-3:latest
    container_name: namenode
    networks:
      - hadoop
    ports:
      - 9870:9870  # hdfs namenode ui
    command: ./entrypoint.sh namenode
    restart: unless-stopped
    volumes:
      - ./hadoop/etc/hadoop/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml
      - ./hadoop/entrypoint.sh:/opt/hadoop/entrypoint.sh
 
  datanode:
    image: hadoop-3:latest
    container_name: datanode
    depends_on: 
      - namenode
    networks:
      - hadoop
    command: ./entrypoint.sh datanode
    restart: unless-stopped
    volumes:
      - ./hadoop/etc/hadoop/hdfs-site.xml:/templates/hdfs-site.xml
      - ./hadoop/entrypoint.sh:/opt/hadoop/entrypoint.sh

  resourcemanager:
    image: hadoop-3:latest
    container_name: resourcemanager
    depends_on: 
      - datanode
      - namenode
    ports:
      - 8088:8088 # yarn resourcemanager ui
    networks:
      - hadoop
    command:  ./entrypoint.sh resourcemanager
    restart: unless-stopped
    volumes:
      - ./hadoop/etc/hadoop/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./hadoop/entrypoint.sh:/opt/hadoop/entrypoint.sh

  nodemanager:
    image: hadoop-3:latest
    container_name: nodemanager
    depends_on: 
      - resourcemanager
    networks:
      - hadoop
    # ports:
    #   - 8042:8042 # yarn nodemanager ui
    command:  ./entrypoint.sh nodemanager
    restart: unless-stopped
    volumes:
      - ./hadoop/etc/hadoop/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./hadoop/entrypoint.sh:/opt/hadoop/entrypoint.sh

  spark:
    image: hadoop-3:latest
    container_name: spark
    depends_on: 
      - resourcemanager
      - namenode
      - datanode
    ports: # only if use SPARK own resource manager
      - 8081:8081 # spark worker ui
      - 8080:8080 # spark master ui
      - 8998:8998 # livy localhost:8998/ui
    networks:
      - hadoop
    command:  ./entrypoint.sh spark rdd/count_lines.py
    restart: unless-stopped
    volumes:
      - ./data/world-cups/:/data/
      - ./hadoop/etc/hadoop/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml
      - ./hadoop/livy/livy.conf:/opt/livy/conf/livy.conf
      - ./spark/conf/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
      - ./pyspark-examples/:/pyspark-examples/
      - ./hadoop/entrypoint.sh:/opt/hadoop/entrypoint.sh


  solr:
    image: solr:8-slim
    ports:
      - 8983:8983
    networks:
      - hadoop

  hive:
    image: hadoop-3:latest
    container_name: hive
    depends_on: 
      - resourcemanager
      - namenode
      - datanode
    ports:
      - 10002:10002
    networks:
      - hadoop
    command: ./entrypoint.sh hive
    volumes:
      - ./hadoop/entrypoint.sh:/opt/hadoop/entrypoint.sh
 
 #################################################
 # hue section below
 #################################################
 # if fail, enter in cointainer and run -> ./build/env/bin/hue migrate

  hue: # see https://github.com/cloudera/hue/tree/master/tools/docker
    image: gethue/hue:4.4.0
    container_name: hue
    depends_on:
      - hive
    ports:
      - 8888:8888
    networks:
      - hadoop
    restart: unless-stopped
    depends_on:
      - huedb
    volumes:
      - ./hue/hue.ini:/usr/share/hue/desktop/conf/hue.ini
      - ./hue/entrypoint.sh:/usr/share/hue/entrypoint.sh
      - ./volumes/hue:/volumes/hue

  huedb:
    image: mysql:8
    command: --default-authentication-plugin=mysql_native_password
    ports:
      - 3306:3306
    networks:
      - hadoop
    restart: unless-stopped
    environment:
      MYSQL_ROOT_PASSWORD: pass
      MYSQL_DATABASE: hue
    volumes:
      - huedb-datavolume:/var/lib/mysql

networks:
  hadoop:


volumes:
  huedb-datavolume: