docker-compose.yml

version: '2'

services:
  spark-master:
    image: renien/spark-stand-alone
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    hostname: master
    container_name: spark-master
    environment:
      MASTER: spark://master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: localhost
      SPARK_DIVER_CORES: 1
      SPARK_DRIVER_MEMORY: 1g
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7077
      - 6066
#      - 4040
    ports:
#      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    volumes:
      - ./conf/master:/conf
      - ./data:/tmp/data

  spark-worker-1:
    image: renien/spark-stand-alone
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    container_name: spark-worker-1
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 1
      SPARK_WORKER_MEMORY: 2g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 8881
    ports:
      - 8081:8081
    volumes:
      - ./conf/worker:/conf
      - ./data:/tmp/data

  #spark-worker-2:
  #  image: gettyimages/spark
  #  command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
  #  container_name: spark-worker-2
  #  environment:
  #    SPARK_CONF_DIR: /conf
  #    SPARK_WORKER_CORES: 1
  #    SPARK_WORKER_MEMORY: 512mb
  #    SPARK_WORKER_PORT: 8882
  #    SPARK_WORKER_WEBUI_PORT: 8082
  #    SPARK_PUBLIC_DNS: localhost
  #  links:
  #    - spark-master
  #  expose:
  #    - 7012
  #    - 7013
  #    - 7014
  #    - 7015
  #    - 8881
  #  ports:
  #    - 8082:8082
  #  volumes:
  #    - ./conf/worker:/conf
  #    - ./data:/tmp/data
  livy:
    image: renien/spark-stand-alone-livy
    container_name: livy
    environment:
      - SPARK_CONF_DIR=/conf
      - SPARK_DIVER_CORES=1
      - SPARK_DRIVER_MEMORY=1g
      - SPARK_MASTER_ENDPOINT=master
      - SPARK_MASTER_PORT=7077
      - LIVY_FILE_LOCAL_DIR_WHITELIST=/opt/jars
    ports:
      - 8998:8998
      - 4040:4040
    volumes:
      - $PWD/jars:/opt/jars
      - $PWD/conf/livy/livy.conf.extra:/usr/local/bin/livy.conf.extra
    depends_on:
      - "spark-master"
      - "spark-worker-1"