docker-compose.yml

version: "3.3"
services:
  spark-master:
    image: cluster-apache-spark:3.0.2
    hostname: spark-master
    container_name: spark-master
    ports:
      - "8090:8080"
      - "8077:7077"
    volumes:
       - ./apps:/opt/spark-apps
       - ./data:/opt/spark-data
    environment:
      - SPARK_LOCAL_IP=spark-master
      - SPARK_WORKLOAD=master

  spark-worker-a:
    image: cluster-apache-spark:3.0.2
    hostname: spark-worker-a
    container_name: spark-worker-a
    ports:
      - "8091:8080"
      - "7000:7000"
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=2G
      - SPARK_DRIVER_MEMORY=1G
      - SPARK_EXECUTOR_MEMORY=1G
      - SPARK_WORKLOAD=worker
      - SPARK_LOCAL_IP=spark-worker-a

    # volumes:
    #    - ./apps:/opt/spark-apps
    #    - ./data:/opt/spark-data
  spark-worker-b:
    image: cluster-apache-spark:3.0.2
    hostname: spark-worker-b
    container_name: spark-worker-b
    ports:
      - "8092:8080"
      - "7001:7000"
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=2G
      - SPARK_DRIVER_MEMORY=1G
      - SPARK_EXECUTOR_MEMORY=1G
      - SPARK_WORKLOAD=worker
      - SPARK_LOCAL_IP=spark-worker-b

    # volumes:
    #     - ./apps:/opt/spark-apps
    #     - ./data:/opt/spark-data