docker-compose.yml

version: '2.1'
services:
  spark-master:
    image: dhruvk/spark-master-livy:2.4
    container_name: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
      - "8998:8998"
    tty: true
    environment:
      - INIT_DAEMON_STEP=setup_spark
      - "constraint:node==<yourmasternode>"

  spark-worker-1:
    image: bde2020/spark-worker:2.4.4-hadoop2.7
    container_name: spark-worker-1
    depends_on:
      - spark-master
    ports:
      - "8081:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "constraint:node==<yourworkernode>"

  spark-worker-2:
    image: bde2020/spark-worker:2.4.4-hadoop2.7
    container_name: spark-worker-2
    depends_on:
      - spark-master
    ports:
      - "8082:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      - "constraint:node==<yourworkernode>" 

  postgres:
    container_name: "postgres"
    image: postgres:9.6
    volumes:
      - ${HOME}/postgres_mount/postgres:/var/lib/postgresql/data        
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow

  webserver:
    container_name: "webserver"
    image: my-airflow-presentation-webserver:latest
    restart: always
    depends_on:
      - postgres
    environment:
      - EXECUTOR=Local
    volumes:
      - ${HOME}/airflow_mount/dags:/usr/local/airflow/dags
      - ${HOME}/airflow_mount/logs:/usr/local/airflow/logs
    ports:
      - "9001:8080"
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3

  scheduler:
    container_name: "scheduler"
    image: my-airflow-presentation-scheduler:latest
    restart: always
    depends_on:
      - webserver
    environment:
      - EXECUTOR=Local
    volumes:
      - ${HOME}/airflow_mount/dags:/usr/local/airflow/dags
      - ${HOME}/airflow_mount/logs:/usr/local/airflow/logs