docker-compose.yml

version: '3'
services:
  spark-submit:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.5 --jars  https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/movies.yaml
      - HIVE_METASTORE_URI=hive:9083
      - USE_BUILTIN_HIVE_METASTORE=false
    volumes:
      - ./output/:/examples/output/
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  hive-tester:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.5 --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/movies_test.yaml
      - HIVE_METASTORE_URI=hive:9083
      - USE_BUILTIN_HIVE_METASTORE=false
    volumes:
      - ./output/:/examples/output/
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  spark-submit-manual-hive-sync:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.5 --jars  https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/manual_hive_sync_config.yaml
      - HIVE_METASTORE_URI=hive:9083
      - USE_BUILTIN_HIVE_METASTORE=false
    volumes:
      - ./output/:/examples/output/
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  hive-tester-manual-hive-sync:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.5 --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/manual_hive_sync_test.yaml
      - HIVE_METASTORE_URI=hive:9083
      - USE_BUILTIN_HIVE_METASTORE=false
    volumes:
      - ./output/:/examples/output/
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  spark-submit-manual-hive-sync-non-partition:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.5 --jars  https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/manual_hive_sync_no_partitions_config.yaml
      - HIVE_METASTORE_URI=hive:9083
      - USE_BUILTIN_HIVE_METASTORE=false
    volumes:
      - ./output/:/examples/output/
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  hive-tester-manual-hive-sync-no-partition:
    image: metorikku/metorikku
    environment:
      - SUBMIT_COMMAND=spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.5 --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/manual_hive_sync_no_partitions_test.yaml
      - HIVE_METASTORE_URI=hive:9083
      - USE_BUILTIN_HIVE_METASTORE=false
    volumes:
      - ./output/:/examples/output/
    entrypoint:
      - /scripts/entrypoint-submit.sh
    depends_on:
      - spark-master
      - spark-worker
  spark-master:
    image: metorikku/metorikku
    entrypoint:
      - /scripts/entrypoint-master.sh
    logging:
      driver: none
  spark-worker:
    image: metorikku/metorikku
    entrypoint:
      - /scripts/entrypoint-worker.sh
    volumes:
      - ./output/:/examples/output/
    logging:
      driver: none
  hive:
    image: metorikku/hive
    environment:
      - CONNECTION_URL=jdbc:mysql://hive-db:3306/hive?useSSL=false
      - CONNECTION_USER_NAME=root
      - CONNECTION_PASSWORD=pass
      - WAREHOUSE_DIR=file:///tmp
      - WAIT_HOSTS=hive-db:3306
    depends_on:
      - hive-db
    volumes:
      - ./output/:/examples/output/
  hive-db:
    image: mysql:5.7.25
    environment:
      - MYSQL_ROOT_PASSWORD=pass
      - MYSQL_DATABASE=hive