cloudbuild.yaml

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# This is a config file used by Cloud Build and the Cloud Build local runner to 
# run a CI pipeline to check if any changes done pass tests.


steps:

- name: 'docker/compose'
  id: 'Launch HAPI Source and Sink Servers for Jdbc Batch Mode Test'
  args: ['-f', './docker/hapi-compose.yml', '-f', './docker/sink-compose.yml', 'up', '--force-recreate', '--remove-orphan', '-d']

# Note license checking will fail on Cloud Build because of .git dependencies
# and there is not much point updating license headers at this stage anyway.
# Ditto for Spotless (fails because of NPM dep).
- name: 'adoptopenjdk/maven-openjdk11'
  id: 'Compile Bunsen and Pipeline'
  entrypoint: /bin/bash
  args:
  - -c
  - mvn --no-transfer-progress -e install -Dlicense.skip=true -Dspotless.apply.skip=true
  waitFor: ['-'] 

- name: 'gcr.io/cloud-builders/docker'
  id: 'Build Uploader Image'
  entrypoint: /bin/bash
  args:
  - -c
  - cd synthea-hiv/uploader; docker build -t ${_REPOSITORY}/synthea-uploader:${_TAG} .

- name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
  id: 'Run Uploader Unit Tests'
  entrypoint: /bin/bash
  args:
  - -c
  - cd /uploader; python -m unittest discover  -p '*_test.py' 

- name: 'gcr.io/cloud-builders/docker'
  id: 'Build E2E Image'
  entrypoint: /bin/bash
  args:
  - -c
  - cd e2e-tests; docker build -t ${_REPOSITORY}/e2e-tests:${_TAG} .

- name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
  id: 'Upload to HAPI'
  env:
    - INPUT_DIR=/workspace/synthea-hiv/sample_data
    - SINK_TYPE=HAPI
    - FHIR_ENDPOINT=http://hapi-server:8080/fhir
    - CORES=--cores 8

- name: 'gcr.io/cloud-builders/docker'
  id: 'Build Pipeline Images'
  entrypoint: /bin/bash
  args:
  - -c
  - cd pipelines/batch; 
    docker build -t ${_REPOSITORY}/batch-pipeline:${_TAG} .;
    cd ../streaming-binlog; 
    docker build -t ${_REPOSITORY}/streaming-pipeline:${_TAG} .;

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline JDBC with HAPI source'
  env:
    - JDBC_MODE_ENABLED=true
    - JDBC_MODE_HAPI=true
    - FHIR_SERVER_URL=http://hapi-server:8080/fhir
    - SINK_PATH=http://sink-server:8080/fhir
    - SINK_USERNAME=hapi
    - SINK_PASSWORD=hapi
    - FHIR_DATABASE_CONFIG_PATH=/workspace/utils/hapi-postgres-config.json
    - PARQUET_PATH=/workspace/e2e-tests/JDBC_HAPI
    - JDBC_FETCH_SIZE=1000

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test JDBC with HAPI source'
  env:
    - PARQUET_SUBDIR=JDBC_HAPI
    - DOCKER_NETWORK=--use_docker_network
    - HAPI_TEST=--hapi

- name: 'docker/compose'
  id: 'Turn down HAPI Source and Sink Servers'
  args: ['-f', './docker/hapi-compose.yml', '-f', './docker/sink-compose.yml', 'down']

- name: 'docker/compose'
  id: 'Launch OpenMRS Server and HAPI Sink Server'
  args: ['-f', './docker/openmrs-compose.yaml', '-f', './docker/sink-compose.yml', 'up', '--force-recreate', '--remove-orphan', '-d']

- name: 'gcr.io/cloud-builders/docker'
  id: 'Wait for Servers Start'
  entrypoint: /bin/bash
  args:
  - -c
  - e2e-tests/wait_for_start.sh --use_docker_network

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Launch Streaming Pipeline'
  entrypoint: /bin/bash
  args: 
    - -c
    -  e2e-tests/wait_for_streaming.sh --use_docker_network

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for STREAMING'
  env:
    - PARQUET_SUBDIR=STREAMING
    - DOCKER_NETWORK=--use_docker_network
    - STREAMING_TEST=--streaming

- name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
  id: 'Upload to OpenMRS'
  env:
    - CONVERT=--convert_to_openmrs
    - INPUT_DIR=/workspace/synthea-hiv/sample_data
    - SINK_TYPE=OpenMRS
    - FHIR_ENDPOINT=http://openmrs:8080/openmrs/ws/fhir2/R4

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline NON_JDBC'
  env:
    - PARQUET_PATH=/workspace/e2e-tests/NON_JDBC
    - SINK_PATH=http://sink-server:8080/fhir
    - SINK_USERNAME=hapi
    - SINK_PASSWORD=hapi

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for NON_JDBC'
  env:
    - PARQUET_SUBDIR=NON_JDBC
    - DOCKER_NETWORK=--use_docker_network

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline JDBC with OpenMRS source'
  env:
    - JDBC_MODE_ENABLED=true
    - PARQUET_PATH=/workspace/e2e-tests/JDBC_OPENMRS
    - SINK_PATH=http://sink-server:8080/fhir
    - SINK_USERNAME=hapi
    - SINK_PASSWORD=hapi
    - FHIR_DATABASE_CONFIG_PATH=/workspace/utils/dbz_event_to_fhir_config.json

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test JDBC with OpenMRS source'
  env:
    - PARQUET_SUBDIR=JDBC_OPENMRS
    - DOCKER_NETWORK=--use_docker_network

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Test Indicators'
  entrypoint: /bin/bash
  args:
  - -c
  - 'cd dwh; ./validate_indicators.sh'

- name: 'docker/compose'
  id: 'Turn down Webserver and HAPI Server'
  args: ['-f', './docker/openmrs-compose.yaml', '-f', './docker/sink-compose.yml', 'down']

substitutions:
  # To use substitutions in your local build, use the flag --substitutions 
  # along with the key=value pair that you want to substitute
  # More details here: https://cloud.google.com/build/docs/build-debug-locally
  _TAG: local # Cloud Build replaces this with the Commit SHA
  _REPOSITORY: fhir-analytics # Cloud Build replaces this with 
                              # us-docker.pkg.dev/${PROJECT_ID}/fhir-analytics

images: 
  # If run locally, images are available on your local machine through Docker
  # You can then re-tag images and push to your own Docker repo
  - '${_REPOSITORY}/streaming-pipeline:${_TAG}'
  - '${_REPOSITORY}/batch-pipeline:${_TAG}'
  - '${_REPOSITORY}/e2e-tests:${_TAG}'
  - '${_REPOSITORY}/synthea-uploader:${_TAG}'
logsBucket: "gs://cloud-build-gh-logs"
timeout: '2h'
options:
  machineType: 'N1_HIGHCPU_32'