From a2222b3e833a8c05561de123e0305ae4a565388a Mon Sep 17 00:00:00 2001 From: Chris Cundill Date: Wed, 9 Oct 2024 09:12:23 +0100 Subject: [PATCH] Added required HOISTED_COLLECTION_DATASET_BUCKET_NAME env var --- dags/collection_generator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dags/collection_generator.py b/dags/collection_generator.py index 8016aef..473419d 100644 --- a/dags/collection_generator.py +++ b/dags/collection_generator.py @@ -122,7 +122,12 @@ def configure_dag(**kwargs): {"name": "COLLECTION_NAME", "value": collection}, { "name": "COLLECTION_DATASET_BUCKET_NAME", - "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"collection-dataset-bucket-name\") | string }}'"}, + "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"collection-dataset-bucket-name\") | string }}'" + }, + { + "name": "HOISTED_COLLECTION_DATASET_BUCKET_NAME", + "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"collection-dataset-bucket-name\") | string }}'" + }, # {"name": "TRANSFORMED_JOBS", "value": str('{{ task_instance.xcom_pull(task_ids="configure-dag", key="transformed-jobs") | string }}')}, {"name": "TRANSFORMED_JOBS", "value":"'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"transformed-jobs\") | string }}'"}, {"name": "DATASET_JOBS", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"dataset-jobs\") | string }}'"}