From 26c070185a5020df4d79114eba7b611f81798a87 Mon Sep 17 00:00:00 2001 From: digital-land-bot Date: Sat, 5 Oct 2024 00:04:49 +0000 Subject: [PATCH] Updated makerules 2024-10-05 --- makerules/pipeline.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/makerules/pipeline.mk b/makerules/pipeline.mk index 2cfa207..bee4489 100644 --- a/makerules/pipeline.mk +++ b/makerules/pipeline.mk @@ -107,7 +107,6 @@ PIPELINE_CONFIG_FILES=\ endif define run-pipeline - mkdir -p $(@D) $(ISSUE_DIR)$(notdir $(@D)) $(OPERATIONAL_ISSUE_DIR) $(COLUMN_FIELD_DIR)$(notdir $(@D)) $(DATASET_RESOURCE_DIR)$(notdir $(@D)) $(CONVERTED_RESOURCE_DIR)$(notdir $(@D)) digital-land ${DIGITAL_LAND_OPTS} --dataset $(notdir $(@D)) --pipeline-dir $(PIPELINE_DIR) $(DIGITAL_LAND_FLAGS) pipeline $(1) --issue-dir $(ISSUE_DIR)$(notdir $(@D)) --column-field-dir $(COLUMN_FIELD_DIR)$(notdir $(@D)) --dataset-resource-dir $(DATASET_RESOURCE_DIR)$(notdir $(@D)) --converted-resource-dir $(CONVERTED_RESOURCE_DIR)$(notdir $(@D)) --config-path $(CACHE_DIR)config.sqlite3 --organisation-path $(CACHE_DIR)organisation.csv $(PIPELINE_FLAGS) $< $@ endef @@ -178,6 +177,7 @@ init:: $(CACHE_DIR)organisation.csv echo "File not found at $$url"; \ fi; \ done + mkdir -p $(@D) $(ISSUE_DIR)$(notdir $(@D)) $(OPERATIONAL_ISSUE_DIR) $(COLUMN_FIELD_DIR)$(notdir $(@D)) $(DATASET_RESOURCE_DIR)$(notdir $(@D)) $(CONVERTED_RESOURCE_DIR)$(notdir $(@D)) makerules:: curl -qfsL '$(MAKERULES_URL)pipeline.mk' > makerules/pipeline.mk @@ -190,6 +190,7 @@ save-transformed:: aws s3 sync $(CONVERTED_RESOURCE_DIR) s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(REPOSITORY)/$(CONVERTED_RESOURCE_DIR) --no-progress save-dataset:: + @mkdir -p $(DATASET_DIR) aws s3 sync $(DATASET_DIR) s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(REPOSITORY)/$(DATASET_DIR) --no-progress @mkdir -p $(FLATTENED_DIR) ifeq ($(HOISTED_COLLECTION_DATASET_BUCKET_NAME),digital-land-$(ENVIRONMENT)-collection-dataset-hoisted) @@ -203,6 +204,7 @@ save-expectations:: aws s3 sync $(EXPECTATION_DIR) s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(EXPECTATION_DIR) --exclude "*" --include "*.csv" --no-progress save-performance:: + @mkdir -p $(PERFORMANCE_DIR) aws s3 sync $(PERFORMANCE_DIR) s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(PERFORMANCE_DIR) --no-progress # convert an individual resource