From 5dc3fa870431471040752356174e62dfee1c22a0 Mon Sep 17 00:00:00 2001 From: digital-land-bot Date: Fri, 14 Jun 2024 00:04:16 +0000 Subject: [PATCH] Updated makerules 2024-06-14 --- makerules/collection.mk | 4 ++-- makerules/pipeline.mk | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/makerules/collection.mk b/makerules/collection.mk index 38e7254..cb9f312 100644 --- a/makerules/collection.mk +++ b/makerules/collection.mk @@ -52,10 +52,10 @@ second-pass:: collection collect:: $(COLLECTION_CONFIG_FILES) @mkdir -p $(RESOURCE_DIR) - digital-land collect $(ENDPOINT_CSV) + digital-land ${DIGITAL_LAND_OPTS} collect $(ENDPOINT_CSV) collection:: - digital-land collection-save-csv + digital-land ${DIGITAL_LAND_OPTS} collection-save-csv clobber-today:: rm -rf $(LOG_FILES_TODAY) $(COLLECTION_INDEX) diff --git a/makerules/pipeline.mk b/makerules/pipeline.mk index 7fdc689..401cd7d 100644 --- a/makerules/pipeline.mk +++ b/makerules/pipeline.mk @@ -89,26 +89,26 @@ endif define run-pipeline mkdir -p $(@D) $(ISSUE_DIR)$(notdir $(@D)) $(COLUMN_FIELD_DIR)$(notdir $(@D)) $(DATASET_RESOURCE_DIR)$(notdir $(@D)) - digital-land --dataset $(notdir $(@D)) $(DIGITAL_LAND_FLAGS) pipeline $(1) --issue-dir $(ISSUE_DIR)$(notdir $(@D)) --column-field-dir $(COLUMN_FIELD_DIR)$(notdir $(@D)) --dataset-resource-dir $(DATASET_RESOURCE_DIR)$(notdir $(@D)) $(PIPELINE_FLAGS) $< $@ + digital-land ${DIGITAL_LAND_OPTS} --dataset $(notdir $(@D)) $(DIGITAL_LAND_FLAGS) pipeline $(1) --issue-dir $(ISSUE_DIR)$(notdir $(@D)) --column-field-dir $(COLUMN_FIELD_DIR)$(notdir $(@D)) --dataset-resource-dir $(DATASET_RESOURCE_DIR)$(notdir $(@D)) $(PIPELINE_FLAGS) $< $@ endef define build-dataset = mkdir -p $(@D) - time digital-land --dataset $(notdir $(basename $@)) dataset-create --output-path $(basename $@).sqlite3 $(^) + time digital-land ${DIGITAL_LAND_OPTS} --dataset $(notdir $(basename $@)) dataset-create --output-path $(basename $@).sqlite3 $(^) time datasette inspect $(basename $@).sqlite3 --inspect-file=$(basename $@).sqlite3.json - time digital-land --dataset $(notdir $(basename $@)) dataset-entries $(basename $@).sqlite3 $@ + time digital-land ${DIGITAL_LAND_OPTS} --dataset $(notdir $(basename $@)) dataset-entries $(basename $@).sqlite3 $@ mkdir -p $(FLATTENED_DIR) - time digital-land --dataset $(notdir $(basename $@)) dataset-entries-flattened $@ $(FLATTENED_DIR) + time digital-land ${DIGITAL_LAND_OPTS} --dataset $(notdir $(basename $@)) dataset-entries-flattened $@ $(FLATTENED_DIR) md5sum $@ $(basename $@).sqlite3 csvstack $(ISSUE_DIR)$(notdir $(basename $@))/*.csv > $(basename $@)-issue.csv mkdir -p $(EXPECTATION_DIR) - time digital-land expectations-dataset-checkpoint --output-dir=$(EXPECTATION_DIR) --specification-dir=specification --data-path=$(basename $@).sqlite3 + time digital-land ${DIGITAL_LAND_OPTS} expectations-dataset-checkpoint --output-dir=$(EXPECTATION_DIR) --specification-dir=specification --data-path=$(basename $@).sqlite3 csvstack $(EXPECTATION_DIR)/**/$(notdir $(basename $@))-results.csv > $(basename $@)-expectation-result.csv csvstack $(EXPECTATION_DIR)/**/$(notdir $(basename $@))-issues.csv > $(basename $@)-expectation-issue.csv endef collection:: - digital-land collection-pipeline-makerules > collection/pipeline.mk + digital-land ${DIGITAL_LAND_OPTS} collection-pipeline-makerules > collection/pipeline.mk -include collection/pipeline.mk @@ -168,7 +168,7 @@ save-expectations:: # .. this assumes conversion is the same for every dataset, but it may not be soon var/converted/%.csv: collection/resource/% mkdir -p var/converted/ - digital-land convert $< + digital-land ${DIGITAL_LAND_OPTS} convert $< transformed:: @mkdir -p $(TRANSFORMED_DIR)