Split collection and package building

Use a separate job to  build the package (organisation.csv file) after the collection.

.github/workflows/build_package.yml

@@ -0,0 +1,109 @@
+name: Run collection
+on:
+  workflow_call:
+    secrets:
+      DLB_BOT_EMAIL:
+        required: true
+      DLB_BOT_TOKEN:
+        required: true
+      DLB_BOT_USERNAME:
+        required: true
+      AWS_S3_ACCESS_KEY_ID:
+        required: true
+      AWS_S3_SECRET_ACCESS_KEY:
+        required: true
+env:
+  DLB_BOT_EMAIL: ${{ secrets.DLB_BOT_EMAIL }}
+  DLB_BOT_TOKEN: ${{ secrets.DLB_BOT_TOKEN }}
+  DLB_BOT_USERNAME: ${{ secrets.DLB_BOT_USERNAME }}
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Free up disk space
+      run: |
+        df -h
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/ghc
+        echo
+        df -h
+
+    - uses: actions/checkout@v3
+      with:
+        lfs: true
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+
+    - name: Configure git
+      run: |
+        git config user.email "${DLB_BOT_EMAIL}"
+        git config user.name "${DLB_BOT_USERNAME}"
+        git remote set-url origin https://${DLB_BOT_USERNAME}:${DLB_BOT_TOKEN}@github.com/${GITHUB_REPOSITORY}.git
+        git checkout ${GITHUB_REF_NAME}
+
+    - name: Update makerules
+      run: make makerules
+
+    - name: Install dependencies
+      run: make init
+
+    - name: Create the package
+      run: make package
+
+    - name: Configure Development AWS Credentials
+      if: always()
+      uses: aws-actions/configure-aws-credentials@v1-node16
+      with:
+        aws-access-key-id: ${{secrets.DEVELOPMENT_AWS_ACCESS_KEY_ID}}
+        aws-secret-access-key: ${{secrets.DEVELOPMENT_AWS_ACCESS_SECRET}}
+        aws-region: eu-west-2
+
+    - name: Save datasets to Development S3
+      env:
+        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.DEVELOPMENT_DATA_S3_BUCKET}}
+        HOISTED_COLLECTION_DATASET_BUCKET_NAME: ${{secrets.DEVELOPMENT_DATA_S3_BUCKET}}
+      run: make save-dataset
+
+    - name: Save expectations to Development S3
+      if: always()
+      env:
+        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.DEVELOPMENT_DATA_S3_BUCKET}}
+      run: make save-expectations
+
+    - name: Configure Staging AWS Credentials
+      if: always()
+      uses: aws-actions/configure-aws-credentials@v1-node16
+      with:
+        aws-access-key-id: ${{secrets.STAGING_AWS_ACCESS_KEY_ID}}
+        aws-secret-access-key: ${{secrets.STAGING_AWS_ACCESS_SECRET}}
+        aws-region: eu-west-2
+
+    - name: Save datasets to Staging S3
+      env:
+        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.STAGING_DATA_S3_BUCKET}}
+        HOISTED_COLLECTION_DATASET_BUCKET_NAME: ${{secrets.STAGING_DATA_S3_BUCKET}}
+      run: make save-dataset
+
+    - name: Save expectations to Staging S3
+      if: always()
+      env:
+        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.STAGING_DATA_S3_BUCKET}}
+      run: make save-expectations
+
+    - name: Configure Production AWS Credentials
+      if: always()
+      uses: aws-actions/configure-aws-credentials@v1-node16
+      with:
+        aws-access-key-id: ${{secrets.PROD_AWS_ACCESS_KEY_ID}}
+        aws-secret-access-key: ${{secrets.PROD_AWS_ACCESS_SECRET}}
+        aws-region: eu-west-2
+
+    - name: Save datasets to Prod S3
+      env:
+        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.PRODUCTION_DATA_S3_BUCKET}}
+        HOISTED_COLLECTION_DATASET_BUCKET_NAME: ${{secrets.PRODUCTION_DATA_S3_BUCKET}}
+      run: make save-dataset

.github/workflows/run.yml

@@ -7,4 +7,7 @@ jobs:
   call-workflow:
     uses: digital-land/collection-template/.github/workflows/callable_run.yml@main
     secrets: inherit
-
+  build-package:
+    needs: call-workflow
+    uses: ./.github/workflows/build_package.yml
+    secrets: inherit

Makefile

@@ -3,29 +3,14 @@ include makerules/development.mk
 include makerules/collection.mk
 include makerules/pipeline.mk
 
-# 
-#  Combine the individual organisation datasets into a single organisation.csv  
-#  TBD: make from a specification datapackage definition
-#
-ORGANISATION_DATASETS=\
-	$(DEVELOPMENT_CORPORATION_DATASET)\
-	$(GOVERNMENT_ORGANISATION_DATASET)\
-	$(LOCAL_AUTHORITY_DATASET)\
-	$(NATIONAL_PARK_AUTHORITY_DATASET)\
-	$(NONPROFIT_DATASET)\
-	$(PASSENGER_TRANSPORT_EXECUTIVE_DATASET)\
-	$(PUBLIC_AUTHORITY_DATASET)\
-	$(REGIONAL_PARK_AUTHORITY_DATASET)\
-	$(WASTE_AUTHORITY_DATASET)
+package:: dataset/organisation.csv dataset/organisation-check.csv
 
-dataset:: dataset/organisation.csv dataset/organisation-check.csv
-
-dataset/organisation.csv: $(ORGANISATION_DATASETS)
-	python3 bin/create_organisation_csv.py --output-path $@ 
+dataset/organisation.csv:
+	digital-land organisation-create --download-url="https://files.planning.data.gov.uk/organisation-collection/dataset" --output-path $@
 
 # check organisation datapackage
 dataset/organisation-check.csv: dataset/organisation.csv var/cache/local-planning-authority.csv
-	python3 bin/check_organisation_csv.py --output-path $@ 
+	digital-land organisation-check --output-path $@
 
 var/cache/local-planning-authority.csv:
 	@mkdir -p $(CACHE_DIR)