Get files using sync (#9)

Use aws sync rather than CDN to get the files to build the package.

.github/workflows/build_package.yml

@@ -51,6 +51,14 @@ jobs:
     - name: Install dependencies
       run: make init
 
+    - name: Configure Production AWS Credentials
+      if: always()
+      uses: aws-actions/configure-aws-credentials@v1-node16
+      with:
+        aws-access-key-id: ${{secrets.PROD_AWS_ACCESS_KEY_ID}}
+        aws-secret-access-key: ${{secrets.PROD_AWS_ACCESS_SECRET}}
+        aws-region: eu-west-2
+
     - name: Create the package
       run: make package
 
@@ -68,12 +76,6 @@ jobs:
         HOISTED_COLLECTION_DATASET_BUCKET_NAME: ${{secrets.DEVELOPMENT_DATA_S3_BUCKET}}
       run: make save-dataset
 
-    - name: Save expectations to Development S3
-      if: always()
-      env:
-        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.DEVELOPMENT_DATA_S3_BUCKET}}
-      run: make save-expectations
-
     - name: Configure Staging AWS Credentials
       if: always()
       uses: aws-actions/configure-aws-credentials@v1-node16
@@ -88,12 +90,6 @@ jobs:
         HOISTED_COLLECTION_DATASET_BUCKET_NAME: ${{secrets.STAGING_DATA_S3_BUCKET}}
       run: make save-dataset
 
-    - name: Save expectations to Staging S3
-      if: always()
-      env:
-        COLLECTION_DATASET_BUCKET_NAME: ${{secrets.STAGING_DATA_S3_BUCKET}}
-      run: make save-expectations
-
     - name: Configure Production AWS Credentials
       if: always()
       uses: aws-actions/configure-aws-credentials@v1-node16

Makefile

@@ -3,14 +3,12 @@ include makerules/development.mk
 include makerules/collection.mk
 include makerules/pipeline.mk
 
-all:: package
-
 package:: dataset/organisation.csv dataset/organisation-check.csv
 
 dataset/organisation.csv:
-	digital-land organisation-create --download-url="https://files.planning.data.gov.uk/organisation-collection/dataset" --output-path $@
+	aws s3 sync s3://$(ENVIRONMENT)-collection-data/organisation-collection/dataset $(DATASET_DIR) --no-progress
+	digital-land organisation-create --dataset-dir=$(DATASET_DIR) --output-path $@
 
-# check organisation datapackage
 dataset/organisation-check.csv: dataset/organisation.csv var/cache/local-planning-authority.csv
 	digital-land organisation-check --output-path $@