Merge remote-tracking branch 'origin/main' into renameToCheck

.github/workflows/deploy.yml

@@ -67,6 +67,20 @@ jobs:
 
       - id: run-acceptance-tests
         run: npm run test:acceptance:ci
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+      - id: docker-compose-logs
+        if: always()
+        run: |
+          docker compose -f docker-compose-real-backend.yml logs --tail frontend
+          docker compose -f docker-compose-real-backend.yml logs --tail request-api
+
 
       - id: stop-containers
         if: always()

docker-compose-real-backend.yml

@@ -6,6 +6,10 @@ services:
       context: .
     environment:
       NODE_ENV: ci
+      AWS_ACCESS_KEY_ID: example
+      AWS_SECRET_ACCESS_KEY: example
+      AWS_DEFAULT_REGION: eu-west-2
+      AWS_ENDPOINT_URL: http://localstack:4566
     ports:
       - "8085:5000"
 
@@ -33,6 +37,7 @@ services:
 
   request-api:
     image: public.ecr.aws/l6z6v3j6/development-pub-async-request-api:main
+    # image: request-api:latest # Uncomment when working locally with this file
     environment:
       PYTHONUNBUFFERED: 1
       AWS_ENDPOINT_URL: http://localstack:4566
@@ -55,6 +60,7 @@ services:
 
   request-processor:
     image: public.ecr.aws/l6z6v3j6/development-pub-async-request-processor:main
+    # image: request-processor:latest # Uncomment when working locally with this file
     environment:
       PYTHONUNBUFFERED: 1
       AWS_ENDPOINT_URL: http://localstack:4566
@@ -69,7 +75,7 @@ services:
       REQUEST_FILES_BUCKET_NAME: dluhc-data-platform-request-files-local
     restart: on-failure
     deploy:
-      replicas: 1
+      replicas: 2
     volumes:
       - "./request-processor-celery/docker_volume:/opt"
 

localstack_bootstrap/s3_bootstrap.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# enable debug
+# set -x
+
+echo "configuring s3"
+echo "==================="
+LOCALSTACK_HOST=localhost
+AWS_REGION=eu-west-2
+
+create_upload_bucket() {
+    local BUCKET_NAME_TO_CREATE=$1
+    awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 s3api create-bucket --bucket ${BUCKET_NAME_TO_CREATE} --region ${AWS_REGION} --create-bucket-configuration LocationConstraint=${AWS_REGION}
+    awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 s3api put-bucket-cors --bucket ${BUCKET_NAME_TO_CREATE} --cors-configuration file:///etc/localstack/init/ready.d/cors-config.json
+}
+
+upload_file_to_bucket() {
+    local FILENAME=$1
+    local FILEPATH=$2
+    local BUCKET_NAME=$3
+    awslocal s3api put-object --bucket ${BUCKET_NAME} --key ${FILENAME} --body ${FILEPATH}
+}
+
+create_upload_bucket "dluhc-data-platform-request-files-local"

package.json

@@ -25,7 +25,7 @@
     "test:coverage": "vitest test/unit test/integration test/contract  --coverage",
     "test:integration": "NODE_ENV=test playwright test --config ./test/integration/playwright.config.js",
     "test:acceptance": "NODE_ENV=development playwright test --config ./test/acceptance/playwright.config.js",
-    "test:acceptance:ci": "NODE_ENV=ci playwright test --config ./test/acceptance/playwright.config.js --grep @url",
+    "test:acceptance:ci": "NODE_ENV=ci playwright test --config ./test/acceptance/playwright.config.js",
     "playwright-codegen": "playwright codegen http://localhost:5000",
     "lint": "standard",
     "lint:fix": "standard --fix"

src/controllers/resultsController.js

@@ -6,50 +6,47 @@ const errorsTemplate = 'results/errors'
 const noErrorsTemplate = 'results/no-errors'
 
 class ResultsController extends PageController {
-  async configure (req, res, next) {
+  async locals (req, res, next) {
     try {
-      this.result = await getRequestData(req.params.id)
-      if (!this.result.isComplete()) {
+      const result = await getRequestData(req.params.id)
+      req.form.options.data = result
+
+      if (!result.isComplete()) {
         res.redirect(`/status/${req.params.id}`)
         return
-      } else if (this.result.isFailed()) {
-        this.template = failedRequestTemplate
-      } else if (this.result.hasErrors()) {
-        this.template = errorsTemplate
-        await this.result.fetchResponseDetails(req.params.pageNumber, 50, 'error')
+      } else if (result.isFailed()) {
+        req.form.options.template = failedRequestTemplate
+      } else if (result.hasErrors()) {
+        req.form.options.template = errorsTemplate
+        await result.fetchResponseDetails(req.params.pageNumber, 50, 'error')
+      } else {
+        req.form.options.template = noErrorsTemplate
+        await result.fetchResponseDetails(req.params.pageNumber)
+      }
+
+      req.form.options.requestParams = result.getParams()
+
+      if (req.form.options.template !== failedRequestTemplate) {
+        req.form.options.errorSummary = result.getErrorSummary()
+        req.form.options.columns = result.getColumns()
+        req.form.options.fields = result.getFields()
+        req.form.options.mappings = result.getFieldMappings()
+        req.form.options.verboseRows = result.getRowsWithVerboseColumns(result.hasErrors())
+        req.form.options.geometries = result.getGeometries()
+        req.form.options.pagination = result.getPagination(req.params.pageNumber)
+        req.form.options.id = req.params.id
       } else {
-        this.template = noErrorsTemplate
-        await this.result.fetchResponseDetails(req.params.pageNumber)
+        req.form.options.error = result.getError()
       }
 
-      super.configure(req, res, next)
+      super.locals(req, res, next)
     } catch (error) {
       next(error, req, res, next)
     }
   }
 
-  async locals (req, res, next) {
-    req.form.options.template = this.template
-    req.form.options.requestParams = this.result.getParams()
-
-    if (this.template !== failedRequestTemplate) {
-      req.form.options.errorSummary = this.result.getErrorSummary()
-      req.form.options.columns = this.result.getColumns()
-      req.form.options.fields = this.result.getFields()
-      req.form.options.mappings = this.result.getFieldMappings()
-      req.form.options.verboseRows = this.result.getRowsWithVerboseColumns(this.result.hasErrors())
-      req.form.options.geometries = this.result.getGeometries()
-      req.form.options.pagination = this.result.getPagination(req.params.pageNumber)
-      req.form.options.id = req.params.id
-    } else {
-      req.form.options.error = this.result.getError()
-    }
-
-    super.locals(req, res, next)
-  }
-
   noErrors (req, res, next) {
-    return !this.result.hasErrors()
+    return !req.form.options.data.hasErrors()
   }
 }
 

src/controllers/statusController.js

@@ -3,21 +3,16 @@ import { getRequestData } from '../utils/asyncRequestApi.js'
 import { finishedProcessingStatuses } from '../utils/utils.js'
 
 class StatusController extends PageController {
-  async configure (req, res, next) {
+  async locals (req, res, next) {
     try {
-      this.result = await getRequestData(req.params.id)
-      super.configure(req, res, next)
+      req.form.options.data = await getRequestData(req.params.id)
+      req.form.options.processingComplete = finishedProcessingStatuses.includes(req.form.options.data.status)
+      req.form.options.pollingEndpoint = `/api/status/${req.form.options.data.id}`
+      super.locals(req, res, next)
     } catch (error) {
       next(error, req, res, next)
     }
   }
-
-  async locals (req, res, next) {
-    req.form.options.data = this.result
-    req.form.options.processingComplete = finishedProcessingStatuses.includes(this.result.status)
-    req.form.options.pollingEndpoint = `/api/status/${this.result.id}`
-    super.locals(req, res, next)
-  }
 }
 
 export default StatusController

src/controllers/submitUrlController.js

@@ -7,8 +7,6 @@ import { allowedFileTypes } from '../utils/utils.js'
 
 class SubmitUrlController extends UploadController {
   async post (req, res, next) {
-    this.resetValidationErrorMessage()
-
     const localValidationErrorType = await SubmitUrlController.localUrlValidation(req.body.url)
 
     if (localValidationErrorType) {

src/controllers/uploadController.js

@@ -1,29 +1,14 @@
 'use strict'
 import PageController from './pageController.js'
 import config from '../../config/index.js'
-import logger from '../utils/logger.js'
 
 class UploadController extends PageController {
   apiRoute = config.asyncRequestApi.url + config.asyncRequestApi.requestsEndpoint
 
-  locals (req, res, next) {
-    req.form.options.validationError = this.validationErrorMessage
-    super.locals(req, res, next)
-  }
-
   async post (req, res, next) {
     super.post(req, res, next)
   }
 
-  resetValidationErrorMessage () {
-    this.validationErrorMessage = undefined
-  }
-
-  validationError (type, message, errorObject, req) {
-    logger.error({ type, message, errorObject })
-    this.validationErrorMessage = message
-  }
-
   getBaseFormData (req) {
     return {
       dataset: req.sessionModel.get('dataset'),

src/controllers/uploadFileController.js

@@ -26,8 +26,6 @@ class UploadFileController extends UploadController {
   }
 
   async post (req, res, next) {
-    this.resetValidationErrorMessage()
-
     let dataFileForLocalValidation = null
 
     if (req.file) {

test/acceptance/request_check.test.js

@@ -13,7 +13,8 @@ import StartPage from '../PageObjectModels/startPage'
 import { datasets } from '../PageObjectModels/datasetPage'
 import { uploadMethods } from '../PageObjectModels/uploadMethodPage'
 
-test.setTimeout(50000)
+test.setTimeout(300000)
+
 test.describe('Request Check', () => {
   test.describe('with javascript enabled', () => {
     test('request check of a @datafile', async ({ page }) => {
@@ -229,7 +230,7 @@ test.describe('Request Check', () => {
       await statusPage.expectCheckStatusButtonToBeVisible()
       const id = await statusPage.getIdFromUrl()
 
-      await page.waitForTimeout(3000) // wait for 3 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
+      await page.waitForTimeout(5000) // wait for 10 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
 
       const resultsPage = await statusPage.clickCheckStatusButton()
 
@@ -260,7 +261,7 @@ test.describe('Request Check', () => {
       await statusPage.expectCheckStatusButtonToBeVisible()
       const id = await statusPage.getIdFromUrl()
 
-      await page.waitForTimeout(3000) // wait for 3 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
+      await page.waitForTimeout(5000) // wait for 5 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
 
       const resultsPage = await statusPage.clickCheckStatusButton()
 

test/integration/playwright.config.js

@@ -86,7 +86,7 @@ export default defineConfig({
   /* Run your local dev server before starting the tests */
   webServer: {
     command: 'NODE_ENV=test npm run start',
-    url: 'http://127.0.0.1:5000',
+    url: `http://127.0.0.1:${config.port}`,
     reuseExistingServer: !process.env.CI
   }
 })