Merge remote-tracking branch 'origin/main' into refactor/splitRequest…

…DataClass

.github/workflows/deploy.yml

@@ -67,6 +67,20 @@ jobs:
 
       - id: run-acceptance-tests
         run: npm run test:acceptance:ci
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+      - id: docker-compose-logs
+        if: always()
+        run: |
+          docker compose -f docker-compose-real-backend.yml logs --tail frontend
+          docker compose -f docker-compose-real-backend.yml logs --tail request-api
+
 
       - id: stop-containers
         if: always()

docker-compose-real-backend.yml

@@ -6,6 +6,10 @@ services:
       context: .
     environment:
       NODE_ENV: ci
+      AWS_ACCESS_KEY_ID: example
+      AWS_SECRET_ACCESS_KEY: example
+      AWS_DEFAULT_REGION: eu-west-2
+      AWS_ENDPOINT_URL: http://localstack:4566
     ports:
       - "8085:5000"
 
@@ -33,6 +37,7 @@ services:
 
   request-api:
     image: public.ecr.aws/l6z6v3j6/development-pub-async-request-api:main
+    # image: request-api:latest # Uncomment when working locally with this file
     environment:
       PYTHONUNBUFFERED: 1
       AWS_ENDPOINT_URL: http://localstack:4566
@@ -55,6 +60,7 @@ services:
 
   request-processor:
     image: public.ecr.aws/l6z6v3j6/development-pub-async-request-processor:main
+    # image: request-processor:latest # Uncomment when working locally with this file
     environment:
       PYTHONUNBUFFERED: 1
       AWS_ENDPOINT_URL: http://localstack:4566
@@ -69,7 +75,7 @@ services:
       REQUEST_FILES_BUCKET_NAME: dluhc-data-platform-request-files-local
     restart: on-failure
     deploy:
-      replicas: 1
+      replicas: 2
     volumes:
       - "./request-processor-celery/docker_volume:/opt"
 

localstack_bootstrap/s3_bootstrap.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# enable debug
+# set -x
+
+echo "configuring s3"
+echo "==================="
+LOCALSTACK_HOST=localhost
+AWS_REGION=eu-west-2
+
+create_upload_bucket() {
+    local BUCKET_NAME_TO_CREATE=$1
+    awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 s3api create-bucket --bucket ${BUCKET_NAME_TO_CREATE} --region ${AWS_REGION} --create-bucket-configuration LocationConstraint=${AWS_REGION}
+    awslocal --endpoint-url=http://${LOCALSTACK_HOST}:4566 s3api put-bucket-cors --bucket ${BUCKET_NAME_TO_CREATE} --cors-configuration file:///etc/localstack/init/ready.d/cors-config.json
+}
+
+upload_file_to_bucket() {
+    local FILENAME=$1
+    local FILEPATH=$2
+    local BUCKET_NAME=$3
+    awslocal s3api put-object --bucket ${BUCKET_NAME} --key ${FILENAME} --body ${FILEPATH}
+}
+
+create_upload_bucket "dluhc-data-platform-request-files-local"

package.json

@@ -25,7 +25,7 @@
     "test:coverage": "vitest test/unit test/integration test/contract  --coverage",
     "test:integration": "NODE_ENV=test playwright test --config ./test/integration/playwright.config.js",
     "test:acceptance": "NODE_ENV=development playwright test --config ./test/acceptance/playwright.config.js",
-    "test:acceptance:ci": "NODE_ENV=ci playwright test --config ./test/acceptance/playwright.config.js --grep @url",
+    "test:acceptance:ci": "NODE_ENV=ci playwright test --config ./test/acceptance/playwright.config.js",
     "playwright-codegen": "playwright codegen http://localhost:5000",
     "lint": "standard",
     "lint:fix": "standard --fix"

src/controllers/resultsController.js

@@ -7,47 +7,48 @@ const noErrorsTemplate = 'results/no-errors'
 
 class ResultsController extends PageController {
   async locals (req, res, next) {
-    let requestData
     try {
-      requestData = await getRequestData(req.params.id)
-    } catch (e) {
-      next(e, req, res, next)
-      return
-    }
-
-    if (!requestData.isComplete()) {
-      res.redirect(`/status/${req.params.id}`)
-      return
-    }
-
-    let responseDetails
-
-    if (requestData.isFailed()) {
-      req.form.options.template = failedRequestTemplate
-    } else if (requestData.hasErrors()) {
-      req.form.options.template = errorsTemplate
-      responseDetails = await requestData.fetchResponseDetails(req.params.pageNumber, 50, 'error')
-    } else {
-      req.form.options.template = noErrorsTemplate
-      responseDetails = await requestData.fetchResponseDetails(req.params.pageNumber)
-    }
-
-    req.form.options.requestParams = requestData.getParams()
-
-    if (req.form.options.template !== failedRequestTemplate) {
-      req.form.options.errorSummary = requestData.getErrorSummary()
-      req.form.options.columns = responseDetails.getColumns()
-      req.form.options.fields = responseDetails.getFields()
-      req.form.options.mappings = responseDetails.getFieldMappings()
-      req.form.options.verboseRows = responseDetails.getRowsWithVerboseColumns(requestData.hasErrors())
-      req.form.options.geometries = responseDetails.getGeometries()
-      req.form.options.pagination = responseDetails.getPagination(req.params.pageNumber)
-      req.form.options.id = req.params.id
-    } else {
-      req.form.options.error = requestData.getError()
+      const requestData = await getRequestData(req.params.id)
+      req.form.options.data = requestData
+
+      let responseDetails
+
+      if (!requestData.isComplete()) {
+        res.redirect(`/status/${req.params.id}`)
+        return
+      } else if (requestData.isFailed()) {
+        req.form.options.template = failedRequestTemplate
+      } else if (requestData.hasErrors()) {
+        req.form.options.template = errorsTemplate
+        responseDetails = await requestData.fetchResponseDetails(req.params.pageNumber, 50, 'error')
+      } else {
+        req.form.options.template = noErrorsTemplate
+        responseDetails = await requestData.fetchResponseDetails(req.params.pageNumber)
+      }
+
+      req.form.options.requestParams = requestData.getParams()
+
+      if (req.form.options.template !== failedRequestTemplate) {
+        req.form.options.errorSummary = requestData.getErrorSummary()
+        req.form.options.columns = responseDetails.getColumns()
+        req.form.options.fields = responseDetails.getFields()
+        req.form.options.mappings = responseDetails.getFieldMappings()
+        req.form.options.verboseRows = responseDetails.getRowsWithVerboseColumns(requestData.hasErrors())
+        req.form.options.geometries = responseDetails.getGeometries()
+        req.form.options.pagination = responseDetails.getPagination(req.params.pageNumber)
+        req.form.options.id = req.params.id
+      } else {
+        req.form.options.error = requestData.getError()
+      }
+
+      super.locals(req, res, next)
+    } catch (error) {
+      next(error, req, res, next)
     }
+  }
 
-    super.locals(req, res, next)
+  noErrors (req, res, next) {
+    return !req.form.options.data.hasErrors()
   }
 }
 

src/controllers/statusController.js

@@ -4,11 +4,14 @@ import { finishedProcessingStatuses } from '../utils/utils.js'
 
 class StatusController extends PageController {
   async locals (req, res, next) {
-    const requestData = await getRequestData(req.params.id)
-    req.form.options.data = requestData
-    req.form.options.processingComplete = finishedProcessingStatuses.includes(requestData.status)
-    req.form.options.pollingEndpoint = `/api/status/${requestData.id}`
-    super.locals(req, res, next)
+    try {
+      req.form.options.data = await getRequestData(req.params.id)
+      req.form.options.processingComplete = finishedProcessingStatuses.includes(req.form.options.data.status)
+      req.form.options.pollingEndpoint = `/api/status/${req.form.options.data.id}`
+      super.locals(req, res, next)
+    } catch (error) {
+      next(error, req, res, next)
+    }
   }
 }
 

src/controllers/submitUrlController.js

@@ -7,8 +7,6 @@ import { allowedFileTypes } from '../utils/utils.js'
 
 class SubmitUrlController extends UploadController {
   async post (req, res, next) {
-    this.resetValidationErrorMessage()
-
     const localValidationErrorType = await SubmitUrlController.localUrlValidation(req.body.url)
 
     if (localValidationErrorType) {

src/controllers/uploadController.js

@@ -1,29 +1,14 @@
 'use strict'
 import PageController from './pageController.js'
 import config from '../../config/index.js'
-import logger from '../utils/logger.js'
 
 class UploadController extends PageController {
   apiRoute = config.asyncRequestApi.url + config.asyncRequestApi.requestsEndpoint
 
-  locals (req, res, next) {
-    req.form.options.validationError = this.validationErrorMessage
-    super.locals(req, res, next)
-  }
-
   async post (req, res, next) {
     super.post(req, res, next)
   }
 
-  resetValidationErrorMessage () {
-    this.validationErrorMessage = undefined
-  }
-
-  validationError (type, message, errorObject, req) {
-    logger.error({ type, message, errorObject })
-    this.validationErrorMessage = message
-  }
-
   getBaseFormData (req) {
     return {
       dataset: req.sessionModel.get('dataset'),

src/controllers/uploadFileController.js

@@ -26,8 +26,6 @@ class UploadFileController extends UploadController {
   }
 
   async post (req, res, next) {
-    this.resetValidationErrorMessage()
-
     let dataFileForLocalValidation = null
 
     if (req.file) {

test/PageObjectModels/BasePage.js

@@ -38,7 +38,11 @@ export default class BasePage {
       expect(await errorLink.isVisible(), 'Page should show an error summary that is a link to the problem field').toBeTruthy()
       expect(await fieldError.isVisible(), 'Page should show the error message next to the problem field').toBeTruthy()
       await errorLink.click()
-      const problemFieldIsFocused = await this.page.$eval(fieldName, (el) => el === document.activeElement)
+      let problemFieldIsFocused = await this.page.$eval(fieldName, (el) => el === document.activeElement)
+      if (problemFieldIsFocused === undefined) { // sometimes the page load is slow, so this returns nothing. if this happens. wait 0.5s and try again
+        await new Promise(resolve => setTimeout(resolve, 500))
+        problemFieldIsFocused = await this.page.$eval(fieldName, (el) => el === document.activeElement)
+      }
       expect(problemFieldIsFocused, 'The focus should be on the problem field').toBeTruthy()
     }
 

test/acceptance/request_check.test.js

@@ -13,7 +13,8 @@ import StartPage from '../PageObjectModels/startPage'
 import { datasets } from '../PageObjectModels/datasetPage'
 import { uploadMethods } from '../PageObjectModels/uploadMethodPage'
 
-test.setTimeout(50000)
+test.setTimeout(300000)
+
 test.describe('Request Check', () => {
   test.describe('with javascript enabled', () => {
     test('request check of a @datafile', async ({ page }) => {
@@ -229,7 +230,7 @@ test.describe('Request Check', () => {
       await statusPage.expectCheckStatusButtonToBeVisible()
       const id = await statusPage.getIdFromUrl()
 
-      await page.waitForTimeout(3000) // wait for 3 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
+      await page.waitForTimeout(5000) // wait for 10 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
 
       const resultsPage = await statusPage.clickCheckStatusButton()
 
@@ -260,7 +261,7 @@ test.describe('Request Check', () => {
       await statusPage.expectCheckStatusButtonToBeVisible()
       const id = await statusPage.getIdFromUrl()
 
-      await page.waitForTimeout(3000) // wait for 3 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
+      await page.waitForTimeout(5000) // wait for 5 seconds for processing. could be smarter about this so we dont have to wait 3 seconds
 
       const resultsPage = await statusPage.clickCheckStatusButton()
 

test/integration/playwright.config.js

@@ -86,7 +86,7 @@ export default defineConfig({
   /* Run your local dev server before starting the tests */
   webServer: {
     command: 'NODE_ENV=test npm run start',
-    url: 'http://127.0.0.1:5000',
+    url: `http://127.0.0.1:${config.port}`,
     reuseExistingServer: !process.env.CI
   }
 })