Merge remote-tracking branch 'origin/main' into subpath-reorganization

config/default.yaml

@@ -32,4 +32,7 @@ email: {
     },
     dataManagementEmail: '[email protected]'
 }
+validations: {
+    maxFileSize: 100000000
+}
 

readme.md

@@ -8,7 +8,7 @@ This is the frontend for the LPA Data Validator application. It is a nodeJS expr
 - Install the node packages
     ```
     npm install
-    ``` 
+    ```
 - setup husky pre-commit hooks
     ```
     npm run prepare
@@ -19,6 +19,16 @@ This is the frontend for the LPA Data Validator application. It is a nodeJS expr
     ```
 
 ## Running the application
+
+**Prerequisite**: Install [Docker Desktop](https://www.docker.com/products/docker-desktop/)
+
+```sh
+docker-compose -f docker-compose-real-backend-minus-frontend.yml up -d;
+
+npm run start:local;
+```
+
+### Alternative methods of starting application
 - Run the application
     ```
     npm run start
@@ -27,11 +37,30 @@ This is the frontend for the LPA Data Validator application. It is a nodeJS expr
     ```
     npm run start
     ```
-- run the application, using a local api
+- Run the application, using a local api
     ```
     npm run start:local
     ```
-- run the application, using a local api in watch mode
+- Run the application, using a local api in watch mode
     ```
     npm run start:local:watch
-    ```
+    ```
+- Run the application using docker
+    ```
+    docker-compose -f docker-compose-real-backend.yml up
+    ```
+- Run the application (without the frontend) using docker
+    ```
+    docker-compose -f docker-compose-real-backend-minus-frontend.yml up
+    ```
+
+## Mac users only
+
+If you are a Mac user, please note that port 5000 is used by AirPlay Receiver. In order to use the application, you will need to switch off AirPlay Receiver.
+
+To switch off AirPlay Receiver, follow these steps:
+1. Open System Preferences on your Mac.
+2. Search for "AirDrop & Handoff".
+3. In the "AirPlay Receiver" toggle, select "Off".
+
+Once you have switched off AirPlay Receiver, you should be able to use the application without any issues.

src/controllers/OrganisationsController.js

@@ -4,6 +4,9 @@ import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
 import { dataSubjects } from '../utils/utils.js'
 import { statusToTagClass } from '../filters/filters.js'
+import { render } from '../utils/custom-renderer.js'
+import { templateSchema } from '../routes/schemas.js'
+import * as v from 'valibot'
 
 // get a list of available datasets
 const availableDatasets = Object.values(dataSubjects)
@@ -13,6 +16,12 @@ const availableDatasets = Object.values(dataSubjects)
       .map(dataset => dataset.value)
   )
 
+function validateAndRender (res, name, params) {
+  const schema = templateSchema.get(name) ?? v.any()
+  logger.info(`rendering '${name}' with schema=<${schema ? 'defined' : 'any'}>`, { type: types.App })
+  return render(res, name, schema, params)
+}
+
 /**
  * Returns a status tag object with a text label and a CSS class based on the status.
  *
@@ -94,7 +103,7 @@ const organisationsController = {
         datasetsWithErrors
       }
 
-      res.render('organisations/overview.html', params)
+      validateAndRender(res, 'organisations/overview.html', params)
     } catch (error) {
       logger.warn('organisationsController.getOverview(): ' + error.message ?? error.errorMessage, { type: types.App })
       next(error)
@@ -124,7 +133,7 @@ const organisationsController = {
         return acc
       }, {})
 
-      res.render('organisations/find.html', { alphabetisedOrgs })
+      validateAndRender(res, 'organisations/find.html', { alphabetisedOrgs })
     } catch (err) {
       logger.warn('organisationsController.getOrganisations(): ' + err.message ?? err.errorMessage, { type: types.App })
       next(err)
@@ -144,7 +153,7 @@ const organisationsController = {
     try {
     // get the organisation name
       const lpa = req.params.lpa
-      const organisationResult = await datasette.runQuery(`SELECT name FROM organisation WHERE organisation = '${lpa}'`)
+      const organisationResult = await datasette.runQuery(`SELECT name, organisation FROM organisation WHERE organisation = '${lpa}'`)
       const organisation = organisationResult.formattedData[0]
 
       // get the dataset name
@@ -157,9 +166,15 @@ const organisationsController = {
         dataset
       }
 
-      res.render('organisations/get-started.html', params)
+      validateAndRender(res, 'organisations/get-started.html', params)
     } catch (err) {
-      logger.error(err)
+      logger.warn({
+        message: `OrganisationsController.getStarted(): ${err.message}`,
+        endpoint: req.originalUrl,
+        errorStack: err.stack,
+        errorMessage: err.message,
+        type: types.App
+      })
       next(err)
     }
   },
@@ -203,9 +218,14 @@ const organisationsController = {
         dataset
       }
 
-      res.render('organisations/datasetTaskList.html', params)
+      validateAndRender(res, 'organisations/datasetTaskList.html', params)
     } catch (e) {
-      logger.warn(`getDAtasetTaskList() failed for lpa='${lpa}', datasetId='${datasetId}'`, { type: types.App })
+      logger.warn(`getDatasetTaskList() failed for lpa='${lpa}', datasetId='${datasetId}'`,
+        {
+          type: types.App,
+          errorMessage: e.message,
+          errorStack: e.stack
+        })
       next(e)
     }
   },
@@ -235,7 +255,7 @@ const organisationsController = {
           latest_200_date: last200Datetime
         }
       }
-      res.render('organisations/http-error.html', params)
+      validateAndRender(res, 'organisations/http-error.html', params)
     } catch (e) {
       logger.warn(`conditionalTaskListHandler() failed for lpa='${lpa}', datasetId='${datasetId}'`, { type: types.App })
       next(e)
@@ -373,7 +393,7 @@ const organisationsController = {
         issueType
       }
 
-      res.render('organisations/issueDetails.html', params)
+      validateAndRender(res, 'organisations/issueDetails.html', params)
     } catch (e) {
       logger.warn(`getIssueDetails() failed for lpa='${lpa}', datasetId='${datasetId}', issue=${issueType}, entityNumber=${entityNumber}, resourceId=${resourceId}`, { type: types.App })
       next(e)

src/controllers/submitUrlController.js

@@ -2,8 +2,10 @@ import UploadController from './uploadController.js'
 import { postUrlRequest } from '../services/asyncRequestApi.js'
 import { URL } from 'url'
 import logger from '../utils/logger.js'
+import { types } from '../utils/logging.js'
 import axios from 'axios'
 import { allowedFileTypes } from '../utils/utils.js'
+import config from '../../config/index.js'
 
 class SubmitUrlController extends UploadController {
   async post (req, res, next) {
@@ -17,7 +19,12 @@ class SubmitUrlController extends UploadController {
       const errors = {
         url: new SubmitUrlController.Error(error.key, error, req, res)
       }
-      logger.warn('SubmitUrlController: local validation failed during url submission', error)
+      logger.warn({
+        message: 'SubmitUrlController: local validation failed during url submission',
+        error: JSON.stringify(error),
+        submittedUrl: `${req.body.url ?? '<no url provided>'}`,
+        type: types.DataValidation
+      })
       return next(errors)
     }
 
@@ -91,10 +98,7 @@ class SubmitUrlController extends UploadController {
     try {
       const contentLength = response.headers['content-length']
 
-      // Convert content length to MB
-      const sizeInMB = contentLength / (1024 * 1024)
-
-      return sizeInMB <= 10
+      return contentLength <= config.validations.maxFileSize
     } catch (err) {
       console.warn(err)
       return true // for now we will allow this file as we can't be sure

src/controllers/uploadFileController.js

@@ -47,7 +47,10 @@ class UploadFileController extends UploadController {
       const errors = {
         datafile: new UploadFileController.Error(error.key, error, req, res)
       }
-      logger.warn('UploadFileController: local validation failed during file upload', error)
+      logger.info('UploadFileController: local validation failed during file upload', {
+        type: types.App,
+        fileValidationError: error
+      })
       return next(errors)
     }
 
@@ -123,9 +126,7 @@ class UploadFileController extends UploadController {
   }
 
   static sizeIsValid (datafile) {
-    const maxSize = 10 * 1024 * 1024 // 10MB
-
-    if (datafile.size > maxSize) {
+    if (datafile.size > config.validations.maxFileSize) {
       return false
     }
 

src/filters/makeDatasetSlugToReadableNameFilter.js

@@ -18,9 +18,8 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => {
   return (slug) => {
     const name = datasetNameMapping.get(slug)
     if (!name) {
-      // throw new Error(`Can't find a name for ${slug}`)
       // ToDo: work out what to do here? potentially update it with data from datasette
-      logger.warn(`can't find a name for ${slug}`)
+      logger.debug(`can't find a name for ${slug}`)
       return slug
     }
     return name

src/filters/validationMessageLookup.js

@@ -17,7 +17,7 @@ const validationMessages = {
   datafile: {
     required: 'Select a file',
     fileType: 'The selected file must be a CSV, GeoJSON, GML or GeoPackage file',
-    fileSize: 'The selected file must be smaller than 10MB',
+    fileSize: 'The selected file must be smaller than 100MB',
     fileNameTooLong: 'The selected file name must be less than 100 characters',
     fileNameInvalidCharacters: 'The selected file name must not contain any of the following characters: / \\ : * ? " < > |',
     fileNameDoubleExtension: 'The selected file name must not contain two file extensions',
@@ -30,7 +30,7 @@ const validationMessages = {
     length: 'The URL must be less than 2048 characters',
     exists: 'The URL does not exist',
     filetype: 'The file referenced by URL must be a CSV, GeoJSON, GML or GeoPackage file',
-    size: 'The file referenced by URL must be smaller than 10MB'
+    size: 'The file referenced by URL must be smaller than 100MB'
   },
   validationResult: {
     required: 'Unable to contact the API'

src/models/requestData.js

@@ -29,7 +29,7 @@ export default class RequestData {
 
   getErrorSummary () {
     if (!this.response || !this.response.data || !this.response.data['error-summary']) {
-      logger.warn('trying to get error summary when there is none: request id: ' + this.id)
+      logger.warn('trying to get error summary when there is none', { requestId: this.id })
       return []
     }
     return this.response.data['error-summary']
@@ -45,7 +45,7 @@ export default class RequestData {
 
   getError () {
     if (!this.response) {
-      logger.warn('trying to get error when there are none: request id: ' + this.id)
+      logger.warn('trying to get error when there are none', { requestId: this.id })
       return { message: 'An unknown error occurred.' }
     }
 
@@ -54,11 +54,11 @@ export default class RequestData {
 
   hasErrors () {
     if (!this.response || !this.response.data) {
-      logger.warn('trying to check for errors when there are none: request id: ' + this.id)
+      logger.warn('trying to check for errors when there are none', { requestId: this.id })
       return true
     }
     if (this.response.data['error-summary'] == null) {
-      logger.warn('trying to check for errors but there is no error-summary: request id: ' + this.id)
+      logger.warn('trying to check for errors but there is no error-summary', { requestId: this.id })
       return true
     }
     return this.response.data['error-summary'].length > 0
@@ -71,7 +71,7 @@ export default class RequestData {
 
   getColumnFieldLog () {
     if (!this.response || !this.response.data || !this.response.data['column-field-log']) {
-      logger.warn('trying to get column field log when there is none: request id: ' + this.id)
+      logger.warn('trying to get column field log when there is none', { requestId: this.id })
       return []
     }
     return this.response.data['column-field-log']

src/models/responseDetails.js

@@ -11,15 +11,15 @@ export default class ResponseDetails {
 
   getRows () {
     if (!this.response) {
-      logger.warn('trying to get response details when there are none: request id: ' + this.id)
+      logger.warn('trying to get response details when there are none', { requestId: this.id })
       return []
     }
     return this.response
   }
 
   getColumnFieldLog () {
     if (!this.columnFieldLog) {
-      logger.warn('trying to get column field log when there is none: request id: ' + this.id)
+      logger.warn('trying to get column field log when there is none', { requestId: this.id })
       return []
     }
     return this.columnFieldLog
@@ -69,7 +69,7 @@ export default class ResponseDetails {
   // This function returns an array of rows with verbose columns
   getRowsWithVerboseColumns (filterNonErrors = false) {
     if (!this.response) {
-      logger.warn('trying to get response details when there are none: request id: ' + this.id)
+      logger.warn('trying to get response details when there are none', { requestId: this.id })
       return []
     }
 
@@ -105,7 +105,7 @@ export default class ResponseDetails {
 
   getGeometries () {
     if (!this.response) {
-      logger.warn('trying to get response details when there are none: request id: ' + this.id)
+      logger.warn('trying to get response details when there are none', { requestId: this.id })
       return undefined
     }