diff --git a/config/default.yaml b/config/default.yaml index a96168be..d45fa93d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -4,5 +4,6 @@ logs: app: false error: false api: { - url: http://localhost:3000/api + url: http://127.0.0.1:8082, + validationEndpoint: /api/dataset/validate/file/request/ } diff --git a/src/assets/scss/_scrollable-container.scss b/src/assets/scss/_scrollable-container.scss new file mode 100644 index 00000000..6f3b4475 --- /dev/null +++ b/src/assets/scss/_scrollable-container.scss @@ -0,0 +1,79 @@ +.app-scrollable-container { + + table { + margin-bottom: 20px; + th, td { + white-space: nowrap; + } + + td.app-wrap { + // white-space: unset; + max-width: 500px; + overflow: hidden; + text-overflow: ellipsis; + + .app-inset-text__value { + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + max-height: 500px; + @include govuk-responsive-margin(6, $direction: "bottom"); + padding-left: 15px; + padding-right: 15px; + overflow-x: auto; + border: 1px solid rgb(169, 169, 169); + background-image: + // Shadows + linear-gradient(to right, white, white), + linear-gradient(to right, white, white), + + // Shadow covers + linear-gradient(to right, rgba(0,0,0,.25), rgba(255,255,255,0)), + linear-gradient(to left, rgba(0,0,0,.25), rgba(255,255,255,0)); + + background-position: left center, right center, left center, right center; + background-repeat: no-repeat; + background-color: white; + background-size: 20px 100%, 20px 100%, 10px 100%, 10px 100%; + + /* Opera doesn't support this in the shorthand */ + background-attachment: local, local, scroll, scroll; + + scrollbar-width: thin; + scrollbar-color: #8A8A8A #DFE9EB; + + /* Chrome, Edge and Safari */ + &::-webkit-scrollbar { + height: 10px; + width: 10px; + } + &::-webkit-scrollbar-track { + border-radius: 5px; + background-color: #e8e8e8; + } + + &::-webkit-scrollbar-track:hover { + background-color: #B8C0C2; + } + + &::-webkit-scrollbar-track:active { + background-color: #B8C0C2; + } + + &::-webkit-scrollbar-thumb { + border-radius: 0px; + background-color: $govuk-border-colour; + } + + &::-webkit-scrollbar-thumb:hover { + background-color: #5F5F5F; + } + + &::-webkit-scrollbar-thumb:active { + background-color: #474D54; + } + +} \ No newline at end of file diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss index a6ceec3f..081eca0b 100644 --- a/src/assets/scss/index.scss +++ b/src/assets/scss/index.scss @@ -1,2 +1,37 @@ @import "node_modules/govuk-frontend/govuk/all"; -@import "node_modules/@x-govuk/govuk-prototype-components/x-govuk/all"; \ No newline at end of file +@import "node_modules/@x-govuk/govuk-prototype-components/x-govuk/all"; +@import "src/assets/scss/_scrollable-container.scss"; + +.app-inset-text---error { + border-left: 5px solid govuk-colour('red'); + padding: govuk-spacing(1) govuk-spacing(2); + margin: 0; + + .app-inset-text__value { + margin-bottom: govuk-spacing(1); + } + + .app-inset-text__error { + color: govuk-colour('red'); + font-weight: bold; + } + + + } + + .app-inset-text---warning { + border-left: 5px solid govuk-colour('blue'); + padding: govuk-spacing(1) govuk-spacing(2); + margin: 0; + + .app-inset-text__value { + margin-bottom: govuk-spacing(1); + } + + .app-inset-text__warning { + color: govuk-colour('blue'); + font-weight: bold; + } + + + } \ No newline at end of file diff --git a/src/controllers/errorsController.js b/src/controllers/errorsController.js new file mode 100644 index 00000000..9026b102 --- /dev/null +++ b/src/controllers/errorsController.js @@ -0,0 +1,56 @@ +'use strict' + +const { Controller } = require('hmpo-form-wizard') + +class ErrorsController extends Controller { + get (req, res, next) { + const json = req.sessionModel.get('validationResult') + + const aggregatedIssues = {} + const issueCounts = {} + + json['issue-log'].forEach(issue => { + const entryNumber = issue['entry-number'] + + const rowColumns = json['converted-csv'][issue['line-number'] - 1] + if (!(entryNumber in aggregatedIssues)) { + aggregatedIssues[entryNumber] = Object.keys(rowColumns).reduce((acc, key) => { + acc[key] = { + error: false, + value: rowColumns[key] + } + return acc + }, {}) + } + + if (entryNumber in aggregatedIssues) { + aggregatedIssues[entryNumber][issue.field] = { + error: this.lookupIssueType(issue['issue-type']), + value: rowColumns[issue.field] + } + issueCounts[issue.field] = issueCounts[issue.field] ? issueCounts[issue.field] + 1 : 1 + } + }) + + const rows = Object.keys(aggregatedIssues).map(key => { + return { + entryNumber: key, + columns: aggregatedIssues[key] + } + }) + + req.form.options.rows = rows + req.form.options.issueCounts = issueCounts + req.form.options.dataset = req.sessionModel.get('dataset') + req.form.options.dataSubject = req.sessionModel.get('data-subject') + + super.get(req, res, next) + } + + lookupIssueType (issueType) { + // this needs to be implemented once we know what the issue types are + return issueType + } +} + +module.exports = ErrorsController diff --git a/src/controllers/uploadController.js b/src/controllers/uploadController.js index 01c57bc9..fc00d304 100644 --- a/src/controllers/uploadController.js +++ b/src/controllers/uploadController.js @@ -7,7 +7,9 @@ const { Controller } = require('hmpo-form-wizard') const { readFile } = require('node:fs/promises') const { lookup } = require('mime-types') -const apiRoute = 'http://127.0.0.1:8082/api/dataset/validate/file/request/' +const config = require('../../config') + +const apiRoute = config.api.url + config.api.validationEndpoint class UploadController extends Controller { middlewareSetup () { @@ -35,10 +37,8 @@ class UploadController extends Controller { const json = await result.json() - console.log(json) - - // send the response back to the user - res.send(json) + req.sessionModel.set('validationResult', json) + super.post(req, res, next) } catch (e) { res.send(e) } diff --git a/src/routes/form-wizard/fields.js b/src/routes/form-wizard/fields.js index 1a97b402..7d8a3c7f 100644 --- a/src/routes/form-wizard/fields.js +++ b/src/routes/form-wizard/fields.js @@ -7,5 +7,8 @@ module.exports = { }, datafile: { validate: 'required' + }, + validationResult: { + validate: 'required' } } diff --git a/src/routes/form-wizard/steps.js b/src/routes/form-wizard/steps.js index 0a77de9a..6a066ded 100644 --- a/src/routes/form-wizard/steps.js +++ b/src/routes/form-wizard/steps.js @@ -16,7 +16,10 @@ module.exports = { }, '/upload': { controller: require('../../controllers/uploadController'), - fields: ['datafile', 'path'], - next: 'done' + next: 'errors' + }, + '/errors': { + controller: require('../../controllers/errorsController'), + next: 'transformations' } } diff --git a/src/views/errors.html b/src/views/errors.html new file mode 100644 index 00000000..1cae3853 --- /dev/null +++ b/src/views/errors.html @@ -0,0 +1,141 @@ +{% extends "layouts/main.html" %} + +{% from 'govuk/components/back-link/macro.njk' import govukBackLink %} +{% from 'govuk/components/button/macro.njk' import govukButton %} +{% from 'govuk/components/radios/macro.njk' import govukRadios %} +{% from 'govuk/components/inset-text/macro.njk' import govukInsetText %} + +{% set pageName = 'There’s a problem with your data' %} + +{% block beforeContent %} + {{ govukBackLink({ + text: "Back", + href: "javascript:window.history.back()" + }) }} +{% endblock %} + +{% block content %} + +
+
+ + {% if options.dataset %} + {{options.dataset}} + {% else %} + {{options.dataSubject}} + {% endif %} + +

+ {{pageName}} +

+ + +
+
+ +
+
+

+ Records with errors +

+ +
+ + + + + + + + + + + + + + + + {% for row in options.rows %} + + + + + + + + + + + + {% endfor %} + + +
ReferenceNameGeometryStart dateLegislationNotesPointEnd dateDocument URL
+ {% if row.columns.Reference.error %} + {{ govukInsetText({ + classes: "app-inset-text---error", + html: '

Reference is missing

' + }) }} + {% else %} + {{row.columns.Reference.value}} + {% endif %} + 		{{row.columns.Name.value}} + {% if row.columns.Geometry.error %} + {{ govukInsetText({ + classes: "app-inset-text---error", + html: '

'+row.columns.Geometry.value+'

'+row.columns.Geometry.error +'

' + }) }} + {% else %} + {{row.columns.Geometry.value}} + {% endif %} + 		+ {% if row.columns['Start date'].error %} + {{ govukInsetText({ + classes: "app-inset-text---error", + html: '

'+row.columns['Start date'].value+'

'+row.columns['Start date'].error +'

' + }) }} + {% else %} + {{row.columns['Start date'].value}} + {% endif %} + 		{{row.columns.Legisliation.value}}{{row.columns.Notes.value}}{{row.columns.Point.value}}{{row.columns['End date'].value}}{{row.columns['Document URL'].value}}
+
+
+
+ +
+
+
+ + {{ govukRadios({ + name: "check[fixErrors]", + fieldset: { + legend: { + text: "Do you want to fix the errors before you publish the data?", + classes: "govuk-fieldset__legend--m" + } + }, + value: data.check.fixErrors, + items: [ + { + value: "Yes", + text: "Yes" + }, + { + value: "No", + text: "No, publish the valid data and I’ll fix the invalid data later" + } + ] + }) }} + + {{ govukButton({ + text: "Continue" + }) }} +
+ +
+
+{% endblock %} diff --git a/test/acceptance/upload_data.test.js b/test/acceptance/upload_data.test.js index f3b8c8b1..accd4261 100644 --- a/test/acceptance/upload_data.test.js +++ b/test/acceptance/upload_data.test.js @@ -1,6 +1,6 @@ import { test } from '@playwright/test' -test('Upload data', async ({ page }) => { +test('Enter form information', async ({ page }) => { await page.goto('/') await page.getByRole('button', { name: 'Start now' }).click() @@ -20,5 +20,6 @@ test('Upload data', async ({ page }) => { await page.getByText('Upload data').click() const fileChooser = await fileChooserPromise await fileChooser.setFiles('test/testData/conservation-area.csv') + await page.getByRole('button', { name: 'Continue' }).click() }) diff --git a/test/testData/API_RUN_PIPELINE_RESPONSE.json b/test/testData/API_RUN_PIPELINE_RESPONSE.json new file mode 100644 index 00000000..bff90878 --- /dev/null +++ b/test/testData/API_RUN_PIPELINE_RESPONSE.json @@ -0,0 +1,23 @@ +{ + "converted-csv": [ + {"Reference": "CA6", "Name": "Camden Square", "Geometry": "POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))", "Start date": "01/04/1980", "Legislation": "", "Notes": "", "Point": "POINT (-0.130484959448 51.544845663239)", "End date": "", "Document URL": "https://www.camden.gov.uk/camden-square-conservation-area-appraisal-and-management-strategy"}, + {"Reference": "CA20", "Name": "Holly Lodge Estate", "Geometry": "POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))", "Start date": "01/06/1992", "Legislation": "", "Notes": "", "Point": "POINT (-0.150097204178 51.564975754948)", "End date": "", "Document URL": "https://www.camden.gov.uk/holly-lodge-conservation-area"}, + {"Reference": "CA9", "Name": "Dartmouth Park", "Geometry": "POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))", "Start date": "01/06/1992", "Legislation": "", "Notes": "", "Point": "POINT (-0.145442349961 51.559999511433)", "End date": "", "Document URL": "https://www.camden.gov.uk/dartmouth-park-conservation-area-appraisal-and-management-strategy"} + ], + "issue-log": [ + {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "geometry", "issue-type": "OSGB", "value": ""}, + {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "organisation", "issue-type": "default-value", "value": "local-authority-eng:MAL"}, + {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "entry-date", "issue-type": "default-value", "value": "2020-09-14"} + ], + "flattened-csv": { + "entities": [ + {"dataset": "conservation-area", "end-date": "", "entity": "44000001", "entry-date": "2022-04-12", "geometry": "MULTIPOLYGON (((-0.307721 51.724964,-0.307831 51.725040,-0.307919 51.725121,-0.309609 51.724308,-0.309719 51.724396)))", "name": "Shafford Mill", "organisation-entity": "16", "point": "POINT(-0.370182 51.770914)", "prefix": "conservation-area", "reference": "5071", "start-date": "1980-07-31", "typology": "geography"}, + {"dataset": "conservation-area", "end-date": "", "entity": "44000002", "entry-date": "2022-04-12", "geometry": "MULTIPOLYGON (((-0.372417 51.774343,-0.372532 51.774138,-0.372688 51.773884,-0.372776 51.773857,-0.372692 51.773743)))", "name": "Potters Crouch", "organisation-entity": "16", "point": "POINT(-0.383726 51.734475)", "prefix": "conservation-area", "reference": "5074", "start-date": "1977-07-27", "typology": "geography"} + ] + }, + "column-field-log": [ + {"entry-date": "2023-10-03T10:13:45Z", "dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "column": "WKT", "field": "geometry"}, + {"entry-date": "2023-10-03T10:13:45Z", "dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "column": "details", "field": "name"}, + {"entry-date": "2023-10-03T10:13:45Z", "dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "column": "ogc_fid", "field": "reference"} + ] +} diff --git a/test/unit/errorsController.test.js b/test/unit/errorsController.test.js new file mode 100644 index 00000000..01f461c6 --- /dev/null +++ b/test/unit/errorsController.test.js @@ -0,0 +1,102 @@ +import ErrorsController from '../../src/controllers/errorsController.js' + +import { describe, it, expect, vi } from 'vitest' + +import mockApiValue from '../testData/API_RUN_PIPELINE_RESPONSE.json' + +describe('ErrorsController', () => { + const options = { + route: '/errors' + } + const errorsController = new ErrorsController(options) + + it('correctly serves the errors page when the session data is correctly set', async () => { + const session = { + validationResult: mockApiValue, + dataset: 'test-dataset', + 'data-subject': 'test-data-subject' + } + const req = { + sessionModel: { + get: (key) => session[key] + }, + form: { + options: { + } + } + } + const res = {} + const next = vi.fn() + + await errorsController.get(req, res, next) + + const expectedFormValues = { + options: { + rows: [ + { + entryNumber: '1', + columns: { + 'Document URL': { + error: false, + value: 'https://www.camden.gov.uk/holly-lodge-conservation-area' + }, + 'End date': { + error: false, + value: '' + }, + Geometry: { + error: false, + value: 'POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))' + }, + Legislation: { + error: false, + value: '' + }, + Name: { + error: false, + value: 'Holly Lodge Estate' + }, + Notes: { + error: false, + value: '' + }, + Point: { + error: false, + value: 'POINT (-0.150097204178 51.564975754948)' + }, + Reference: { + error: false, + value: 'CA20' + }, + 'Start date': { + error: false, + value: '01/06/1992' + }, + 'entry-date': { + error: 'default-value', + value: undefined + }, + geometry: { + error: 'OSGB', + value: undefined + }, + organisation: { + error: 'default-value', + value: undefined + } + } + } + ], + issueCounts: { + 'entry-date': 1, + geometry: 1, + organisation: 1 + }, + dataset: 'test-dataset', + dataSubject: 'test-data-subject' + } + } + + expect(req.form).toEqual(expectedFormValues) + }) +}) diff --git a/test/unit/errorsPage.test.js b/test/unit/errorsPage.test.js new file mode 100644 index 00000000..05bb97b5 --- /dev/null +++ b/test/unit/errorsPage.test.js @@ -0,0 +1,90 @@ +import { describe, it, expect } from 'vitest' + +import nunjucks from 'nunjucks' +const { govukMarkdown } = require('@x-govuk/govuk-prototype-filters') + +const nunjucksEnv = nunjucks.configure([ + 'src/views', + 'node_modules/govuk-frontend/', + 'node_modules/@x-govuk/govuk-prototype-components/' +], { + dev: true, + noCache: true, + watch: true +}) + +nunjucksEnv.addFilter('govukMarkdown', govukMarkdown) + +describe('errors page', () => { + it('renders the correct number of errors', () => { + const params = { + options: { + rows: [ + { + entryNumber: '1', + columns: { + 'Document URL': { + error: false, + value: 'https://www.camden.gov.uk/holly-lodge-conservation-area' + }, + 'End date': { + error: false, + value: '' + }, + Geometry: { + error: 'fake error', + value: 'POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))' + }, + Legislation: { + error: false, + value: '' + }, + Name: { + error: false, + value: 'Holly Lodge Estate' + }, + Notes: { + error: false, + value: '' + }, + Point: { + error: false, + value: 'POINT (-0.150097204178 51.564975754948)' + }, + Reference: { + error: false, + value: 'CA20' + }, + 'Start date': { + error: false, + value: '01/06/1992' + }, + 'entry-date': { + error: 'default-value', + value: undefined + }, + geometry: { + error: 'OSGB', + value: undefined + }, + organisation: { + error: 'default-value', + value: undefined + } + } + } + ], + issueCounts: { + geography: 1 + }, + dataset: 'Datasubject test', + dataSubject: 'dataset test' + } + } + const html = nunjucks.render('errors.html', params).replace(/(\r\n|\n|\r)/gm, '').replace(/\t/gm, '').replace(/\s+/g, ' ') + + expect(html).toContain('
  • 1 issue, relating to geography
    • ') + expect(html).toContain(' Datasubject test ') + expect(html).toContain('

    POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))

    fake error

    ') + }) +}) diff --git a/test/unit/test.test.js b/test/unit/test.test.js deleted file mode 100644 index c17a1d5b..00000000 --- a/test/unit/test.test.js +++ /dev/null @@ -1,11 +0,0 @@ -// write a hello world test in vitest - -// Path: test/unit/test.test.js - -import { describe, it, expect } from 'vitest' - -describe('hello world', () => { - it('hello world', () => { - expect('hello world').toBe('hello world') - }) -}) diff --git a/test/unit/uploadController.test.js b/test/unit/uploadController.test.js new file mode 100644 index 00000000..9222a783 --- /dev/null +++ b/test/unit/uploadController.test.js @@ -0,0 +1,39 @@ +import UploadController from '../../src/controllers/uploadController.js' + +import { describe, it, expect, vi } from 'vitest' + +import mockApiValue from '../testData/API_RUN_PIPELINE_RESPONSE.json' + +describe('UploadController', () => { + const options = { + route: '/upload' + } + const uploadController = new UploadController(options) + it('posting correct data adds the validation result to the session', async () => { + global.fetch = vi.fn().mockResolvedValue({ + json: vi.fn().mockResolvedValue(mockApiValue) + }) + + expect(uploadController.post).toBeDefined() + + const req = { + file: { + path: 'readme.md', + originalname: 'conservation_area.csv' + }, + sessionModel: { + get: () => 'test', + set: vi.fn() + } + } + const res = { + send: vi.fn(), + redirect: vi.fn() + } + const next = vi.fn() + + await uploadController.post(req, res, next) + + expect(req.sessionModel.set).toHaveBeenCalledWith('validationResult', mockApiValue) + }) +})