diff --git a/config/default.yaml b/config/default.yaml index 905d92b6..8fa6aae6 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -1,6 +1,7 @@ port: 5000 api: { url: https://publish-api.planning.data.gov.uk, + localUrl: http://localhost:8082, port: 8082, validationEndpoint: /api/dataset/validate/file/request, requestTimeout: 50000 diff --git a/src/controllers/errorsController.js b/src/controllers/errorsController.js index 690fe08d..ef9850e3 100644 --- a/src/controllers/errorsController.js +++ b/src/controllers/errorsController.js @@ -37,8 +37,20 @@ class ErrorsController extends PageController { apiResponseData['issue-log'].forEach(issue => { if (issue.severity === severityLevels.error) { const entryNumber = issue['entry-number'] - const rowValues = apiResponseData['converted-csv'][issue['line-number'] - 2] + + // remove any keys from row values where a mapping exists to this column + Object.keys(rowValues).forEach(originalColumnName => { + // if a mapping exists to this column name, remove it from the row values + const mappingToThisColumn = apiResponseData['column-field-log'].find(columnField => columnField.field === originalColumnName) + if (mappingToThisColumn) { + const mappingExistsInRowValues = Object.keys(rowValues).includes(mappingToThisColumn.column) + if (mappingExistsInRowValues) { + delete rowValues[originalColumnName] + } + } + }) + if (!(entryNumber in aggregatedIssues)) { aggregatedIssues[entryNumber] = Object.keys(rowValues).reduce((acc, originalColumnName) => { const mappedColumnName = this.lookupMappedColumnNameFromOriginal(originalColumnName, apiResponseData['column-field-log']) @@ -51,13 +63,14 @@ class ErrorsController extends PageController { } if (entryNumber in aggregatedIssues) { - const columnName = this.lookupMappedColumnNameFromOriginal(issue.field, apiResponseData['column-field-log']) - aggregatedIssues[entryNumber][columnName] = { + const columnName = this.lookupOriginalColumnNameFromMapped(issue.field, apiResponseData['column-field-log']) + + aggregatedIssues[entryNumber][issue.field] = { issue: { type: issue['issue-type'], description: issue.description }, - value: rowValues[issue.field] + value: rowValues[columnName] } } } diff --git a/src/controllers/uploadController.js b/src/controllers/uploadController.js index 49e20221..4de18257 100644 --- a/src/controllers/uploadController.js +++ b/src/controllers/uploadController.js @@ -12,7 +12,8 @@ import hash from '../utils/hasher.js' const upload = multer({ dest: 'uploads/' }) -const apiRoute = config.api.url + config.api.validationEndpoint +const apiEndpoint = process.env.NODE_ENV === 'test' ? config.api.localUrl : config.api.url +const apiRoute = apiEndpoint + config.api.validationEndpoint class UploadController extends PageController { middlewareSetup () { diff --git a/test/testData/API_RUN_PIPELINE_RESPONSE.json b/test/testData/API_RUN_PIPELINE_RESPONSE.json index 4c5110e3..c8cff098 100644 --- a/test/testData/API_RUN_PIPELINE_RESPONSE.json +++ b/test/testData/API_RUN_PIPELINE_RESPONSE.json @@ -5,7 +5,7 @@ {"Name": "Dartmouth Park", "Geometry": "POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))", "Start date": "01/06/1992", "Legislation": "", "Notes": "", "Point": "POINT (-0.145442349961 51.559999511433)", "End date": "", "Document URL": "https://www.camden.gov.uk/dartmouth-park-conservation-area-appraisal-and-management-strategy"} ], "issue-log": [ - {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "Start date", "issue-type": "invalid-value", "description": "invalid-value", "value": "40/04/1980", "severity": "error"}, + {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "start-date", "issue-type": "invalid-value", "description": "invalid-value", "value": "40/04/1980", "severity": "error"}, {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "geometry", "issue-type": "OSGB", "description": "OSGB", "value": "", "severity": "info"}, {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "organisation", "issue-type": "default-value", "description": "default-value", "value": "local-authority-eng:MAL", "severity": "info"}, {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "entry-date", "issue-type": "default-value", "description": "default-value", "value": "2020-09-14", "severity": "info"} diff --git a/test/unit/errorsController.test.js b/test/unit/errorsController.test.js index fcc7bacf..f6fbe61e 100644 --- a/test/unit/errorsController.test.js +++ b/test/unit/errorsController.test.js @@ -139,5 +139,55 @@ describe('ErrorsController', () => { expect(aggregatedIssues).toEqual(expectedAggregatedIssues) }) + + it('returns the correct values when a mapped column and a column with the same name exist', () => { + const mockApiValueModified = { ...mockApiValue } + + mockApiValueModified['converted-csv'][0]['start-date'] = 'this should be discarded as another column maps to start-date' + + const expectedAggregatedIssues = { + 1: { + 'document-URL': { + issue: false, + value: 'https://www.camden.gov.uk/camden-square-conservation-area-appraisal-and-management-strategy' + }, + 'end-date': { + issue: false, + value: '' + }, + geometry: { + issue: false, + value: 'POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))' + }, + legislation: { + issue: false, + value: '' + }, + name: { + issue: false, + value: 'Camden Square' + }, + notes: { + issue: false, + value: '' + }, + point: { + issue: false, + value: 'POINT (-0.130484959448 51.544845663239)' + }, + 'start-date': { + issue: { + type: 'invalid-value', + description: 'invalid-value' + }, + value: '40/04/1980' + } + } + } + + const { aggregatedIssues } = errorsController.getAggregatedErrors(mockApiValueModified) + + expect(aggregatedIssues).toEqual(expectedAggregatedIssues) + }) }) })