Merge pull request #55 from digital-land/bug/emptyValuesWhereDuplicat…

…eColumnsProvided Bug/empty values where duplicate columns provided
digital-land · Feb 12, 2024 · ba9bc48 · ba9bc48
2 parents ed384a0 + ad6f7b1
commit ba9bc48
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 6 deletions.
diff --git a/config/default.yaml b/config/default.yaml
@@ -1,6 +1,7 @@
 port: 5000
 api: {
     url: https://publish-api.planning.data.gov.uk,
+    localUrl: http://localhost:8082,
     port: 8082,
     validationEndpoint: /api/dataset/validate/file/request,
     requestTimeout: 50000

diff --git a/src/controllers/errorsController.js b/src/controllers/errorsController.js
@@ -37,8 +37,20 @@ class ErrorsController extends PageController {
     apiResponseData['issue-log'].forEach(issue => {
       if (issue.severity === severityLevels.error) {
         const entryNumber = issue['entry-number']
-
         const rowValues = apiResponseData['converted-csv'][issue['line-number'] - 2]
+
+        // remove any keys from row values where a mapping exists to this column
+        Object.keys(rowValues).forEach(originalColumnName => {
+          // if a mapping exists to this column name, remove it from the row values
+          const mappingToThisColumn = apiResponseData['column-field-log'].find(columnField => columnField.field === originalColumnName)
+          if (mappingToThisColumn) {
+            const mappingExistsInRowValues = Object.keys(rowValues).includes(mappingToThisColumn.column)
+            if (mappingExistsInRowValues) {
+              delete rowValues[originalColumnName]
+            }
+          }
+        })
+
         if (!(entryNumber in aggregatedIssues)) {
           aggregatedIssues[entryNumber] = Object.keys(rowValues).reduce((acc, originalColumnName) => {
             const mappedColumnName = this.lookupMappedColumnNameFromOriginal(originalColumnName, apiResponseData['column-field-log'])
@@ -51,13 +63,14 @@ class ErrorsController extends PageController {
         }
 
         if (entryNumber in aggregatedIssues) {
-          const columnName = this.lookupMappedColumnNameFromOriginal(issue.field, apiResponseData['column-field-log'])
-          aggregatedIssues[entryNumber][columnName] = {
+          const columnName = this.lookupOriginalColumnNameFromMapped(issue.field, apiResponseData['column-field-log'])
+
+          aggregatedIssues[entryNumber][issue.field] = {
             issue: {
               type: issue['issue-type'],
               description: issue.description
             },
-            value: rowValues[issue.field]
+            value: rowValues[columnName]
           }
         }
       }

diff --git a/src/controllers/uploadController.js b/src/controllers/uploadController.js
@@ -12,7 +12,8 @@ import hash from '../utils/hasher.js'
 
 const upload = multer({ dest: 'uploads/' })
 
-const apiRoute = config.api.url + config.api.validationEndpoint
+const apiEndpoint = process.env.NODE_ENV === 'test' ? config.api.localUrl : config.api.url
+const apiRoute = apiEndpoint + config.api.validationEndpoint
 
 class UploadController extends PageController {
   middlewareSetup () {

diff --git a/test/testData/API_RUN_PIPELINE_RESPONSE.json b/test/testData/API_RUN_PIPELINE_RESPONSE.json
@@ -5,7 +5,7 @@
     {"Name": "Dartmouth Park", "Geometry": "POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))", "Start date": "01/06/1992", "Legislation": "", "Notes": "", "Point": "POINT (-0.145442349961 51.559999511433)", "End date": "", "Document URL": "https://www.camden.gov.uk/dartmouth-park-conservation-area-appraisal-and-management-strategy"}
   ],
   "issue-log": [
-    {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "Start date", "issue-type": "invalid-value", "description": "invalid-value", "value": "40/04/1980", "severity": "error"},
+    {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "start-date", "issue-type": "invalid-value", "description": "invalid-value", "value": "40/04/1980", "severity": "error"},
     {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "geometry", "issue-type": "OSGB", "description": "OSGB", "value": "", "severity": "info"},
     {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "organisation", "issue-type": "default-value", "description": "default-value", "value": "local-authority-eng:MAL", "severity": "info"},
     {"dataset": "conservation-area", "resource": "0b4284077da580a6daea59ee2227f9c7c55a9a45d57ef470d82418a4391ddf9a", "line-number": "2", "entry-number": "1", "field": "entry-date", "issue-type": "default-value", "description": "default-value", "value": "2020-09-14", "severity": "info"}

diff --git a/test/unit/errorsController.test.js b/test/unit/errorsController.test.js
@@ -139,5 +139,55 @@ describe('ErrorsController', () => {
 
       expect(aggregatedIssues).toEqual(expectedAggregatedIssues)
     })
+
+    it('returns the correct values when a mapped column and a column with the same name exist', () => {
+      const mockApiValueModified = { ...mockApiValue }
+
+      mockApiValueModified['converted-csv'][0]['start-date'] = 'this should be discarded as another column maps to start-date'
+
+      const expectedAggregatedIssues = {
+        1: {
+          'document-URL': {
+            issue: false,
+            value: 'https://www.camden.gov.uk/camden-square-conservation-area-appraisal-and-management-strategy'
+          },
+          'end-date': {
+            issue: false,
+            value: ''
+          },
+          geometry: {
+            issue: false,
+            value: 'POLYGON ((-0.125888391245 51.54316508186, -0.125891457623 51.543177267548, -0.125903428774 51.54322160042))'
+          },
+          legislation: {
+            issue: false,
+            value: ''
+          },
+          name: {
+            issue: false,
+            value: 'Camden Square'
+          },
+          notes: {
+            issue: false,
+            value: ''
+          },
+          point: {
+            issue: false,
+            value: 'POINT (-0.130484959448 51.544845663239)'
+          },
+          'start-date': {
+            issue: {
+              type: 'invalid-value',
+              description: 'invalid-value'
+            },
+            value: '40/04/1980'
+          }
+        }
+      }
+
+      const { aggregatedIssues } = errorsController.getAggregatedErrors(mockApiValueModified)
+
+      expect(aggregatedIssues).toEqual(expectedAggregatedIssues)
+    })
   })
 })