From a8609e4d1f1efeb38d5ecac8d9b369ceb27fecf7 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 1 Oct 2024 11:09:13 +0100
Subject: [PATCH 001/109] setup routing and empty middleware chain for table

---
 src/controllers/OrganisationsController.js | 2 ++
 src/middleware/issueTable.middleware.js    | 1 +
 src/routes/organisations.js                | 2 +-
 3 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 src/middleware/issueTable.middleware.js

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 78816dba..e014d4ef 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -1,6 +1,7 @@
 import getDatasetTaskListMiddleware from '../middleware/datasetTaskList.middleware.js'
 import getDatasetOverviewMiddleware from '../middleware/datasetOverview.middleware.js'
 import getIssueDetailsMiddleware from '../middleware/issueDetails.middleware.js'
+import getIssueTableMiddleware from '../middleware/issueTable.middleware.js'
 import getOrganisationsMiddleware from '../middleware/organisations.middleware.js'
 import getGetStartedMiddleware from '../middleware/getStarted.middleware.js'
 import getOverviewMiddleware from '../middleware/overview.middleware.js'
@@ -10,6 +11,7 @@ const organisationsController = {
   getDatasetTaskListMiddleware,
   getDatasetOverviewMiddleware,
   getIssueDetailsMiddleware,
+  getIssueTableMiddleware,
   getGetStartedMiddleware,
   getOverviewMiddleware
 }
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
new file mode 100644
index 00000000..9859f079
--- /dev/null
+++ b/src/middleware/issueTable.middleware.js
@@ -0,0 +1 @@
+export default []
diff --git a/src/routes/organisations.js b/src/routes/organisations.js
index 69c68332..46de1cf5 100644
--- a/src/routes/organisations.js
+++ b/src/routes/organisations.js
@@ -6,7 +6,7 @@ const router = express.Router()
 router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStartedMiddleware)
 router.get('/:lpa/:dataset/overview', OrganisationsController.getDatasetOverviewMiddleware)
 router.get('/:lpa/:dataset/:issue_type/:issue_field/:pageNumber', OrganisationsController.getIssueDetailsMiddleware)
-router.get('/:lpa/:dataset/:issue_type/:issue_field', OrganisationsController.getIssueDetailsMiddleware)
+router.get('/:lpa/:dataset/:issue_type/:issue_field', OrganisationsController.getIssueTableMiddleware)
 router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskListMiddleware)
 router.get('/:lpa', OrganisationsController.getOverviewMiddleware)
 router.get('/', OrganisationsController.getOrganisationsMiddleware)

From abb2c8f3edd6bba1eb3106c6d31b53bc15ccd9d9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 1 Oct 2024 15:20:34 +0100
Subject: [PATCH 002/109] get basic page rendering with table populated with
 mock data

---
 src/middleware/issueTable.middleware.js |  88 +++++++++++++++++++-
 src/routes/schemas.js                   |  29 +++++++
 src/views/organisations/issueTable.html | 105 ++++++++++++++++++++++++
 3 files changed, 221 insertions(+), 1 deletion(-)
 create mode 100644 src/views/organisations/issueTable.html

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 9859f079..70aa2470 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1 +1,87 @@
-export default []
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
+
+const validateIssueTableQueryParams = (req, res, next) => {
+  next()
+}
+
+const fetchEntitiesWithIssues = (req, res, next) => {
+  next()
+}
+
+const prepareIssueTableTemplateParams = (req, res, next) => {
+  const { issue_type: issueType } = req.params
+
+  req.templateParams = {
+    organisation: req.orgInfo,
+    dataset: req.dataset,
+    errorHeading: 'error Heading (ToDo)',
+    issueItems: [],
+    issueType,
+    tableParams: {
+      columns: ['col1', 'col2', 'col3'],
+      rows: [
+        {
+          columns: {
+            field1: {
+              error: undefined,
+              value: 'value11'
+            },
+            field2: {
+              error: {
+                message: 'error in value12'
+              },
+              value: 'value12'
+            },
+            field3: {
+              error: undefined,
+              value: 'value13'
+            }
+          }
+        },
+        {
+          columns: {
+            field1: {
+              error: undefined,
+              value: 'value21'
+            },
+            field2: {
+              error: undefined,
+              value: 'value22'
+            },
+            field3: {
+              error: {
+                message: 'error in value23'
+              },
+              value: 'value23'
+            }
+          }
+        }
+      ],
+      fields: ['field1', 'field2', 'field3']
+    }
+  }
+  next()
+}
+
+const getIssueTable = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/issueTable.html',
+  handlerName: 'getIssueTable'
+})
+
+export default [
+  validateIssueTableQueryParams,
+  parallel([
+    fetchOrgInfo,
+    fetchDatasetInfo
+  ]),
+  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  parallel([
+    fetchEntitiesWithIssues,
+    fetchEntityCount
+  ]),
+  prepareIssueTableTemplateParams,
+  getIssueTable,
+  logPageError
+]
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 258ee427..7c813296 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -41,6 +41,22 @@ export const datasetStatusEnum = {
 const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
 const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
 
+const tableParams = v.strictObject({
+  columns: v.array(NonEmptyString),
+  rows: v.array(v.strictObject({
+    columns: v.objectWithRest(
+      {},
+      v.strictObject({
+        error: v.optional(v.object({
+          message: v.string()
+        })),
+        value: v.string()
+      })
+    )
+  })),
+  fields: v.array(NonEmptyString)
+})
+
 export const OrgOverviewPage = v.strictObject({
   organisation: OrgField,
   datasets: v.array(v.strictObject({
@@ -162,6 +178,18 @@ export const OrgIssueDetails = v.strictObject({
   pageNumber: v.integer()
 })
 
+export const OrgIssueTable = v.strictObject({
+  organisation: OrgField,
+  dataset: DatasetNameField,
+  errorHeading: v.optional(NonEmptyString),
+  issueItems: v.array(v.strictObject({
+    html: v.string(),
+    href: v.url()
+  })),
+  issueType: NonEmptyString,
+  tableParams
+})
+
 export const CheckAnswers = v.strictObject({
   values: v.strictObject({
     lpa: NonEmptyString,
@@ -213,6 +241,7 @@ export const templateSchema = new Map([
   ['organisations/datasetTaskList.html', OrgDatasetTaskList],
   ['organisations/http-error.html', OrgEndpointError],
   ['organisations/issueDetails.html', OrgIssueDetails],
+  ['organisations/issueTable.html', OrgIssueTable],
 
   ['errorPages/503', UptimeParams],
   ['errorPages/500', ErrorParams],
diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
new file mode 100644
index 00000000..40408950
--- /dev/null
+++ b/src/views/organisations/issueTable.html
@@ -0,0 +1,105 @@
+{% extends "layouts/main.html" %}
+
+{% from 'govuk/components/error-summary/macro.njk' import govukErrorSummary %}
+{% from "govuk/components/summary-list/macro.njk" import govukSummaryList %}
+{% from "govuk/components/pagination/macro.njk" import govukPagination %}
+{% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %}
+
+{% from "../components/table.html" import table %}
+
+{% set serviceType = 'Submit'%}
+
+{% if issueEntitiesCount > 1 %}
+  {% set pageName %}{{organisation.name}} - {{dataset.name}} - Issues (Page {{pageNumber}} of {{issueEntitiesCount}}){% endset %}
+{% else %}
+  {% set pageName %}{{organisation.name}} - {{dataset.name}} - Issues{% endset %}
+{%endif%}
+
+{% block beforeContent %}
+
+{{ super() }}
+
+{{ govukBreadcrumbs({
+  items: [
+    {
+      text: "Home",
+      href: "/"
+    },
+    {
+      text: "Organisations",
+      href: '/organisations'
+    },
+    {
+      text: organisation.name,
+      href: '/organisations/' + organisation.organisation
+    },
+    {
+      text: dataset.name | capitalize,
+      href: '/organisations/' + organisation.organisation + '/' + dataset.dataset
+    },
+    {
+      text: issueType | capitalize
+    }
+  ]
+}) }}
+
+{% endblock %}
+
+{% block content %}
+
+
+<div class="govuk-grid-row">
+  {% include "includes/_dataset-page-header.html" %}
+</div>
+
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    {{ govukErrorSummary({
+      titleText: errorHeading if errorHeading else 'There is a problem',
+      errorList: issueItems
+    }) }}
+
+    {% if false and entry.geometries and entry.geometries.length %}
+      <div
+        id="map"
+        class="app-map"
+        role="region"
+        aria-label="Static map showing {{dataset.name}} for {{organisation.name}}.">
+      </div>
+    {% endif %}
+
+    {{ table(tableParams) }}
+
+  </div>
+</div>
+
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <h3 class="govuk-heading-m">
+      How to improve {{ organisation.name }}’s data
+    </h3>
+
+    <ol class="govuk-list govuk-list--number">
+      <li>Fix the errors indicated</li>
+      <li>Use the <a href="/check">check service</a> to make sure the data meets
+        the standard</li>
+      <li>Publish the updated data on the data URL</li>
+    </ol>
+  </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+  {{ super() }}
+  {% if entry.geometries and entry.geometries.length %}
+    <link href='https://unpkg.com/maplibre-gl@latest/dist/maplibre-gl.css' rel='stylesheet' />
+    <script>
+      window.serverContext = {
+        containerId: 'map',
+        geometries: {{ entry.geometries | dump | safe }},
+        mapType: "static"
+      }
+    </script>
+    <script src="/public/js/map.bundle.js"></script>
+  {% endif %}
+{% endblock %}
\ No newline at end of file

From 04912343be1b2884b416ef8b368f016571ee5343 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 12:57:47 +0100
Subject: [PATCH 003/109] now get basic table

---
 src/middleware/common.middleware.js          |  14 +++
 src/middleware/datasetOverview.middleware.js |  16 +--
 src/middleware/issueTable.middleware.js      | 101 +++++++++----------
 src/services/performanceDbApi.js             |  29 ++++++
 4 files changed, 94 insertions(+), 66 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 97e480ec..179b03e9 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -3,6 +3,7 @@ import { types } from '../utils/logging.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { fetchOne, FetchOptions, FetchOneFallbackPolicy, fetchMany } from './middleware.builders.js'
 import * as v from 'valibot'
+import json5 from 'json5'
 
 /**
  * Middleware. Set `req.handlerName` to a string that will identify
@@ -92,3 +93,16 @@ export const fetchLpaDatasetIssues = fetchMany({
   query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
   result: 'issues'
 })
+
+export const fetchSpecification = fetchOne({
+  query: ({ req }) => `select * from specification WHERE specification = '${req.dataset.collection}'`,
+  result: 'specification'
+})
+
+export const pullOutDatasetSpecification = (req, res, next) => {
+  const { specification } = req
+  const collectionSpecifications = json5.parse(specification.json)
+  const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
+  req.specification = datasetSpecification
+  next()
+}
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index 503540db..69657309 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,8 +1,7 @@
-import { fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, isResourceAccessible, isResourceIdInParams, logPageError, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
 import { fetchOne, fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
 import performanceDbApi from '../services/performanceDbApi.js'
-import json5 from 'json5'
 
 const fetchColumnSummary = fetchMany({
   query: ({ params }) => `select * from endpoint_dataset_resource_summary
@@ -14,19 +13,6 @@ const fetchColumnSummary = fetchMany({
   dataset: FetchOptions.performanceDb
 })
 
-const fetchSpecification = fetchOne({
-  query: ({ req }) => `select * from specification WHERE specification = '${req.dataset.collection}'`,
-  result: 'specification'
-})
-
-export const pullOutDatasetSpecification = (req, res, next) => {
-  const { specification } = req
-  const collectionSpecifications = json5.parse(specification.json)
-  const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
-  req.specification = datasetSpecification
-  next()
-}
-
 const fetchSources = fetchMany({
   query: ({ params }) => `
     select rhe.endpoint, rhe.endpoint_url, rhe.status, rhe.exception, rhe.resource, rhe.latest_log_entry_date, rhe.endpoint_entry_date, rhe.endpoint_end_date, rhe.resource_start_date, rhe.resource_end_date, s.documentation_url
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 70aa2470..f0499c25 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,16 +1,57 @@
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams } from './common.middleware.js'
-import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
+import performanceDbApi from '../services/performanceDbApi.js'
+import { fetchDatasetInfo, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
 
 const validateIssueTableQueryParams = (req, res, next) => {
   next()
 }
 
-const fetchEntitiesWithIssues = (req, res, next) => {
-  next()
-}
+// given an lpa and a dataset, we want to get all the entities with issues, and their accompanying issues
+const fetchEntitiesWithIssues = fetchMany({
+  query: ({ req, params }) => performanceDbApi.entitiesAndIssuesQuery(req.resource.resource),
+  result: 'entitiesWithIssues',
+  dataset: FetchOptions.fromParams
+})
 
 const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType } = req.params
+  const { entitiesWithIssues, specification } = req
+
+  const tableParams = {
+    columns: specification.fields.map(field => field.field),
+    fields: specification.fields.map(field => field.field),
+    rows: entitiesWithIssues.map(entity => {
+      const columns = {}
+
+      specification.fields.forEach(fieldObject => {
+        const { field } = fieldObject
+        if (entity[field]) {
+          columns[field] = { value: entity[field] }
+        } else {
+          columns[field] = { value: '' }
+        }
+      })
+
+      let issues
+      try {
+        issues = JSON.parse(entity.issues)
+      } catch (e) {
+        console.log(e)
+      }
+
+      Object.entries(issues).forEach(([field, issueType]) => {
+        if (columns[field]) {
+          columns[field].error = { message: issueType }
+        } else {
+          columns[field] = { value: '', error: { message: issueType } }
+        }
+      })
+
+      return {
+        columns
+      }
+    })
+  }
 
   req.templateParams = {
     organisation: req.orgInfo,
@@ -18,48 +59,7 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
     errorHeading: 'error Heading (ToDo)',
     issueItems: [],
     issueType,
-    tableParams: {
-      columns: ['col1', 'col2', 'col3'],
-      rows: [
-        {
-          columns: {
-            field1: {
-              error: undefined,
-              value: 'value11'
-            },
-            field2: {
-              error: {
-                message: 'error in value12'
-              },
-              value: 'value12'
-            },
-            field3: {
-              error: undefined,
-              value: 'value13'
-            }
-          }
-        },
-        {
-          columns: {
-            field1: {
-              error: undefined,
-              value: 'value21'
-            },
-            field2: {
-              error: undefined,
-              value: 'value22'
-            },
-            field3: {
-              error: {
-                message: 'error in value23'
-              },
-              value: 'value23'
-            }
-          }
-        }
-      ],
-      fields: ['field1', 'field2', 'field3']
-    }
+    tableParams
   }
   next()
 }
@@ -77,10 +77,9 @@ export default [
     fetchDatasetInfo
   ]),
   fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
-  parallel([
-    fetchEntitiesWithIssues,
-    fetchEntityCount
-  ]),
+  fetchEntitiesWithIssues,
+  fetchSpecification,
+  pullOutDatasetSpecification,
   prepareIssueTableTemplateParams,
   getIssueTable,
   logPageError
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 6b6de303..0ad7acba 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -402,5 +402,34 @@ ORDER BY
     const query = this.entityCountQuery(orgEntity)
     const result = await datasette.runQuery(query, dataset)
     return result.formattedData[0].entity_count
+  },
+
+  entitiesAndIssuesQuery (resource) {
+    return /* sql */ `
+      SELECT
+          e.*,
+          fr.entry_number,
+          '{' || GROUP_CONCAT(
+            '"' || i.field || '": "' || i.issue_type || '"',
+            ',' || CHAR(10)
+          ) || '}' AS issues
+        from
+          entity e
+          LEFT JOIN (
+            SELECT
+              DISTINCT fr.entry_number,
+              f.entity
+            FROM
+              fact_resource fr
+              INNER JOIN fact f ON fr.fact = f.fact
+            WHERE
+              fr.resource = '${resource}'
+          ) fr ON fr.entity = e.entity
+          LEFT JOIN issue i ON i.entry_number = fr.entry_number
+        WHERE
+          i.resource = '${resource}'
+        GROUP BY
+          (e.entity)
+      `
   }
 }

From 1be5202d832296ee84beeeb698bbd8b7396b6abd Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 13:36:44 +0100
Subject: [PATCH 004/109] make table wider

---
 src/views/organisations/issueTable.html | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
index 40408950..7e7bf86d 100644
--- a/src/views/organisations/issueTable.html
+++ b/src/views/organisations/issueTable.html
@@ -68,8 +68,13 @@
       </div>
     {% endif %}
 
-    {{ table(tableParams) }}
+    
+  </div>
+</div>
 
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-full">
+    {{ table(tableParams) }}
   </div>
 </div>
 

From 03929f093ad41030a520c9241d44adfc2b2e260d Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 13:37:04 +0100
Subject: [PATCH 005/109] fix bug in fetchEntityCount

---
 src/middleware/common.middleware.js          |  2 +-
 src/middleware/datasetOverview.middleware.js | 11 ++---------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 179b03e9..c8f49bed 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -57,7 +57,7 @@ export const takeResourceIdFromParams = (req) => {
 }
 
 export const fetchEntityCount = fetchOne({
-  query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource),
+  query: ({ req }) => performanceDbApi.entityCountQuery(req.orgInfo.entity),
   result: 'entityCount',
   dataset: FetchOptions.fromParams,
   fallbackPolicy: FetchOneFallbackPolicy.continue
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index 69657309..730a0993 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,7 +1,6 @@
-import { fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
-import { fetchOne, fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
-import performanceDbApi from '../services/performanceDbApi.js'
 
 const fetchColumnSummary = fetchMany({
   query: ({ params }) => `select * from endpoint_dataset_resource_summary
@@ -24,12 +23,6 @@ const fetchSources = fetchMany({
   result: 'sources'
 })
 
-const fetchEntityCount = fetchOne({
-  query: ({ req }) => performanceDbApi.entityCountQuery(req.orgInfo.entity),
-  result: 'entityCount',
-  dataset: FetchOptions.fromParams
-})
-
 export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
   const { orgInfo, specification, columnSummary, entityCount, sources, dataset } = req
 

From 9029aea7d7526aac596ed0bc98e9e0662bd67da5 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 13:37:20 +0100
Subject: [PATCH 006/109] have the table generate the correct heading

---
 src/middleware/issueTable.middleware.js | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index f0499c25..77e029ee 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,5 +1,5 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
 
 const validateIssueTableQueryParams = (req, res, next) => {
@@ -14,8 +14,9 @@ const fetchEntitiesWithIssues = fetchMany({
 })
 
 const prepareIssueTableTemplateParams = (req, res, next) => {
-  const { issue_type: issueType } = req.params
-  const { entitiesWithIssues, specification } = req
+  const { issue_type: issueType, issue_field: issueField } = req.params
+  const { entitiesWithIssues, specification, entityCount: entityCountRow } = req
+  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
   const tableParams = {
     columns: specification.fields.map(field => field.field),
@@ -53,10 +54,12 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
     })
   }
 
+  const errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: entitiesWithIssues.length, entityCount, field: issueField }, true)
+
   req.templateParams = {
     organisation: req.orgInfo,
     dataset: req.dataset,
-    errorHeading: 'error Heading (ToDo)',
+    errorHeading,
     issueItems: [],
     issueType,
     tableParams
@@ -80,6 +83,7 @@ export default [
   fetchEntitiesWithIssues,
   fetchSpecification,
   pullOutDatasetSpecification,
+  fetchEntityCount,
   prepareIssueTableTemplateParams,
   getIssueTable,
   logPageError

From f41e52e50c7d57b5983df87498573852f1746079 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 14:12:10 +0100
Subject: [PATCH 007/109] standardise pagination link types to number instead
 of item

---
 src/middleware/issueDetails.middleware.js            | 2 +-
 test/unit/middleware/issueDetails.middleware.test.js | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 62693b87..e84e402e 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -250,7 +250,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
       }
     } else {
       return {
-        type: 'item',
+        type: 'number',
         number: item,
         href: `${BaseSubpath}${item}`,
         current: pageNumber === parseInt(item)
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 74e9923b..143d049c 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -103,7 +103,7 @@ describe('issueDetails.middleware.js', () => {
             current: true,
             href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
             number: 1,
-            type: 'item'
+            type: 'number'
           }]
         },
         issueEntitiesCount: 1,
@@ -212,7 +212,7 @@ describe('issueDetails.middleware.js', () => {
             current: true,
             href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
             number: 1,
-            type: 'item'
+            type: 'number'
           }]
         },
         issueEntitiesCount: 1,

From 04339f7f1c3c5fd45355dc340f2291291708b13a Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 14:12:50 +0100
Subject: [PATCH 008/109] enable table to accept html for cell values

---
 src/views/components/table.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/views/components/table.html b/src/views/components/table.html
index 8147f7b0..92afd37e 100644
--- a/src/views/components/table.html
+++ b/src/views/components/table.html
@@ -16,14 +16,14 @@
                         {% for field in params.fields %}
                             {% set columnData = row.columns[field] %}
                             <td class="govuk-table__cell app-wrap">
-                                
+                                {% set cellText = columnData.html | safe if columnData.html else columnData.value%}
                                 {% if columnData.error %}
                                     {{ govukInsetText({
                                         classes: "app-inset-text---error",
-                                        html: '<p class="app-inset-text__value">'+columnData.value | striptags +'</p> <p class="app-inset-text__error">'+columnData.error.message | prettifyColumnName +'</p>'
+                                        html: '<p class="app-inset-text__value">'+cellText | striptags +'</p> <p class="app-inset-text__error">'+columnData.error.message | prettifyColumnName +'</p>'
                                     }) }}
-                                {% elseif columnData.value %}
-                                    {{ columnData.value }}
+                                {% elseif cellText %}
+                                    {{ cellText }}
                                 {% endif %}
                             </td>
                         {% endfor %}

From 0e54fc1bd8ab563455438a219f8364d57e646901 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 14:13:15 +0100
Subject: [PATCH 009/109] enable schema to accept html for cell values

---
 src/routes/schemas.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 7c813296..74795129 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -46,11 +46,12 @@ const tableParams = v.strictObject({
   rows: v.array(v.strictObject({
     columns: v.objectWithRest(
       {},
-      v.strictObject({
+      v.object({
         error: v.optional(v.object({
           message: v.string()
         })),
-        value: v.string()
+        value: v.optional(v.string()),
+        html: v.optional(v.string())
       })
     )
   })),

From de4933f51c3fcf866614694f7ef48056e4363695 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 14:13:35 +0100
Subject: [PATCH 010/109] have issue table reference values be links to entity
 issue details

---
 src/middleware/issueTable.middleware.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 77e029ee..75ff92f6 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -14,19 +14,23 @@ const fetchEntitiesWithIssues = fetchMany({
 })
 
 const prepareIssueTableTemplateParams = (req, res, next) => {
-  const { issue_type: issueType, issue_field: issueField } = req.params
+  const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
   const { entitiesWithIssues, specification, entityCount: entityCountRow } = req
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
   const tableParams = {
     columns: specification.fields.map(field => field.field),
     fields: specification.fields.map(field => field.field),
-    rows: entitiesWithIssues.map(entity => {
+    rows: entitiesWithIssues.map((entity, index) => {
       const columns = {}
 
       specification.fields.forEach(fieldObject => {
         const { field } = fieldObject
-        if (entity[field]) {
+        if (field === 'reference') {
+          const entityPageNumber = index + 1
+          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/${entityPageNumber}`
+          columns[field] = { html: `<a href="${entityLink}">${entity[field]}</a>` }
+        } else if (entity[field]) {
           columns[field] = { value: entity[field] }
         } else {
           columns[field] = { value: '' }

From 99409da5db6c4344546ea14ea3c06e0a8226effc Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 16:07:30 +0100
Subject: [PATCH 011/109] move test over

---
 .../unit/middleware/common.middleware.test.js | 30 +++++++++++++++++++
 .../datasetOverview.middleware.test.js        | 17 +----------
 2 files changed, 31 insertions(+), 16 deletions(-)
 create mode 100644 test/unit/middleware/common.middleware.test.js

diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
new file mode 100644
index 00000000..64d92360
--- /dev/null
+++ b/test/unit/middleware/common.middleware.test.js
@@ -0,0 +1,30 @@
+import { describe, it, expect } from 'vitest'
+import { pullOutDatasetSpecification } from '../../../src/middleware/common.middleware'
+
+describe('pullOutDatasetSpecification', () => {
+  const req = {
+    params: {
+      lpa: 'mock-lpa',
+      dataset: 'mock-dataset'
+    },
+    dataset: {
+      name: 'mock dataset',
+      dataset: 'mock-dataset',
+      collection: 'mock-collection'
+    }
+  }
+  const res = {}
+
+  it('', () => {
+    const reqWithSpecification = {
+      ...req,
+      specification: {
+        json: JSON.stringify([
+          { dataset: 'mock-dataset', foo: 'bar' }
+        ])
+      }
+    }
+    pullOutDatasetSpecification(reqWithSpecification, res, () => {})
+    expect(reqWithSpecification.specification).toEqual({ dataset: 'mock-dataset', foo: 'bar' })
+  })
+})
diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js
index 2380faa0..71e89715 100644
--- a/test/unit/middleware/datasetOverview.middleware.test.js
+++ b/test/unit/middleware/datasetOverview.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest'
-import { prepareDatasetOverviewTemplateParams, pullOutDatasetSpecification } from '../../../src/middleware/datasetOverview.middleware.js'
+import { prepareDatasetOverviewTemplateParams } from '../../../src/middleware/datasetOverview.middleware.js'
 
 describe('Dataset Overview Middleware', () => {
   const req = {
@@ -15,21 +15,6 @@ describe('Dataset Overview Middleware', () => {
   }
   const res = {}
 
-  describe('pullOutDatasetSpecification', () => {
-    it('', () => {
-      const reqWithSpecification = {
-        ...req,
-        specification: {
-          json: JSON.stringify([
-            { dataset: 'mock-dataset', foo: 'bar' }
-          ])
-        }
-      }
-      pullOutDatasetSpecification(reqWithSpecification, res, () => {})
-      expect(reqWithSpecification.specification).toEqual({ dataset: 'mock-dataset', foo: 'bar' })
-    })
-  })
-
   describe('prepareDatasetOverviewTemplateParams', () => {
     it('should prepare template params for dataset overview', async () => {
       const reqWithResults = {

From 709b19efe4cdbc96c72ff75898f5c432e9c46aac Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 2 Oct 2024 16:17:10 +0100
Subject: [PATCH 012/109] add some new tests for issueTablePage (needs more
 work)

---
 .../organisations/issueTablePage.test.js      | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 test/unit/views/organisations/issueTablePage.test.js

diff --git a/test/unit/views/organisations/issueTablePage.test.js b/test/unit/views/organisations/issueTablePage.test.js
new file mode 100644
index 00000000..8b3e2424
--- /dev/null
+++ b/test/unit/views/organisations/issueTablePage.test.js
@@ -0,0 +1,86 @@
+import { describe, it, expect } from 'vitest'
+import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
+import { JSDOM } from 'jsdom'
+import { runGenericPageTests } from './generic-page.js'
+import config from '../../config/index.js'
+import mocker from '../utils/mocker.js'
+import { OrgIssueTable } from '../../../../src/routes/schemas.js'
+
+const nunjucks = setupNunjucks({})
+
+const seed = new Date().getTime()
+
+describe(`issueDetails.html(seed: ${seed})`, () => {
+  const params = mocker(OrgIssueTable, seed)
+
+  params.issueEntitiesCount = undefined
+
+  const html = nunjucks.render('organisations/issueTable.html', params)
+  const dom = new JSDOM(html)
+  const document = dom.window.document
+
+  runGenericPageTests(html, {
+    pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues - ${config.serviceNames.submit}`,
+    breadcrumbs: [
+      { text: 'Home', href: '/' },
+      { text: 'Organisations', href: '/organisations' },
+      { text: params.organisation.name, href: `/organisations/${params.organisation.organisation}` },
+      { text: params.dataset.name, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` },
+      { text: 'mock issue' }
+    ]
+  })
+
+  it('Renders the correct headings', () => {
+    expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name)
+    expect(document.querySelector('h1').textContent).toContain(params.dataset.name)
+  })
+
+  describe('error summary', () => {
+    it('should render the correct heading', () => {
+      expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorHeading || 'There is a problem')
+    })
+
+    it('should render the correct heading if none is supplied', () => {
+      const noErrorHeadingPageHtml = nunjucks.render('organisations/issueDetails.html', {
+        ...params,
+        errorHeading: undefined
+      })
+
+      const domNoErrorHeading = new JSDOM(noErrorHeadingPageHtml)
+      const documentNoErrorHeading = domNoErrorHeading.window.document
+
+      expect(documentNoErrorHeading.querySelector('.govuk-error-summary__title').textContent).toContain('There is a problem')
+    })
+
+    it('should render the issue items', () => {
+      const issueList = document.querySelector('.govuk-error-summary__list')
+      const issueItemElements = [...issueList.children]
+      expect(issueItemElements.length).toBe(params.issueItems.length)
+
+      issueItemElements.forEach((element, index) => {
+        expect(element.textContent).toContain(params.issueItems[index].html)
+      })
+    })
+  })
+
+  describe('Table', () => {
+    it('should render the table with the correct values', () => {
+
+    })
+
+    it('should render the references as links to the issue details view', () => {
+
+    })
+
+    it('should render the table with the correct errors', () => {
+
+    })
+  })
+
+  describe('multi page', () => {
+    // runGenericPageTests
+
+    it('correctly renders the pagination component', () => {
+    })
+  })
+})

From bb2a16e8547d9eef5258095c3fe1c66a28df1429 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 3 Oct 2024 09:57:58 +0100
Subject: [PATCH 013/109] change issue entity view path

---
 src/middleware/issueDetails.middleware.js | 2 +-
 src/middleware/issueTable.middleware.js   | 4 ++--
 src/routes/organisations.js               | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index e84e402e..684f6612 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -189,7 +189,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   let errorHeading
   let issueItems
 
-  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
+  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entity/`
 
   if (Object.keys(issuesByEntryNumber).length < entityCount) {
     errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 75ff92f6..4bfc7903 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -3,10 +3,10 @@ import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo,
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
 
 const validateIssueTableQueryParams = (req, res, next) => {
+  // ToDo
   next()
 }
 
-// given an lpa and a dataset, we want to get all the entities with issues, and their accompanying issues
 const fetchEntitiesWithIssues = fetchMany({
   query: ({ req, params }) => performanceDbApi.entitiesAndIssuesQuery(req.resource.resource),
   result: 'entitiesWithIssues',
@@ -28,7 +28,7 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
         const { field } = fieldObject
         if (field === 'reference') {
           const entityPageNumber = index + 1
-          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/${entityPageNumber}`
+          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entity/${entityPageNumber}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field]}</a>` }
         } else if (entity[field]) {
           columns[field] = { value: entity[field] }
diff --git a/src/routes/organisations.js b/src/routes/organisations.js
index 46de1cf5..80bd1498 100644
--- a/src/routes/organisations.js
+++ b/src/routes/organisations.js
@@ -5,7 +5,7 @@ const router = express.Router()
 
 router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStartedMiddleware)
 router.get('/:lpa/:dataset/overview', OrganisationsController.getDatasetOverviewMiddleware)
-router.get('/:lpa/:dataset/:issue_type/:issue_field/:pageNumber', OrganisationsController.getIssueDetailsMiddleware)
+router.get('/:lpa/:dataset/:issue_type/:issue_field/entity/:pageNumber', OrganisationsController.getIssueDetailsMiddleware)
 router.get('/:lpa/:dataset/:issue_type/:issue_field', OrganisationsController.getIssueTableMiddleware)
 router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskListMiddleware)
 router.get('/:lpa', OrganisationsController.getOverviewMiddleware)

From a8d00798b79256a41ca2deda812618fa71092f4d Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 3 Oct 2024 10:01:40 +0100
Subject: [PATCH 014/109] make pageNumber param optional

---
 src/routes/organisations.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/routes/organisations.js b/src/routes/organisations.js
index 80bd1498..c9290c22 100644
--- a/src/routes/organisations.js
+++ b/src/routes/organisations.js
@@ -5,8 +5,8 @@ const router = express.Router()
 
 router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStartedMiddleware)
 router.get('/:lpa/:dataset/overview', OrganisationsController.getDatasetOverviewMiddleware)
-router.get('/:lpa/:dataset/:issue_type/:issue_field/entity/:pageNumber', OrganisationsController.getIssueDetailsMiddleware)
-router.get('/:lpa/:dataset/:issue_type/:issue_field', OrganisationsController.getIssueTableMiddleware)
+router.get('/:lpa/:dataset/:issue_type/:issue_field/entity/:pageNumber?', OrganisationsController.getIssueDetailsMiddleware)
+router.get('/:lpa/:dataset/:issue_type/:issue_field/:pageNumber?', OrganisationsController.getIssueTableMiddleware)
 router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskListMiddleware)
 router.get('/:lpa', OrganisationsController.getOverviewMiddleware)
 router.get('/', OrganisationsController.getOrganisationsMiddleware)

From 0a27f67a5fea3380f2508cbbdecb0efba42a3e59 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 3 Oct 2024 10:29:34 +0100
Subject: [PATCH 015/109] add validateIssueTableQueryParams middleware

---
 src/middleware/issueTable.middleware.js | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 4bfc7903..ca237637 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,11 +1,20 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
+import * as v from 'valibot'
 
-const validateIssueTableQueryParams = (req, res, next) => {
-  // ToDo
-  next()
-}
+export const IssueTableQueryParams = v.object({
+  lpa: v.string(),
+  dataset: v.string(),
+  issue_type: v.string(),
+  issue_field: v.string(),
+  pageNumber: v.optional(v.string()),
+  resourceId: v.optional(v.string())
+})
+
+const validateIssueTableQueryParams = validateQueryParams.bind({
+  schema: IssueTableQueryParams
+})
 
 const fetchEntitiesWithIssues = fetchMany({
   query: ({ req, params }) => performanceDbApi.entitiesAndIssuesQuery(req.resource.resource),

From 924610ad3e5efcb8c26dbde06815818e8d1c54ab Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 10:42:34 +0100
Subject: [PATCH 016/109] add pagination to table view

---
 src/middleware/common.middleware.js       | 19 ++++++
 src/middleware/issueDetails.middleware.js | 35 ++---------
 src/middleware/issueTable.middleware.js   | 76 ++++++++++++++++++++---
 src/routes/organisations.js               |  2 +-
 src/routes/schemas.js                     | 48 +++++++-------
 src/services/performanceDbApi.js          |  4 +-
 src/views/organisations/issueDetails.html |  6 +-
 src/views/organisations/issueTable.html   |  9 +++
 8 files changed, 137 insertions(+), 62 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index c8f49bed..6f3f5100 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -106,3 +106,22 @@ export const pullOutDatasetSpecification = (req, res, next) => {
   req.specification = datasetSpecification
   next()
 }
+
+/**
+ *
+ * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
+ *
+ * Requires `req.resource.resource`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+export async function fetchIssueEntitiesCount (req, res, next) {
+  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { resource: resourceId } = req.resource
+  console.assert(resourceId, 'missng resource id')
+  const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId)
+  req.issueEntitiesCount = parseInt(issueEntitiesCount)
+  next()
+}
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 684f6612..f5f029a3 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,7 +1,7 @@
 import performanceDbApi from '../services/performanceDbApi.js'
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
@@ -79,40 +79,14 @@ async function reformatIssuesToBeByEntryNumber (req, res, next) {
  *
  */
 async function fetchEntry (req, res, next) {
-  const { dataset: datasetId, pageNumber } = req.params
-  const { issuesByEntryNumber } = req
-  const pageNum = pageNumber ? parseInt(pageNumber) : 1
-  req.pageNumber = pageNum
-
-  // look at issue Entries and get the index of that entry - 1
-
-  const entityNum = Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
+  const { dataset: datasetId, entryNumber } = req.params
 
   req.entryData = await performanceDbApi.getEntry(
     req.resource.resource,
-    entityNum,
+    entryNumber,
     datasetId
   )
-  req.entryNumber = entityNum
-  next()
-}
-
-/**
- *
- * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
- *
- * Requires `req.resource.resource`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchIssueEntitiesCount (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  console.assert(resourceId, 'missng resource id')
-  const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId)
-  req.issueEntitiesCount = parseInt(issueEntitiesCount)
+  req.entryNumber = entryNumber
   next()
 }
 
@@ -266,6 +240,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     issueItems,
     entry,
     issueType,
+    issueField,
     pagination: paginationObj,
     issueEntitiesCount,
     pageNumber
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index ca237637..dc635b46 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,8 +1,11 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { pagination } from '../utils/pagination.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
+const paginationPageLength = 50
+
 export const IssueTableQueryParams = v.object({
   lpa: v.string(),
   dataset: v.string(),
@@ -16,15 +19,28 @@ const validateIssueTableQueryParams = validateQueryParams.bind({
   schema: IssueTableQueryParams
 })
 
+const setDefaultQueryParams = (req, res, next) => {
+  if (!req.params.pageNumber) {
+    req.params.pageNumber = 1
+  }
+  next()
+}
+
 const fetchEntitiesWithIssues = fetchMany({
-  query: ({ req, params }) => performanceDbApi.entitiesAndIssuesQuery(req.resource.resource),
+  query: ({ req, params }) => {
+    const paginationSettings = {
+      limit: paginationPageLength,
+      offset: paginationPageLength * (params.pageNumber - 1)
+    }
+    return performanceDbApi.entitiesAndIssuesQuery(req.resource.resource, paginationSettings)
+  },
   result: 'entitiesWithIssues',
   dataset: FetchOptions.fromParams
 })
 
 const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
-  const { entitiesWithIssues, specification, entityCount: entityCountRow } = req
+  const { entitiesWithIssues, specification, entityCount: entityCountRow, issueEntitiesCount, pagination } = req
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
   const tableParams = {
@@ -36,8 +52,7 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
       specification.fields.forEach(fieldObject => {
         const { field } = fieldObject
         if (field === 'reference') {
-          const entityPageNumber = index + 1
-          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entity/${entityPageNumber}`
+          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${entity.entry_number}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field]}</a>` }
         } else if (entity[field]) {
           columns[field] = { value: entity[field] }
@@ -67,7 +82,7 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
     })
   }
 
-  const errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: entitiesWithIssues.length, entityCount, field: issueField }, true)
+  const errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
 
   req.templateParams = {
     organisation: req.orgInfo,
@@ -75,11 +90,55 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
     errorHeading,
     issueItems: [],
     issueType,
-    tableParams
+    tableParams,
+    pagination
   }
   next()
 }
 
+const createPaginationTemplatePrams = (req, res, next) => {
+  const { issueEntitiesCount } = req
+  const { pageNumber, lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+
+  const totalPages = Math.ceil(issueEntitiesCount / paginationPageLength)
+
+  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
+
+  const paginationObj = {}
+  if (pageNumber > 1) {
+    paginationObj.previous = {
+      href: `${BaseSubpath}${pageNumber - 1}`
+    }
+  }
+
+  if (pageNumber < totalPages) {
+    paginationObj.next = {
+      href: `${BaseSubpath}${pageNumber + 1}`
+    }
+  }
+
+  paginationObj.items = pagination(totalPages, pageNumber).map(item => {
+    if (item === '...') {
+      return {
+        type: 'ellipsis',
+        ellipsis: true,
+        href: '#'
+      }
+    } else {
+      return {
+        type: 'number',
+        number: item,
+        href: `${BaseSubpath}${item}`,
+        current: pageNumber === parseInt(item)
+      }
+    }
+  })
+
+  req.pagination = paginationObj
+
+  next()
+}
+
 const getIssueTable = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/issueTable.html',
@@ -88,15 +147,18 @@ const getIssueTable = renderTemplate({
 
 export default [
   validateIssueTableQueryParams,
+  setDefaultQueryParams,
   parallel([
     fetchOrgInfo,
     fetchDatasetInfo
   ]),
   fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchEntitiesWithIssues,
+  fetchIssueEntitiesCount,
   fetchSpecification,
   pullOutDatasetSpecification,
   fetchEntityCount,
+  createPaginationTemplatePrams,
   prepareIssueTableTemplateParams,
   getIssueTable,
   logPageError
diff --git a/src/routes/organisations.js b/src/routes/organisations.js
index c9290c22..d3a0221f 100644
--- a/src/routes/organisations.js
+++ b/src/routes/organisations.js
@@ -5,7 +5,7 @@ const router = express.Router()
 
 router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStartedMiddleware)
 router.get('/:lpa/:dataset/overview', OrganisationsController.getDatasetOverviewMiddleware)
-router.get('/:lpa/:dataset/:issue_type/:issue_field/entity/:pageNumber?', OrganisationsController.getIssueDetailsMiddleware)
+router.get('/:lpa/:dataset/:issue_type/:issue_field/entry/:entryNumber?', OrganisationsController.getIssueDetailsMiddleware)
 router.get('/:lpa/:dataset/:issue_type/:issue_field/:pageNumber?', OrganisationsController.getIssueTableMiddleware)
 router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskListMiddleware)
 router.get('/:lpa', OrganisationsController.getOverviewMiddleware)
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index e131c305..789ad1e7 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -58,6 +58,28 @@ const tableParams = v.strictObject({
   fields: v.array(NonEmptyString)
 })
 
+const paginationParams = v.optional(v.strictObject({
+  previous: v.optional(v.strictObject({
+    href: v.string()
+  })),
+  next: v.optional(v.strictObject({
+    href: v.string()
+  })),
+  items: v.array(v.variant('type', [
+    v.strictObject({
+      type: v.literal('number'),
+      number: v.integer(),
+      href: v.string(),
+      current: v.boolean()
+    }),
+    v.strictObject({
+      type: v.literal('ellipsis'),
+      ellipsis: v.literal(true),
+      href: v.string()
+    })
+  ]))
+}))
+
 export const OrgOverviewPage = v.strictObject({
   organisation: OrgField,
   datasets: v.array(v.strictObject({
@@ -148,6 +170,7 @@ export const OrgIssueDetails = v.strictObject({
     href: v.url()
   })),
   issueType: NonEmptyString,
+  issueField: NonEmptyString,
   entry: v.strictObject({
     title: NonEmptyString,
     fields: v.array(v.strictObject({
@@ -157,27 +180,7 @@ export const OrgIssueDetails = v.strictObject({
     })),
     geometries: v.optional(v.array(v.string()))
   }),
-  pagination: v.optional(v.strictObject({
-    previous: v.optional(v.strictObject({
-      href: v.string()
-    })),
-    next: v.optional(v.strictObject({
-      href: v.string()
-    })),
-    items: v.array(v.variant('type', [
-      v.strictObject({
-        type: v.literal('number'),
-        number: v.integer(),
-        href: v.string(),
-        current: v.boolean()
-      }),
-      v.strictObject({
-        type: v.literal('ellipsis'),
-        ellipsis: v.literal(true),
-        href: v.string()
-      })
-    ]))
-  })),
+  pagination: paginationParams,
   issueEntitiesCount: v.integer(),
   pageNumber: v.integer()
 })
@@ -191,7 +194,8 @@ export const OrgIssueTable = v.strictObject({
     href: v.url()
   })),
   issueType: NonEmptyString,
-  tableParams
+  tableParams,
+  pagination: paginationParams
 })
 
 export const CheckAnswers = v.strictObject({
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 91d7a86c..79c33b01 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -440,7 +440,7 @@ export default {
     return result.formattedData[0].entity_count
   },
 
-  entitiesAndIssuesQuery (resource) {
+  entitiesAndIssuesQuery (resource, pagination) {
     return /* sql */ `
       SELECT
           e.*,
@@ -466,6 +466,8 @@ export default {
           i.resource = '${resource}'
         GROUP BY
           (e.entity)
+        LIMIT ${pagination.limit}
+        OFFSET ${pagination.offset}
       `
   }
 }
diff --git a/src/views/organisations/issueDetails.html b/src/views/organisations/issueDetails.html
index 72379f28..4dc614c8 100644
--- a/src/views/organisations/issueDetails.html
+++ b/src/views/organisations/issueDetails.html
@@ -36,7 +36,11 @@
       href: '/organisations/' + organisation.organisation + '/' + dataset.dataset
     },
     {
-      text: issueType | capitalize
+      text: issueType | capitalize,
+      href: '/organisations/' + organisation.organisation + '/' + dataset.dataset + '/' + issueType + '/' + issueField
+    },
+    {
+      text: entry.title | capitalize
     }
   ]
 }) }}
diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
index 7e7bf86d..64e2212c 100644
--- a/src/views/organisations/issueTable.html
+++ b/src/views/organisations/issueTable.html
@@ -78,6 +78,15 @@
   </div>
 </div>
 
+
+{% if pagination.items | length > 1 %}
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    {{ govukPagination(pagination) }}
+  </div>
+</div>
+{% endif %}
+
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
     <h3 class="govuk-heading-m">

From 802aa41911b28094a7e286b19c9ef71533a5bec2 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 13:28:09 +0100
Subject: [PATCH 017/109] fix pagination and issues by entity for issue details

---
 src/middleware/issueDetails.middleware.js | 32 +++++++++++++----------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index f5f029a3..ff288172 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -86,7 +86,7 @@ async function fetchEntry (req, res, next) {
     entryNumber,
     datasetId
   )
-  req.entryNumber = entryNumber
+
   next()
 }
 
@@ -156,22 +156,21 @@ const processEntryRow = (issueType, issuesByEntryNumber, row) => {
  * Middleware. Updates req with `templateParams`
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount: entityCountRow } = req
-  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { entryData, issueEntitiesCount, issuesByEntryNumber, entityCount: entityCountRow } = req
+  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField, entryNumber } = req.params
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
   let errorHeading
   let issueItems
 
-  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entity/`
+  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
 
   if (Object.keys(issuesByEntryNumber).length < entityCount) {
     errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
-    issueItems = Object.entries(issuesByEntryNumber).map(([entryNumber, issues], i) => {
-      const pageNum = i + 1
+    issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
       return {
         html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
-        href: `${BaseSubpath}${pageNum}`
+        href: `${BaseSubpath}${entryNumber}`
       }
     })
   } else {
@@ -181,7 +180,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   }
 
   const fields = entryData.map((row) => processEntryRow(issueType, issuesByEntryNumber, row))
-  const entityIssues = Object.values(issuesByEntryNumber)[pageNumber - 1] || []
+  const entityIssues = issuesByEntryNumber[entryNumber] || []
   for (const issue of entityIssues) {
     if (!fields.find((field) => field.key.text === issue.field)) {
       const errorMessage = issue.message || issueType
@@ -202,16 +201,22 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     geometries
   }
 
-  const paginationObj = {}
+  const paginationObj = {
+    items: []
+  }
+
+  const entryNumbers = Object.keys(issuesByEntryNumber)
+  const pageNumber = entryNumbers.findIndex(currentEntryNumber => currentEntryNumber === entryNumber) + 1
+
   if (pageNumber > 1) {
     paginationObj.previous = {
-      href: `${BaseSubpath}${pageNumber - 1}`
+      href: `${BaseSubpath}${entryNumbers[pageNumber - 1]}`
     }
   }
 
   if (pageNumber < issueEntitiesCount) {
     paginationObj.next = {
-      href: `${BaseSubpath}${pageNumber + 1}`
+      href: `${BaseSubpath}${entryNumbers[pageNumber + 1]}`
     }
   }
 
@@ -226,7 +231,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
       return {
         type: 'number',
         number: item,
-        href: `${BaseSubpath}${item}`,
+        href: `${BaseSubpath}${entryNumbers[item - 1]}`,
         current: pageNumber === parseInt(item)
       }
     }
@@ -242,8 +247,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     issueType,
     issueField,
     pagination: paginationObj,
-    issueEntitiesCount,
-    pageNumber
+    issueEntitiesCount
   }
 
   next()

From 68ec3238a5cfbfe54120f4f13901c955c96218c3 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 13:28:32 +0100
Subject: [PATCH 018/109] make sure to filter by issue type and field for issue
 table view

---
 src/middleware/issueTable.middleware.js | 9 +++++++--
 src/services/performanceDbApi.js        | 8 +++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index dc635b46..fb05ba3b 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -28,11 +28,16 @@ const setDefaultQueryParams = (req, res, next) => {
 
 const fetchEntitiesWithIssues = fetchMany({
   query: ({ req, params }) => {
-    const paginationSettings = {
+    const pagination = {
       limit: paginationPageLength,
       offset: paginationPageLength * (params.pageNumber - 1)
     }
-    return performanceDbApi.entitiesAndIssuesQuery(req.resource.resource, paginationSettings)
+    return performanceDbApi.entitiesAndIssuesQuery({
+      resource: req.resource.resource,
+      issueType: req.params.issue_type,
+      issueField: req.params.issue_field,
+      pagination
+    })
   },
   result: 'entitiesWithIssues',
   dataset: FetchOptions.fromParams
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 79c33b01..0eb9587e 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -440,7 +440,7 @@ export default {
     return result.formattedData[0].entity_count
   },
 
-  entitiesAndIssuesQuery (resource, pagination) {
+  entitiesAndIssuesQuery ({ resource, pagination, issueType, issueField }) {
     return /* sql */ `
       SELECT
           e.*,
@@ -462,8 +462,10 @@ export default {
               fr.resource = '${resource}'
           ) fr ON fr.entity = e.entity
           LEFT JOIN issue i ON i.entry_number = fr.entry_number
-        WHERE
-          i.resource = '${resource}'
+        WHERE i.resource = '${resource}'
+        AND i.issue_type = '${issueType}'
+        AND i.field = '${issueField}'
+
         GROUP BY
           (e.entity)
         LIMIT ${pagination.limit}

From 558da864310e5d024be1c0fb30112860c911ea7e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 14:22:02 +0100
Subject: [PATCH 019/109] fix issue details page tests

---
 test/unit/issueDetailsPage.test.js            |  6 +-
 .../issueDetails.middleware.test.js           | 55 +++++++++----------
 2 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/test/unit/issueDetailsPage.test.js b/test/unit/issueDetailsPage.test.js
index ced8b3a3..e4dbc4d8 100644
--- a/test/unit/issueDetailsPage.test.js
+++ b/test/unit/issueDetailsPage.test.js
@@ -183,7 +183,8 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
           ...params.entry,
           geometries: ['POINT(0 0)']
         },
-        issueType: params.issueType
+        issueType: params.issueType,
+        issueField: params.issueField
       }
 
       const mapHtml = nunjucks.render('organisations/issueDetails.html', paramWithGeometry)
@@ -206,7 +207,8 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
           ...params.entry,
           geometries: []
         },
-        issueType: params.issueType
+        issueType: params.issueType,
+        issueField: params.issueField
       }
 
       const mapHtml = nunjucks.render('organisations/issueDetails.html', paramWithGeometry)
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 143d049c..7e32b190 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -13,12 +13,12 @@ describe('issueDetails.middleware.js', () => {
     {
       field: 'start-date',
       value: '02-02-2022',
-      entry_number: 1
+      entry_number: 10
     }
   ]
   const issues = [
     {
-      entry_number: 0,
+      entry_number: 10,
       field: 'start-date',
       value: '02-02-2022'
     }
@@ -32,27 +32,25 @@ describe('issueDetails.middleware.js', () => {
         issue_type: 'test-issue-type',
         issue_field: 'test-issue-field',
         resourceId: 'test-resource-id',
-        entityNumber: '1'
+        entryNumber: '10'
       }
       const req = {
         params: requestParams,
         // middleware supplies the below
-        entryNumber: 1,
         entityCount: { entity_count: 3 },
         issueEntitiesCount: 1,
-        pageNumber: 1,
         orgInfo,
         dataset,
         entryData,
         issues,
         resource: { resource: requestParams.resourceId },
         issuesByEntryNumber: {
-          1: [
+          10: [
             {
               field: 'start-date',
               value: '02-02-2022',
               line_number: 1,
-              entry_number: 1,
+              entry_number: 10,
               message: 'mock message',
               issue_type: 'mock type'
             }
@@ -79,15 +77,15 @@ describe('issueDetails.middleware.js', () => {
           dataset: 'mock-dataset',
           collection: 'mock-collection'
         },
-        errorHeading: 'mockMessageFor: 0',
+        errorHeading: 'mockMessageFor: 10',
         issueItems: [
           {
-            html: 'mock task message 1 in record 1',
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
+            html: 'mock task message 1 in record 10',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
           }
         ],
         entry: {
-          title: 'entry: 1',
+          title: 'entry: 10',
           fields: [
             {
               key: { text: 'start-date' },
@@ -98,16 +96,16 @@ describe('issueDetails.middleware.js', () => {
           geometries: []
         },
         issueType: 'test-issue-type',
+        issueField: 'test-issue-field',
         pagination: {
           items: [{
             current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10',
             number: 1,
             type: 'number'
           }]
         },
-        issueEntitiesCount: 1,
-        pageNumber: 1
+        issueEntitiesCount: 1
       }
 
       expect(req.templateParams).toEqual(expectedTempalteParams)
@@ -118,12 +116,12 @@ describe('issueDetails.middleware.js', () => {
         {
           field: 'start-date',
           value: '02-02-2022',
-          entry_number: 1
+          entry_number: 10
         },
         {
           field: 'geometry',
           value: 'POINT(0 0)',
-          entry_number: 1
+          entry_number: 10
         }
       ]
       const requestParams = {
@@ -132,27 +130,26 @@ describe('issueDetails.middleware.js', () => {
         issue_type: 'test-issue-type',
         issue_field: 'test-issue-field',
         resourceId: 'test-resource-id',
-        entityNumber: '1'
+        entryNumber: '10'
       }
       const req = {
         params: requestParams,
         // middleware supplies the below
-        entryNumber: 1,
+        entryNumber: 10,
         entityCount: { entity_count: 3 },
         issueEntitiesCount: 1,
-        pageNumber: 1,
         orgInfo,
         dataset,
         entryData,
         issues,
         resource: { resource: requestParams.resourceId },
         issuesByEntryNumber: {
-          1: [
+          10: [
             {
               field: 'start-date',
               value: '02-02-2022',
               line_number: 1,
-              entry_number: 1,
+              entry_number: 10,
               message: 'mock message',
               issue_type: 'mock type'
             }
@@ -160,6 +157,7 @@ describe('issueDetails.middleware.js', () => {
         }
         // errorHeading -- set  in prepare* fn
       }
+
       v.parse(IssueDetailsQueryParams, req.params)
 
       issues.forEach(issue => {
@@ -179,15 +177,15 @@ describe('issueDetails.middleware.js', () => {
           dataset: 'mock-dataset',
           collection: 'mock-collection'
         },
-        errorHeading: 'mockMessageFor: 0',
+        errorHeading: 'mockMessageFor: 10',
         issueItems: [
           {
-            html: 'mock task message 1 in record 1',
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
+            html: 'mock task message 1 in record 10',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
           }
         ],
         entry: {
-          title: 'entry: 1',
+          title: 'entry: 10',
           fields: [
             {
               key: { text: 'start-date' },
@@ -207,16 +205,16 @@ describe('issueDetails.middleware.js', () => {
           geometries: ['POINT(0 0)']
         },
         issueType: 'test-issue-type',
+        issueField: 'test-issue-field',
         pagination: {
           items: [{
             current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10',
             number: 1,
             type: 'number'
           }]
         },
-        issueEntitiesCount: 1,
-        pageNumber: 1
+        issueEntitiesCount: 1
       }
 
       expect(req.templateParams).toEqual(expectedTemplateParams)
@@ -264,6 +262,7 @@ describe('issueDetails.middleware.js', () => {
             geometries: ['POINT(0 0)']
           },
           issueType: 'test-issue-type',
+          issueField: 'test-issue-field',
           pagination: {
             items: [{
               current: true,

From 7a315c6eb31a5f66e2a201ed54fbd618e848b551 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 14:37:34 +0100
Subject: [PATCH 020/109] fix issue table view test nunjucks import

---
 test/unit/views/organisations/issueTablePage.test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/views/organisations/issueTablePage.test.js b/test/unit/views/organisations/issueTablePage.test.js
index 8b3e2424..3c8e6c7f 100644
--- a/test/unit/views/organisations/issueTablePage.test.js
+++ b/test/unit/views/organisations/issueTablePage.test.js
@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest'
-import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
+import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
 import { runGenericPageTests } from './generic-page.js'
 import config from '../../config/index.js'

From 5c76397eb1865ff76bd8e9ba622aa725be0d86d8 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 14:45:52 +0100
Subject: [PATCH 021/109] updated table component tests to test for html values

---
 test/unit/tableComponent.test.js | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/unit/tableComponent.test.js b/test/unit/tableComponent.test.js
index 650e9a0d..1f9079a6 100644
--- a/test/unit/tableComponent.test.js
+++ b/test/unit/tableComponent.test.js
@@ -41,7 +41,7 @@ describe('Table Component', () => {
           columns: {
             field1: {
               error: false,
-              value: 'value21'
+              html: '<h1>value21</h1>'
             },
             field2: {
               error: false,
@@ -92,7 +92,11 @@ describe('Table Component', () => {
       expect(columns.length).toEqual(Object.keys(rowData.columns).length)
 
       Object.values(rowData.columns).forEach((field, j) => {
-        expect(columns[j].textContent).toContain(field.value)
+        if (field.value) {
+          expect(columns[j].textContent).toContain(field.value)
+        } else if (field.html) {
+          expect(columns[j].innerHTML).toContain(field.html)
+        }
 
         if (field.error) {
           expect(columns[j].textContent).toContain(field.error.message)

From 3f824df3fe941673f40770178b59a7d4728b85d4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 17:01:19 +0100
Subject: [PATCH 022/109] move shared tests to folder

---
 test/unit/check-answers.test.js                           | 2 +-
 test/unit/check/confirmationPage.test.js                  | 2 +-
 test/unit/choose-datasetPage.test.js                      | 2 +-
 test/unit/dataset-details.test.js                         | 2 +-
 test/unit/datasetTaskListPage.test.js                     | 2 +-
 test/unit/endpointSubmissionForm/confirmationPage.test.js | 2 +-
 test/unit/errorsPage.test.js                              | 2 +-
 test/unit/findPage.test.js                                | 2 +-
 test/unit/get-startedPage.test.js                         | 2 +-
 test/unit/http-errorPage.test.js                          | 2 +-
 test/unit/issueDetailsPage.test.js                        | 2 +-
 test/unit/landingPage.test.js                             | 2 +-
 test/unit/lpa-detailsPage.test.js                         | 2 +-
 test/unit/lpaOverviewPage.test.js                         | 2 +-
 test/unit/noErrorsPage.test.js                            | 2 +-
 test/unit/{ => sharedTests}/generic-page.js               | 0
 test/unit/{ => sharedTests}/paginationTemplateTests.js    | 0
 test/unit/views/organisations/dataset-overview.test.js    | 2 +-
 18 files changed, 16 insertions(+), 16 deletions(-)
 rename test/unit/{ => sharedTests}/generic-page.js (100%)
 rename test/unit/{ => sharedTests}/paginationTemplateTests.js (100%)

diff --git a/test/unit/check-answers.test.js b/test/unit/check-answers.test.js
index 3bf450f4..0b18b899 100644
--- a/test/unit/check-answers.test.js
+++ b/test/unit/check-answers.test.js
@@ -2,7 +2,7 @@
 
 import { describe, expect, it } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import { stripWhitespace } from '../utils/stripWhiteSpace.js'
 
 describe('check-answers View', async () => {
diff --git a/test/unit/check/confirmationPage.test.js b/test/unit/check/confirmationPage.test.js
index deac5f6b..b2617dd0 100644
--- a/test/unit/check/confirmationPage.test.js
+++ b/test/unit/check/confirmationPage.test.js
@@ -2,7 +2,7 @@
 
 import { describe, expect, it } from 'vitest'
 import { setupNunjucks } from '../../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from '../generic-page.js'
+import { runGenericPageTests } from '../sharedTests/generic-page.js'
 import { stripWhitespace } from '../../utils/stripWhiteSpace.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
diff --git a/test/unit/choose-datasetPage.test.js b/test/unit/choose-datasetPage.test.js
index 6036a9b5..0c902cc5 100644
--- a/test/unit/choose-datasetPage.test.js
+++ b/test/unit/choose-datasetPage.test.js
@@ -1,6 +1,6 @@
 import { describe, it } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import { testValidationErrorMessage } from './validation-tests.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
diff --git a/test/unit/dataset-details.test.js b/test/unit/dataset-details.test.js
index d27eebed..2ef9b39d 100644
--- a/test/unit/dataset-details.test.js
+++ b/test/unit/dataset-details.test.js
@@ -2,7 +2,7 @@
 
 import { describe, expect, it } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import { stripWhitespace } from '../utils/stripWhiteSpace.js'
 import { testValidationErrorMessage } from './validation-tests.js'
 import { render } from '../../src/utils/custom-renderer.js'
diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js
index 31909003..d696cd45 100644
--- a/test/unit/datasetTaskListPage.test.js
+++ b/test/unit/datasetTaskListPage.test.js
@@ -1,6 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import jsdom from 'jsdom'
 import mocker from '../utils/mocker.js'
 import { OrgDatasetTaskList } from '../../src/routes/schemas.js'
diff --git a/test/unit/endpointSubmissionForm/confirmationPage.test.js b/test/unit/endpointSubmissionForm/confirmationPage.test.js
index a28dc48f..031d070d 100644
--- a/test/unit/endpointSubmissionForm/confirmationPage.test.js
+++ b/test/unit/endpointSubmissionForm/confirmationPage.test.js
@@ -2,7 +2,7 @@
 
 import { describe, expect, it } from 'vitest'
 import { setupNunjucks } from '../../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from '../generic-page.js'
+import { runGenericPageTests } from '../sharedTests/generic-page.js'
 import { stripWhitespace } from '../../utils/stripWhiteSpace.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
diff --git a/test/unit/errorsPage.test.js b/test/unit/errorsPage.test.js
index 0b7a94a3..6ba8b25e 100644
--- a/test/unit/errorsPage.test.js
+++ b/test/unit/errorsPage.test.js
@@ -8,7 +8,7 @@ import jsdom from 'jsdom'
 import errorResponse from '../../docker/request-api-stub/wiremock/__files/check_file/article-4/request-complete-errors.json'
 import errorResponseDetails from '../../docker/request-api-stub/wiremock/__files/check_file/article-4/request-complete-errors-details.json'
 import ResponseDetails from '../../src/models/responseDetails.js'
-import paginationTemplateTests from './paginationTemplateTests.js'
+import paginationTemplateTests from './sharedTests/paginationTemplateTests.js'
 import prettifyColumnName from '../../src/filters/prettifyColumnName.js'
 
 const nunjucksEnv = nunjucks.configure([
diff --git a/test/unit/findPage.test.js b/test/unit/findPage.test.js
index 394d823c..0f41b6c9 100644
--- a/test/unit/findPage.test.js
+++ b/test/unit/findPage.test.js
@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import jsdom from 'jsdom'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import config from '../../config/index.js'
 import mock from '../utils/mocker.js'
 import { OrgFindPage } from '../../src/routes/schemas.js'
diff --git a/test/unit/get-startedPage.test.js b/test/unit/get-startedPage.test.js
index 54a00569..3936e5b2 100644
--- a/test/unit/get-startedPage.test.js
+++ b/test/unit/get-startedPage.test.js
@@ -1,7 +1,7 @@
 // getStartedPage.test.js
 
 import { describe, it, expect } from 'vitest'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import jsdom from 'jsdom'
 import mocker from '../utils/mocker.js'
 import { OrgGetStarted } from '../../src/routes/schemas.js'
diff --git a/test/unit/http-errorPage.test.js b/test/unit/http-errorPage.test.js
index 83c23831..9a90878d 100644
--- a/test/unit/http-errorPage.test.js
+++ b/test/unit/http-errorPage.test.js
@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import mock from '../utils/mocker.js'
 import { OrgEndpointError } from '../../src/routes/schemas.js'
 
diff --git a/test/unit/issueDetailsPage.test.js b/test/unit/issueDetailsPage.test.js
index e4dbc4d8..e97b2575 100644
--- a/test/unit/issueDetailsPage.test.js
+++ b/test/unit/issueDetailsPage.test.js
@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import config from '../../config/index.js'
 import { OrgIssueDetails } from '../../src/routes/schemas.js'
 import mocker from '../utils/mocker.js'
diff --git a/test/unit/landingPage.test.js b/test/unit/landingPage.test.js
index 6f0c68f5..bf4cdc7a 100644
--- a/test/unit/landingPage.test.js
+++ b/test/unit/landingPage.test.js
@@ -2,7 +2,7 @@
 
 import { describe } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
diff --git a/test/unit/lpa-detailsPage.test.js b/test/unit/lpa-detailsPage.test.js
index 2d6fec2f..daa1cc3d 100644
--- a/test/unit/lpa-detailsPage.test.js
+++ b/test/unit/lpa-detailsPage.test.js
@@ -1,6 +1,6 @@
 import { describe, it } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import { testValidationErrorMessage } from './validation-tests.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
diff --git a/test/unit/lpaOverviewPage.test.js b/test/unit/lpaOverviewPage.test.js
index 2dcfb104..09585faf 100644
--- a/test/unit/lpaOverviewPage.test.js
+++ b/test/unit/lpaOverviewPage.test.js
@@ -1,6 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
-import { runGenericPageTests } from './generic-page.js'
+import { runGenericPageTests } from './sharedTests/generic-page.js'
 import jsdom from 'jsdom'
 import { makeDatasetSlugToReadableNameFilter } from '../../src/filters/makeDatasetSlugToReadableNameFilter.js'
 import mocker from '../utils/mocker.js'
diff --git a/test/unit/noErrorsPage.test.js b/test/unit/noErrorsPage.test.js
index ce9d3a19..cd46e4c7 100644
--- a/test/unit/noErrorsPage.test.js
+++ b/test/unit/noErrorsPage.test.js
@@ -8,7 +8,7 @@ import jsdom from 'jsdom'
 import errorResponse from '../../docker/request-api-stub/wiremock/__files/check_file/article-4/request-complete-errors.json'
 import errorResponseDetails from '../../docker/request-api-stub/wiremock/__files/check_file/article-4/request-complete-errors-details.json'
 import ResponseDetails from '../../src/models/responseDetails.js'
-import paginationTemplateTests from './paginationTemplateTests.js'
+import paginationTemplateTests from './sharedTests/paginationTemplateTests.js'
 import prettifyColumnName from '../../src/filters/prettifyColumnName.js'
 
 const nunjucksEnv = nunjucks.configure([
diff --git a/test/unit/generic-page.js b/test/unit/sharedTests/generic-page.js
similarity index 100%
rename from test/unit/generic-page.js
rename to test/unit/sharedTests/generic-page.js
diff --git a/test/unit/paginationTemplateTests.js b/test/unit/sharedTests/paginationTemplateTests.js
similarity index 100%
rename from test/unit/paginationTemplateTests.js
rename to test/unit/sharedTests/paginationTemplateTests.js
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index bbb5c1bb..74c40b7b 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -1,6 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import jsdom from 'jsdom'
-import { runGenericPageTests } from '../../generic-page.js'
+import { runGenericPageTests } from '../../sharedTests/generic-page.js'
 import { stripWhitespace } from '../../../utils/stripWhiteSpace.js'
 import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
 

From 0685477b3286e025f5c3fbf6304f12e42fbbd3d3 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 7 Oct 2024 17:01:54 +0100
Subject: [PATCH 023/109] add tableTests shared test file, and also improved
 mocking around table params mocks

---
 test/unit/sharedTests/tableTests.js           |  42 ++++++
 test/unit/tableComponent.test.js              | 127 +++++++-----------
 .../organisations/issueTablePage.test.js      | 114 +++++++---------
 test/utils/mocker.js                          |  50 ++++++-
 4 files changed, 189 insertions(+), 144 deletions(-)
 create mode 100644 test/unit/sharedTests/tableTests.js

diff --git a/test/unit/sharedTests/tableTests.js b/test/unit/sharedTests/tableTests.js
new file mode 100644
index 00000000..992b971c
--- /dev/null
+++ b/test/unit/sharedTests/tableTests.js
@@ -0,0 +1,42 @@
+import { describe, it, expect } from 'vitest'
+import prettifyColumnName from '../../../src/filters/prettifyColumnName'
+
+export const runTableTests = (tableParams, document) => {
+  describe('Table Component', () => {
+    const tableHead = document.querySelector('.govuk-table__head')
+
+    it('Renders the correct table headings', () => {
+      const columnHeadings = tableHead.children[0].children
+
+      tableParams.columns.forEach((column, i) => {
+        expect(columnHeadings[i].textContent).toContain(prettifyColumnName(column))
+      })
+    })
+
+    const tableBody = document.querySelector('.govuk-table__body')
+    const rows = tableBody.children
+
+    it('Renders the correct number of rows rows', () => {
+      expect(rows.length).toEqual(tableParams.rows.length)
+    })
+
+    it('Renders the correct row content', () => {
+      tableParams.rows.forEach((rowData, i) => {
+        const columns = rows[i].children
+        expect(columns.length).toEqual(Object.keys(rowData.columns).length)
+
+        Object.values(rowData.columns).forEach((field, j) => {
+          if (field.value) {
+            expect(columns[j].textContent).toContain(field.value)
+          } else if (field.html) {
+            expect(columns[j].innerHTML).toContain(field.html)
+          }
+
+          if (field.error) {
+            expect(columns[j].textContent).toContain(prettifyColumnName(field.error.message))
+          }
+        })
+      })
+    })
+  })
+}
diff --git a/test/unit/tableComponent.test.js b/test/unit/tableComponent.test.js
index 1f9079a6..f2a2f789 100644
--- a/test/unit/tableComponent.test.js
+++ b/test/unit/tableComponent.test.js
@@ -1,6 +1,7 @@
-import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
+import { runTableTests } from './sharedTests/tableTests.js'
+import prettifyColumnName from '../../src/filters/prettifyColumnName.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
@@ -12,96 +13,60 @@ const nunjucksEnv = nunjucks.configure([
   'node_modules/@x-govuk/govuk-prototype-components/'
 ], {})
 
-nunjucksEnv.addFilter('prettifyColumnName', columnName => columnName)
+nunjucksEnv.addFilter('prettifyColumnName', prettifyColumnName)
 
-describe('Table Component', () => {
-  const params = {
-    tableParams: {
-      columns: ['col1', 'col2', 'col3'],
-      rows: [
-        {
-          columns: {
-            field1: {
-              error: false,
-              value: 'value11'
+const params = {
+  tableParams: {
+    columns: ['col1', 'col2', 'col3'],
+    rows: [
+      {
+        columns: {
+          field1: {
+            error: false,
+            value: 'value11'
+          },
+          field2: {
+            error: {
+              message: 'error in value12'
             },
-            field2: {
-              error: {
-                message: 'error in value12'
-              },
-              value: 'value12'
-            },
-            field3: {
-              error: false,
-              value: 'value13'
-            }
+            value: 'value12'
+          },
+          field3: {
+            error: false,
+            value: 'value13'
           }
-        },
-        {
-          columns: {
-            field1: {
-              error: false,
-              html: '<h1>value21</h1>'
-            },
-            field2: {
-              error: false,
-              value: 'value22'
+        }
+      },
+      {
+        columns: {
+          field1: {
+            error: false,
+            html: '<h1>value21</h1>'
+          },
+          field2: {
+            error: false,
+            value: 'value22'
+          },
+          field3: {
+            error: {
+              message: 'error in value23'
             },
-            field3: {
-              error: {
-                message: 'error in value23'
-              },
-              value: 'value23'
-            }
+            value: 'value23'
           }
         }
-      ],
-      fields: ['field1', 'field2', 'field3']
-    }
+      }
+    ],
+    fields: ['field1', 'field2', 'field3']
   }
+}
 
-  const htmlString = `
+const htmlString = `
     {% from "components/table.html" import table %}
     {{ table(tableParams) }}
   `
 
-  const html = nunjucks.renderString(htmlString, params)
-  const dom = new JSDOM(html)
-  const document = dom.window.document
-
-  const tableHead = document.querySelector('.govuk-table__head')
-
-  it('Renders the correct table headings', () => {
-    const columnHeadings = tableHead.children[0].children
-
-    params.tableParams.columns.forEach((column, i) => {
-      expect(columnHeadings[i].textContent).toContain(column)
-    })
-  })
+const html = nunjucks.renderString(htmlString, params)
+const dom = new JSDOM(html)
+const document = dom.window.document
 
-  const tableBody = document.querySelector('.govuk-table__body')
-  const rows = tableBody.children
-
-  it('Renders the correct number of rows rows', () => {
-    expect(rows.length).toEqual(params.tableParams.rows.length)
-  })
-
-  it('Renders the correct row content', () => {
-    params.tableParams.rows.forEach((rowData, i) => {
-      const columns = rows[i].children
-      expect(columns.length).toEqual(Object.keys(rowData.columns).length)
-
-      Object.values(rowData.columns).forEach((field, j) => {
-        if (field.value) {
-          expect(columns[j].textContent).toContain(field.value)
-        } else if (field.html) {
-          expect(columns[j].innerHTML).toContain(field.html)
-        }
-
-        if (field.error) {
-          expect(columns[j].textContent).toContain(field.error.message)
-        }
-      })
-    })
-  })
-})
+runTableTests(params.tableParams, document)
diff --git a/test/unit/views/organisations/issueTablePage.test.js b/test/unit/views/organisations/issueTablePage.test.js
index 3c8e6c7f..12d0798a 100644
--- a/test/unit/views/organisations/issueTablePage.test.js
+++ b/test/unit/views/organisations/issueTablePage.test.js
@@ -1,86 +1,76 @@
 import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
-import { runGenericPageTests } from './generic-page.js'
-import config from '../../config/index.js'
-import mocker from '../utils/mocker.js'
+import { runGenericPageTests } from '../../sharedTests/generic-page.js'
+import config from '../../../../config/index.js'
+import mocker from '../../../utils/mocker.js'
 import { OrgIssueTable } from '../../../../src/routes/schemas.js'
+import { runTableTests } from '../../sharedTests/tableTests.js'
 
 const nunjucks = setupNunjucks({})
 
-const seed = new Date().getTime()
-
-describe(`issueDetails.html(seed: ${seed})`, () => {
-  const params = mocker(OrgIssueTable, seed)
-
-  params.issueEntitiesCount = undefined
-
-  const html = nunjucks.render('organisations/issueTable.html', params)
-  const dom = new JSDOM(html)
-  const document = dom.window.document
-
-  runGenericPageTests(html, {
-    pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues - ${config.serviceNames.submit}`,
-    breadcrumbs: [
-      { text: 'Home', href: '/' },
-      { text: 'Organisations', href: '/organisations' },
-      { text: params.organisation.name, href: `/organisations/${params.organisation.organisation}` },
-      { text: params.dataset.name, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` },
-      { text: 'mock issue' }
-    ]
-  })
-
-  it('Renders the correct headings', () => {
-    expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name)
-    expect(document.querySelector('h1').textContent).toContain(params.dataset.name)
-  })
+const runTestsWithSeed = (seed) => {
+  describe(`issueTable.html(seed: ${seed})`, () => {
+    const params = mocker(OrgIssueTable, seed)
+
+    const html = nunjucks.render('organisations/issueTable.html', params)
+    const dom = new JSDOM(html)
+    const document = dom.window.document
+
+    runGenericPageTests(html, {
+      pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues - ${config.serviceNames.submit}`,
+      breadcrumbs: [
+        { text: 'Home', href: '/' },
+        { text: 'Organisations', href: '/organisations' },
+        { text: params.organisation.name, href: `/organisations/${params.organisation.organisation}` },
+        { text: params.dataset.name, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` },
+        { text: 'mock issue' }
+      ]
+    })
 
-  describe('error summary', () => {
-    it('should render the correct heading', () => {
-      expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorHeading || 'There is a problem')
+    it('Renders the correct headings', () => {
+      expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name)
+      expect(document.querySelector('h1').textContent).toContain(params.dataset.name)
     })
 
-    it('should render the correct heading if none is supplied', () => {
-      const noErrorHeadingPageHtml = nunjucks.render('organisations/issueDetails.html', {
-        ...params,
-        errorHeading: undefined
+    describe('error summary', () => {
+      it('should render the correct heading', () => {
+        expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorHeading || 'There is a problem')
       })
 
-      const domNoErrorHeading = new JSDOM(noErrorHeadingPageHtml)
-      const documentNoErrorHeading = domNoErrorHeading.window.document
-
-      expect(documentNoErrorHeading.querySelector('.govuk-error-summary__title').textContent).toContain('There is a problem')
-    })
+      it('should render the correct heading if none is supplied', () => {
+        const noErrorHeadingPageHtml = nunjucks.render('organisations/issueTable.html', {
+          ...params,
+          errorHeading: undefined
+        })
 
-    it('should render the issue items', () => {
-      const issueList = document.querySelector('.govuk-error-summary__list')
-      const issueItemElements = [...issueList.children]
-      expect(issueItemElements.length).toBe(params.issueItems.length)
+        const domNoErrorHeading = new JSDOM(noErrorHeadingPageHtml)
+        const documentNoErrorHeading = domNoErrorHeading.window.document
 
-      issueItemElements.forEach((element, index) => {
-        expect(element.textContent).toContain(params.issueItems[index].html)
+        expect(documentNoErrorHeading.querySelector('.govuk-error-summary__title').textContent).toContain('There is a problem')
       })
-    })
-  })
 
-  describe('Table', () => {
-    it('should render the table with the correct values', () => {
+      it('should render the issue items', () => {
+        const issueList = document.querySelector('.govuk-error-summary__list')
+        const issueItemElements = [...issueList.children]
+        expect(issueItemElements.length).toBe(params.issueItems.length)
 
+        issueItemElements.forEach((element, index) => {
+          expect(element.textContent).toContain(params.issueItems[index].html)
+        })
+      })
     })
 
-    it('should render the references as links to the issue details view', () => {
-
-    })
+    runTableTests(params.tableParams, document)
 
-    it('should render the table with the correct errors', () => {
+    describe('multi page', () => {
+      // runGenericPageTests
 
+      it('correctly renders the pagination component', () => {
+      })
     })
   })
+}
 
-  describe('multi page', () => {
-    // runGenericPageTests
-
-    it('correctly renders the pagination component', () => {
-    })
-  })
-})
+const seed = new Date().getTime()
+runTestsWithSeed(seed)
diff --git a/test/utils/mocker.js b/test/utils/mocker.js
index d7e24422..17341961 100644
--- a/test/utils/mocker.js
+++ b/test/utils/mocker.js
@@ -29,7 +29,9 @@ export default (schema, seed) => {
 
   const data = JSONSchemaFaker.generate(jsonSchema)
 
-  return data
+  const enhancedData = enhanceMockedData(data, jsonSchema)
+
+  return enhancedData
 }
 
 let xorShiftSeed
@@ -60,3 +62,49 @@ export const generateNextRandomNumber = (seed) => {
 export const resetRandomNumberGenerator = () => {
   xorShiftSeed = undefined
 }
+
+const enhanceMockedData = (data, schema) => {
+  if ('tableParams' in data) {
+    data.tableParams = mockTableParams(data.tableParams, schema.properties.tableParams)
+  }
+
+  return data
+}
+
+const mockTableParams = (tableParams, schema) => {
+  const columnSchema = schema.properties.columns
+  columnSchema.minItems = 2
+  columnSchema.maxItems = 10
+
+  const columns = JSONSchemaFaker.generate(columnSchema)
+
+  const fieldSchema = schema.properties.fields
+  fieldSchema.minItems = columns.length
+  fieldSchema.maxItems = columns.length
+  fieldSchema.uniqueItems = true
+
+  const fields = JSONSchemaFaker.generate(fieldSchema)
+
+  const rowsSchema = { ...schema.properties.rows }
+  rowsSchema.items.properties.columns.required = []
+
+  const rowSchema = { ...schema.properties.rows.items.properties.columns.additionalProperties }
+  rowSchema.oneOff = [
+    { required: ['html'] },
+    { required: ['value'] }
+  ]
+  rowSchema.additionalProperties = false
+
+  fields.forEach(field => {
+    rowsSchema.items.properties.columns.properties[field] = rowSchema
+    rowsSchema.items.properties.columns.required.push(field)
+  })
+
+  rowsSchema.items.properties.columns.additionalProperties = false
+
+  const rows = JSONSchemaFaker.generate(rowsSchema)
+
+  tableParams = { columns, fields, rows }
+
+  return tableParams
+}

From 936387390f922b3e141966533d2f5ae1609e06d2 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 8 Oct 2024 17:57:48 +0100
Subject: [PATCH 024/109] add tests for issueTable.middleware

---
 src/middleware/issueTable.middleware.js       |  11 +-
 src/routes/schemas.js                         |   4 +-
 .../middleware/issueTable.middleware.test.js  | 206 ++++++++++++++++++
 3 files changed, 214 insertions(+), 7 deletions(-)
 create mode 100644 test/unit/middleware/issueTable.middleware.test.js

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index fb05ba3b..51a045c7 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,4 +1,5 @@
 import performanceDbApi from '../services/performanceDbApi.js'
+import logger from '../utils/logger.js'
 import { pagination } from '../utils/pagination.js'
 import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
@@ -19,7 +20,7 @@ const validateIssueTableQueryParams = validateQueryParams.bind({
   schema: IssueTableQueryParams
 })
 
-const setDefaultQueryParams = (req, res, next) => {
+export const setDefaultQueryParams = (req, res, next) => {
   if (!req.params.pageNumber) {
     req.params.pageNumber = 1
   }
@@ -43,7 +44,7 @@ const fetchEntitiesWithIssues = fetchMany({
   dataset: FetchOptions.fromParams
 })
 
-const prepareIssueTableTemplateParams = (req, res, next) => {
+export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
   const { entitiesWithIssues, specification, entityCount: entityCountRow, issueEntitiesCount, pagination } = req
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
@@ -70,7 +71,7 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
       try {
         issues = JSON.parse(entity.issues)
       } catch (e) {
-        console.log(e)
+        logger.warn('issueTableMiddleware:prepareIssueTableParams - entity issues is not valid json', { entityIssues: entity.issues })
       }
 
       Object.entries(issues).forEach(([field, issueType]) => {
@@ -101,7 +102,7 @@ const prepareIssueTableTemplateParams = (req, res, next) => {
   next()
 }
 
-const createPaginationTemplatePrams = (req, res, next) => {
+export const createPaginationTemplatePrams = (req, res, next) => {
   const { issueEntitiesCount } = req
   const { pageNumber, lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
@@ -144,7 +145,7 @@ const createPaginationTemplatePrams = (req, res, next) => {
   next()
 }
 
-const getIssueTable = renderTemplate({
+export const getIssueTable = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/issueTable.html',
   handlerName: 'getIssueTable'
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 990f7059..dcbc474c 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -38,8 +38,8 @@ export const datasetStatusEnum = {
   'Not submitted': 'Not submitted'
 }
 
-const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
-const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
+export const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
+export const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
 
 const tableParams = v.strictObject({
   columns: v.array(NonEmptyString),
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
new file mode 100644
index 00000000..3902fde4
--- /dev/null
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -0,0 +1,206 @@
+import { describe, it, vi, expect } from 'vitest'
+
+import performanceDbApi from '../../../src/services/performanceDbApi.js'
+import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, createPaginationTemplatePrams } from '../../../src/middleware/issueTable.middleware.js'
+// import { pagination } from '../../../src/utils/pagination.js'
+
+import mocker from '../../utils/mocker.js'
+import { DatasetNameField, OrgField } from '../../../src/routes/schemas.js'
+
+vi.mock('../../../src/services/performanceDbApi.js')
+vi.mock('../../../src/utils/pagination.js', () => {
+  return {
+    pagination: vi.fn().mockReturnValue([
+      1, '...', 4, 5
+    ])
+  }
+})
+
+describe('issueTable.middleware.js', () => {
+  describe('setDefaultQueryParams', () => {
+    it('sets the page number when none is set', () => {
+      const req = {
+        params: {}
+      }
+      const next = vi.fn()
+
+      setDefaultQueryParams(req, {}, next)
+
+      expect(req.params.pageNumber).toEqual(1)
+      expect(next).toHaveBeenCalledOnce()
+    })
+
+    it('sets does not change the page number when one is set', () => {
+      const req = {
+        params: {
+          pageNumber: 2
+        }
+      }
+      const next = vi.fn()
+
+      setDefaultQueryParams(req, {}, next)
+      expect(req.params.pageNumber).toEqual(2)
+      expect(next).toHaveBeenCalledOnce()
+    })
+  })
+
+  describe('createPaginationTemplatePrams', () => {
+    it('should correctly set next when there is more than one page', () => {
+      const req = {
+        params: { pageNumber: 1, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
+        issueEntitiesCount: 60
+      }
+      const res = {}
+      const next = vi.fn()
+
+      const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
+
+      createPaginationTemplatePrams(req, res, next)
+
+      expect(req.pagination.previous).not.toBeDefined()
+      expect(req.pagination.next).toBeDefined()
+      expect(req.pagination.next).toEqual({
+        href: `${BaseSubpath}${2}`
+      })
+    })
+
+    it('should correct set previous when the current pageNumber is greater than 1', () => {
+      const req = {
+        params: { pageNumber: 2, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
+        issueEntitiesCount: 60
+      }
+      const res = {}
+      const next = vi.fn()
+
+      const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
+
+      createPaginationTemplatePrams(req, res, next)
+
+      expect(req.pagination.next).not.toBeDefined()
+      expect(req.pagination.previous).toBeDefined()
+      expect(req.pagination.previous).toEqual({
+        href: `${BaseSubpath}${1}`
+      })
+    })
+
+    it('should correctly set the items', () => {
+      const req = {
+        params: { pageNumber: 4, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
+        issueEntitiesCount: 60
+      }
+      const res = {}
+      const next = vi.fn()
+
+      const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
+
+      createPaginationTemplatePrams(req, res, next)
+
+      expect(req.pagination.items).toEqual([
+        {
+          current: false,
+          href: `${BaseSubpath}1`,
+          number: 1,
+          type: 'number'
+        },
+        {
+          ellipsis: true,
+          href: '#',
+          type: 'ellipsis'
+        },
+        {
+          current: true,
+          href: `${BaseSubpath}4`,
+          number: 4,
+          type: 'number'
+        },
+        {
+          current: false,
+          href: `${BaseSubpath}5`,
+          number: 5,
+          type: 'number'
+        }
+      ])
+    })
+  })
+
+  describe('prepareIssueTableTemplateParams', () => {
+    const mockedParams = mocker(IssueTableQueryParams)
+    const mockedOrg = mocker(OrgField)
+    const mockedDataset = mocker(DatasetNameField)
+
+    const req = {
+      params: mockedParams,
+      orgInfo: mockedOrg,
+      dataset: mockedDataset,
+      entitiesWithIssues: [
+        {
+          entry_number: 10,
+          'start-date': 'start-date',
+          reference: 'reference',
+          issues: '{"start-date": "invalid"}'
+        }
+      ],
+      specification: {
+        fields: [
+          { field: 'start-date', type: 'date' },
+          { field: 'reference', type: 'string' }
+        ]
+      },
+      entityCountRow: { entity_count: 50 },
+      issueEntitiesCount: 20,
+      pagination: {
+        items: [
+          { number: 1, href: '/pagenation-link-1' },
+          { number: 2, href: '/pagenation-link-2' }
+        ]
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    it('should correctly set the template params', async () => {
+      const errorHeading = 'errorHeading'
+      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValue(errorHeading)
+
+      prepareIssueTableTemplateParams(req, res, next)
+
+      const tableParams = {
+        columns: [
+          'start-date',
+          'reference'
+        ],
+        fields: [
+          'start-date',
+          'reference'
+        ],
+        rows: [
+          {
+            columns: {
+              reference: {
+                html: `<a href="/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/entry/${req.entitiesWithIssues[0].entry_number}">${req.entitiesWithIssues[0].reference}</a>`
+              },
+              'start-date': {
+                value: req.entitiesWithIssues[0]['start-date'],
+                error: {
+                  message: 'invalid'
+                }
+              }
+            }
+          }
+        ]
+      }
+
+      const expectedTemplateParams = {
+        organisation: req.orgInfo,
+        dataset: req.dataset,
+        errorHeading,
+        issueItems: [],
+        issueType: req.params.issue_type,
+        tableParams,
+        pagination: req.pagination
+      }
+
+      expect(req.templateParams).toEqual(expectedTemplateParams)
+    })
+  })
+})

From 330ab008ec6709fd9f26928dbccea82efbc3d5d9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 09:24:12 +0100
Subject: [PATCH 025/109] test cleanup

---
 src/middleware/common.middleware.js           | 79 +++++++++++++++++++
 src/middleware/issueDetails.middleware.js     | 79 ++-----------------
 src/middleware/issueTable.middleware.js       | 13 ++-
 src/routes/schemas.js                         | 20 ++---
 src/views/organisations/issueDetails.html     |  4 +-
 src/views/organisations/issueTable.html       |  4 +-
 test/unit/issueDetailsPage.test.js            | 17 ++--
 .../issueDetails.middleware.test.js           | 67 +++++++---------
 .../middleware/issueTable.middleware.test.js  | 12 +--
 .../organisations/issueTablePage.test.js      | 11 ++-
 10 files changed, 156 insertions(+), 150 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 6f3f5100..a9f680f5 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -125,3 +125,82 @@ export async function fetchIssueEntitiesCount (req, res, next) {
   req.issueEntitiesCount = parseInt(issueEntitiesCount)
   next()
 }
+
+/**
+*
+* Middleware. Updates `req` with `issues`.
+*
+* Requires `resourceId` in request params or request (in that order).
+*
+* @param {*} req
+* @param {*} res
+* @param {*} next
+*/
+export async function fetchIssues (req, res, next) {
+  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { resource: resourceId } = req.resource
+  if (!resourceId) {
+    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
+    throw Error('fetchIssues: missing resourceId')
+  }
+
+  try {
+    req.issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
+    next()
+  } catch (error) {
+    next(error)
+  }
+}
+
+/**
+ *
+ * Middleware. Updates `req` with `issues`.
+ *
+ * Requires `issues` in request.
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+export async function reformatIssuesToBeByEntryNumber (req, res, next) {
+  const { issues } = req
+  const issuesByEntryNumber = issues.reduce((acc, current) => {
+    acc[current.entry_number] = acc[current.entry_number] || []
+    acc[current.entry_number].push(current)
+    return acc
+  }, {})
+  req.issuesByEntryNumber = issuesByEntryNumber
+  next()
+}
+
+export function formatErrorSummaryParams (req, res, next) {
+  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { issuesByEntryNumber, entityCount: entityCountRow, issueEntitiesCount } = req
+
+  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
+
+  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
+
+  let errorHeading
+  let issueItems
+
+  if (Object.keys(issuesByEntryNumber).length < entityCount) {
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
+    issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
+      return {
+        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
+        href: `${BaseSubpath}${entryNumber}`
+      }
+    })
+  } else {
+    issueItems = [{
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
+    }]
+  }
+
+  req.errorSummary = {
+    heading: errorHeading,
+    items: issueItems
+  }
+  next()
+}
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 45039f5c..327378f0 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,7 +1,5 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import logger from '../utils/logger.js'
-import { types } from '../utils/logging.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, isResourceIdInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
@@ -19,54 +17,6 @@ const validateIssueDetailsQueryParams = validateQueryParams.bind({
   schema: IssueDetailsQueryParams
 })
 
-/**
- *
- * Middleware. Updates `req` with `issues`.
- *
- * Requires `resourceId` in request params or request (in that order).
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchIssues (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  if (!resourceId) {
-    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
-    throw Error('fetchIssues: missing resourceId')
-  }
-
-  try {
-    const issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
-    req.issues = issues
-    next()
-  } catch (error) {
-    next(error)
-  }
-}
-
-/**
- *
- * Middleware. Updates `req` with `issues`.
- *
- * Requires `issues` in request.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function reformatIssuesToBeByEntryNumber (req, res, next) {
-  const { issues } = req
-  const issuesByEntryNumber = issues.reduce((acc, current) => {
-    acc[current.entry_number] = acc[current.entry_number] || []
-    acc[current.entry_number].push(current)
-    return acc
-  }, {})
-  req.issuesByEntryNumber = issuesByEntryNumber
-  next()
-}
-
 /**
  *
  * Middleware. Updates `req` with `entryData`
@@ -130,6 +80,9 @@ const getIssueField = (text, html, classes) => {
 const processEntryRow = (issueType, issuesByEntryNumber, row) => {
   const { entry_number: entryNumber } = row
   console.assert(entryNumber, 'precessEntryRow(): entry_number not in row')
+
+  issuesByEntryNumber = issuesByEntryNumber || {}
+
   let hasError = false
   let issueIndex
   if (issuesByEntryNumber[entryNumber]) {
@@ -156,29 +109,11 @@ const processEntryRow = (issueType, issuesByEntryNumber, row) => {
  * Middleware. Updates req with `templateParams`
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entryData, issueEntitiesCount, issuesByEntryNumber, entityCount: entityCountRow } = req
+  const { entryData, issueEntitiesCount, issuesByEntryNumber, errorSummary } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField, entryNumber } = req.params
-  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
-
-  let errorHeading
-  let issueItems
 
   const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
 
-  if (Object.keys(issuesByEntryNumber).length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
-    issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
-      return {
-        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
-        href: `${BaseSubpath}${entryNumber}`
-      }
-    })
-  } else {
-    issueItems = [{
-      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
-    }]
-  }
-
   const fields = entryData.map((row) => processEntryRow(issueType, issuesByEntryNumber, row))
   const entityIssues = issuesByEntryNumber[entryNumber] || []
   for (const issue of entityIssues) {
@@ -241,8 +176,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   req.templateParams = {
     organisation: req.orgInfo,
     dataset: req.dataset,
-    errorHeading,
-    issueItems,
+    errorSummary,
     entry,
     issueType,
     issueField,
@@ -275,6 +209,7 @@ export default [
     fetchEntityCount,
     fetchIssueEntitiesCount
   ]),
+  formatErrorSummaryParams,
   prepareIssueDetailsTemplateParams,
   getIssueDetails,
   logPageError
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 51a045c7..e2c5554a 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,7 +1,7 @@
 import performanceDbApi from '../services/performanceDbApi.js'
 import logger from '../utils/logger.js'
 import { pagination } from '../utils/pagination.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, fetchSpecification, formatErrorSummaryParams, isResourceIdInParams, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
@@ -46,8 +46,7 @@ const fetchEntitiesWithIssues = fetchMany({
 
 export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
-  const { entitiesWithIssues, specification, entityCount: entityCountRow, issueEntitiesCount, pagination } = req
-  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
+  const { entitiesWithIssues, specification, pagination, errorSummary } = req
 
   const tableParams = {
     columns: specification.fields.map(field => field.field),
@@ -88,13 +87,10 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
     })
   }
 
-  const errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
-
   req.templateParams = {
     organisation: req.orgInfo,
     dataset: req.dataset,
-    errorHeading,
-    issueItems: [],
+    errorSummary,
     issueType,
     tableParams,
     pagination
@@ -162,8 +158,11 @@ export default [
   fetchEntitiesWithIssues,
   fetchIssueEntitiesCount,
   fetchSpecification,
+  fetchIssues,
+  reformatIssuesToBeByEntryNumber,
   pullOutDatasetSpecification,
   fetchEntityCount,
+  formatErrorSummaryParams,
   createPaginationTemplatePrams,
   prepareIssueTableTemplateParams,
   getIssueTable,
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index dcbc474c..6748701c 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -41,6 +41,14 @@ export const datasetStatusEnum = {
 export const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
 export const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
 
+export const errorSummaryField = v.strictObject({
+  heading: v.optional(v.string()),
+  items: v.array(v.strictObject({
+    html: v.string(),
+    href: v.url()
+  }))
+})
+
 const tableParams = v.strictObject({
   columns: v.array(NonEmptyString),
   rows: v.array(v.strictObject({
@@ -165,11 +173,7 @@ export const OrgEndpointError = v.strictObject({
 export const OrgIssueDetails = v.strictObject({
   organisation: OrgField,
   dataset: DatasetNameField,
-  errorHeading: v.optional(NonEmptyString),
-  issueItems: v.array(v.strictObject({
-    html: v.string(),
-    href: v.url()
-  })),
+  errorSummary: errorSummaryField,
   issueType: NonEmptyString,
   issueField: NonEmptyString,
   entry: v.strictObject({
@@ -189,11 +193,7 @@ export const OrgIssueDetails = v.strictObject({
 export const OrgIssueTable = v.strictObject({
   organisation: OrgField,
   dataset: DatasetNameField,
-  errorHeading: v.optional(NonEmptyString),
-  issueItems: v.array(v.strictObject({
-    html: v.string(),
-    href: v.url()
-  })),
+  errorSummary: errorSummaryField,
   issueType: NonEmptyString,
   tableParams,
   pagination: paginationParams
diff --git a/src/views/organisations/issueDetails.html b/src/views/organisations/issueDetails.html
index 4dc614c8..2d11fa9b 100644
--- a/src/views/organisations/issueDetails.html
+++ b/src/views/organisations/issueDetails.html
@@ -57,8 +57,8 @@
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
     {{ govukErrorSummary({
-      titleText: errorHeading if errorHeading else 'There is a problem',
-      errorList: issueItems
+      titleText: errorSummary.heading if errorSummary.heading else 'There is a problem',
+      errorList: errorSummary.items
     }) }}
 
     {% if entry.geometries and entry.geometries.length %}
diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
index 64e2212c..adee5cb5 100644
--- a/src/views/organisations/issueTable.html
+++ b/src/views/organisations/issueTable.html
@@ -55,8 +55,8 @@
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
     {{ govukErrorSummary({
-      titleText: errorHeading if errorHeading else 'There is a problem',
-      errorList: issueItems
+      titleText: errorSummary.heading if errorSummary.heading else 'There is a problem',
+      errorList: errorSummary.items
     }) }}
 
     {% if false and entry.geometries and entry.geometries.length %}
diff --git a/test/unit/issueDetailsPage.test.js b/test/unit/issueDetailsPage.test.js
index e97b2575..d963968e 100644
--- a/test/unit/issueDetailsPage.test.js
+++ b/test/unit/issueDetailsPage.test.js
@@ -37,13 +37,16 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
 
   describe('error summary', () => {
     it('should render the correct heading', () => {
-      expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorHeading || 'There is a problem')
+      expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorSummary.heading || 'There is a problem')
     })
 
     it('should render the correct heading if none is supplied', () => {
       const noErrorHeadingPageHtml = nunjucks.render('organisations/issueDetails.html', {
         ...params,
-        errorHeading: undefined
+        errorSummary: {
+          heading: undefined,
+          items: params.errorSummary.items
+        }
       })
 
       const domNoErrorHeading = new JSDOM(noErrorHeadingPageHtml)
@@ -55,10 +58,10 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
     it('should render the issue items', () => {
       const issueList = document.querySelector('.govuk-error-summary__list')
       const issueItemElements = [...issueList.children]
-      expect(issueItemElements.length).toBe(params.issueItems.length)
+      expect(issueItemElements.length).toBe(params.errorSummary.items.length)
 
       issueItemElements.forEach((element, index) => {
-        expect(element.textContent).toContain(params.issueItems[index].html)
+        expect(element.textContent).toContain(params.errorSummary.items[index].html)
       })
     })
   })
@@ -177,8 +180,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
       const paramWithGeometry = {
         organisation: params.organisation,
         dataset: params.dataset,
-        errorHeading: params.errorHeading,
-        issueItems: params.issueItems,
+        errorSummary: params.errorSummary,
         entry: {
           ...params.entry,
           geometries: ['POINT(0 0)']
@@ -201,8 +203,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
       const paramWithGeometry = {
         organisation: params.organisation,
         dataset: params.dataset,
-        errorHeading: params.errorHeading,
-        issueItems: params.issueItems,
+        errorSummary: params.errorSummary,
         entry: {
           ...params.entry,
           geometries: []
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 7e32b190..938541f0 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -3,6 +3,8 @@ import * as v from 'valibot'
 
 import performanceDbApi from '../../../src/services/performanceDbApi.js'
 import { getIssueDetails, IssueDetailsQueryParams, prepareIssueDetailsTemplateParams } from '../../../src/middleware/issueDetails.middleware.js'
+import mocker from '../../utils/mocker.js'
+import { DatasetNameField, errorSummaryField, OrgField } from '../../../src/routes/schemas.js'
 
 vi.mock('../../../src/services/performanceDbApi.js')
 
@@ -55,8 +57,16 @@ describe('issueDetails.middleware.js', () => {
               issue_type: 'mock type'
             }
           ]
+        },
+        errorSummary: {
+          heading: 'mockHeading',
+          items: [
+            {
+              html: 'mock task message 1 in record 10',
+              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
+            }
+          ]
         }
-        // errorHeading -- set  in prepare* fn
       }
       v.parse(IssueDetailsQueryParams, req.params)
 
@@ -77,13 +87,7 @@ describe('issueDetails.middleware.js', () => {
           dataset: 'mock-dataset',
           collection: 'mock-collection'
         },
-        errorHeading: 'mockMessageFor: 10',
-        issueItems: [
-          {
-            html: 'mock task message 1 in record 10',
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
-          }
-        ],
+        errorSummary: req.errorSummary,
         entry: {
           title: 'entry: 10',
           fields: [
@@ -143,19 +147,23 @@ describe('issueDetails.middleware.js', () => {
         entryData,
         issues,
         resource: { resource: requestParams.resourceId },
+        errorSummary: {
+          heading: 'mock heading',
+          items: [
+            {
+              html: 'mock task message 1 in record 10',
+              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
+            }
+          ]
+        },
         issuesByEntryNumber: {
           10: [
             {
               field: 'start-date',
-              value: '02-02-2022',
-              line_number: 1,
-              entry_number: 10,
-              message: 'mock message',
-              issue_type: 'mock type'
+              message: 'mock message'
             }
           ]
         }
-        // errorHeading -- set  in prepare* fn
       }
 
       v.parse(IssueDetailsQueryParams, req.params)
@@ -177,13 +185,7 @@ describe('issueDetails.middleware.js', () => {
           dataset: 'mock-dataset',
           collection: 'mock-collection'
         },
-        errorHeading: 'mockMessageFor: 10',
-        issueItems: [
-          {
-            html: 'mock task message 1 in record 10',
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
-          }
-        ],
+        errorSummary: req.errorSummary,
         entry: {
           title: 'entry: 10',
           fields: [
@@ -223,24 +225,15 @@ describe('issueDetails.middleware.js', () => {
 
   describe('getIssueDetails', () => {
     it('should call render using the provided template params and correct view', () => {
+      const mockedOrg = mocker(OrgField)
+      const mockedDataset = mocker(DatasetNameField)
+      const mockErrorSummary = mocker(errorSummaryField)
+
       const req = {
         templateParams: {
-          organisation: {
-            name: 'mock lpa',
-            organisation: 'ORG'
-          },
-          dataset: {
-            name: 'mock dataset',
-            dataset: 'mock-dataset',
-            collection: 'mock-collection'
-          },
-          errorHeading: 'mockMessageFor: 0',
-          issueItems: [
-            {
-              html: 'mock task message 1 in record 1',
-              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
-            }
-          ],
+          organisation: mockedOrg,
+          dataset: mockedDataset,
+          errorSummary: mockErrorSummary,
           entry: {
             title: 'entry: 1',
             fields: [
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 3902fde4..e9898e38 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -1,11 +1,9 @@
 import { describe, it, vi, expect } from 'vitest'
-
-import performanceDbApi from '../../../src/services/performanceDbApi.js'
 import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, createPaginationTemplatePrams } from '../../../src/middleware/issueTable.middleware.js'
 // import { pagination } from '../../../src/utils/pagination.js'
 
 import mocker from '../../utils/mocker.js'
-import { DatasetNameField, OrgField } from '../../../src/routes/schemas.js'
+import { DatasetNameField, errorSummaryField, OrgField } from '../../../src/routes/schemas.js'
 
 vi.mock('../../../src/services/performanceDbApi.js')
 vi.mock('../../../src/utils/pagination.js', () => {
@@ -127,11 +125,13 @@ describe('issueTable.middleware.js', () => {
     const mockedParams = mocker(IssueTableQueryParams)
     const mockedOrg = mocker(OrgField)
     const mockedDataset = mocker(DatasetNameField)
+    const mockedErrorSummary = mocker(errorSummaryField)
 
     const req = {
       params: mockedParams,
       orgInfo: mockedOrg,
       dataset: mockedDataset,
+      errorSummary: mockedErrorSummary,
       entitiesWithIssues: [
         {
           entry_number: 10,
@@ -159,9 +159,6 @@ describe('issueTable.middleware.js', () => {
     const next = vi.fn()
 
     it('should correctly set the template params', async () => {
-      const errorHeading = 'errorHeading'
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValue(errorHeading)
-
       prepareIssueTableTemplateParams(req, res, next)
 
       const tableParams = {
@@ -193,8 +190,7 @@ describe('issueTable.middleware.js', () => {
       const expectedTemplateParams = {
         organisation: req.orgInfo,
         dataset: req.dataset,
-        errorHeading,
-        issueItems: [],
+        errorSummary: req.errorSummary,
         issueType: req.params.issue_type,
         tableParams,
         pagination: req.pagination
diff --git a/test/unit/views/organisations/issueTablePage.test.js b/test/unit/views/organisations/issueTablePage.test.js
index 12d0798a..9abeae3a 100644
--- a/test/unit/views/organisations/issueTablePage.test.js
+++ b/test/unit/views/organisations/issueTablePage.test.js
@@ -35,13 +35,16 @@ const runTestsWithSeed = (seed) => {
 
     describe('error summary', () => {
       it('should render the correct heading', () => {
-        expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorHeading || 'There is a problem')
+        expect(document.querySelector('.govuk-error-summary__title').textContent).toContain(params.errorSummary.heading || 'There is a problem')
       })
 
       it('should render the correct heading if none is supplied', () => {
         const noErrorHeadingPageHtml = nunjucks.render('organisations/issueTable.html', {
           ...params,
-          errorHeading: undefined
+          errorSummary: {
+            items: params.errorSummary.items,
+            heading: undefined
+          }
         })
 
         const domNoErrorHeading = new JSDOM(noErrorHeadingPageHtml)
@@ -53,10 +56,10 @@ const runTestsWithSeed = (seed) => {
       it('should render the issue items', () => {
         const issueList = document.querySelector('.govuk-error-summary__list')
         const issueItemElements = [...issueList.children]
-        expect(issueItemElements.length).toBe(params.issueItems.length)
+        expect(issueItemElements.length).toBe(params.errorSummary.items.length)
 
         issueItemElements.forEach((element, index) => {
-          expect(element.textContent).toContain(params.issueItems[index].html)
+          expect(element.textContent).toContain(params.errorSummary.items[index].html)
         })
       })
     })

From d9997a3a6a6d11259f5e116040d3c57f3d213ffe Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 09:50:44 +0100
Subject: [PATCH 026/109] update resource in params to resource not in params

---
 src/middleware/common.middleware.js          | 2 +-
 src/middleware/datasetOverview.middleware.js | 4 ++--
 src/middleware/issueDetails.middleware.js    | 4 ++--
 src/middleware/issueTable.middleware.js      | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index a9f680f5..c77b5a3b 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -41,7 +41,7 @@ export const fetchDatasetInfo = fetchOne({
  */
 export const isResourceAccessible = (req) => req.resourceStatus.status === '200'
 export const isResourceNotAccessible = (req) => !isResourceAccessible(req)
-export const isResourceIdInParams = ({ params }) => !('resourceId' in params)
+export const isResourceIdNotInParams = ({ params }) => !('resourceId' in params)
 
 /**
  * Middleware. Updates req with `resource`.
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index cff08269..0e9611a0 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,4 +1,4 @@
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
 import { fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
 
@@ -96,7 +96,7 @@ export default [
   parallel([
     fetchColumnSummary,
     fetchResourceStatus,
-    fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams)
+    fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams)
   ]),
   fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
   fetchSpecification,
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 327378f0..8c1dbc25 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,5 +1,5 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, isResourceIdInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
@@ -201,7 +201,7 @@ export default [
   validateIssueDetailsQueryParams,
   fetchOrgInfo,
   fetchDatasetInfo,
-  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchIssues,
   reformatIssuesToBeByEntryNumber,
   parallel([
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index e2c5554a..527c0ce0 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,7 +1,7 @@
 import performanceDbApi from '../services/performanceDbApi.js'
 import logger from '../utils/logger.js'
 import { pagination } from '../utils/pagination.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, fetchSpecification, formatErrorSummaryParams, isResourceIdInParams, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, fetchSpecification, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
@@ -154,7 +154,7 @@ export default [
     fetchOrgInfo,
     fetchDatasetInfo
   ]),
-  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchEntitiesWithIssues,
   fetchIssueEntitiesCount,
   fetchSpecification,

From 330cc74c0ab6c01f8716deec7f6e6fe4d9a68148 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 09:50:53 +0100
Subject: [PATCH 027/109] add tests for common middleware

---
 .../unit/middleware/common.middleware.test.js | 96 ++++++++++++++++++-
 1 file changed, 93 insertions(+), 3 deletions(-)

diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 64d92360..2c2e24d2 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -1,5 +1,95 @@
-import { describe, it, expect } from 'vitest'
-import { pullOutDatasetSpecification } from '../../../src/middleware/common.middleware'
+import { describe, it, expect, vi } from 'vitest'
+import { formatErrorSummaryParams, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
+import logger from '../../../src/utils/logger'
+
+vi.mock('../../../src/utils/logger')
+
+describe('logPageError', () => {
+  it('logs an error with handlerName', () => {
+    const loggerMock = vi.fn()
+    logger.warn = loggerMock
+
+    const err = new Error('Test error')
+    const req = { handlerName: 'testHandler', originalUrl: '/test' }
+    const res = {}
+    const next = vi.fn()
+    logPageError(err, req, res, next)
+    expect(loggerMock).toHaveBeenCalledTimes(1)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+})
+
+describe('resource middleware', () => {
+  describe('isResourceAccessible', () => {
+    it('returns true if resourceStatus is 200', () => {
+      const req = { resourceStatus: { status: '200' } }
+      expect(isResourceAccessible(req)).toBe(true)
+    })
+    it('returns false if resourceStatus is not 200', () => {
+      const req = { resourceStatus: { status: '404' } }
+      expect(isResourceAccessible(req)).toBe(false)
+    })
+  })
+
+  describe('isResourceNotAccessible', () => {
+    it('returns false if resourceStatus is 200', () => {
+      const req = { resourceStatus: { status: '200' } }
+      expect(isResourceNotAccessible(req)).toBe(false)
+    })
+    it('returns true if resourceStatus is not 200', () => {
+      const req = { resourceStatus: { status: '404' } }
+      expect(isResourceNotAccessible(req)).toBe(true)
+    })
+  })
+
+  describe('isResourceIdNotInParams', () => {
+    it('returns true if resourceId is in params', () => {
+      const req = { params: { resourceId: 'testId' } }
+      expect(isResourceIdNotInParams(req)).toBe(false)
+    })
+    it('returns false if resourceId is not in params', () => {
+      const req = { params: {} }
+      expect(isResourceIdNotInParams(req)).toBe(true)
+    })
+  })
+
+  describe('takeResourceIdFromParams', () => {
+    it('takes the resourceId from params', () => {
+      const req = { params: { resourceId: 'testId' } }
+      takeResourceIdFromParams(req)
+      expect(req.resource).toEqual({ resource: 'testId' })
+    })
+  })
+})
+
+describe('reformatIssuesToBeByEntryNumber', () => {
+  it('reformats the issues by entry number', async () => {
+    const req = { issues: [{ entry_number: '1', issue: 'testIssue' }, { entry_number: '1', issue: 'testIssue2' }, { entry_number: '2', issue: 'testIssue3' }] }
+    const res = {}
+    const next = vi.fn()
+    await reformatIssuesToBeByEntryNumber(req, res, next)
+    expect(req.issuesByEntryNumber).toBeDefined()
+    expect(req.issuesByEntryNumber['1']).toHaveLength(2)
+    expect(req.issuesByEntryNumber['2']).toHaveLength(1)
+  })
+})
+
+describe('formatErrorSummaryParams', () => {
+  it('formats the error summary params', async () => {
+    const req = {
+      params: { lpa: 'testLpa', dataset: 'testDataset', issue_type: 'testIssueType', issue_field: 'testIssueField' },
+      issuesByEntryNumber: { 1: [{ issue: 'testIssue' }], 2: [{ issue: 'testIssue2' }] },
+      entityCount: { entity_count: 10 },
+      issueEntitiesCount: 5
+    }
+    const res = {}
+    const next = vi.fn()
+    formatErrorSummaryParams(req, res, next)
+    expect(req.errorSummary).toBeDefined()
+    expect(req.errorSummary.heading).toBeDefined()
+    expect(req.errorSummary.items).toHaveLength(2)
+  })
+})
 
 describe('pullOutDatasetSpecification', () => {
   const req = {
@@ -15,7 +105,7 @@ describe('pullOutDatasetSpecification', () => {
   }
   const res = {}
 
-  it('', () => {
+  it('pulls out the data specification', () => {
     const reqWithSpecification = {
       ...req,
       specification: {

From 2fa1c4e4ed750d760ddd565c91c8488495b03fe9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 09:56:50 +0100
Subject: [PATCH 028/109] remove json5 import and use

---
 src/middleware/common.middleware.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index c77b5a3b..8e1652a4 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -3,7 +3,6 @@ import { types } from '../utils/logging.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { fetchOne, FetchOptions, FetchOneFallbackPolicy, fetchMany } from './middleware.builders.js'
 import * as v from 'valibot'
-import json5 from 'json5'
 
 /**
  * Middleware. Set `req.handlerName` to a string that will identify
@@ -101,7 +100,7 @@ export const fetchSpecification = fetchOne({
 
 export const pullOutDatasetSpecification = (req, res, next) => {
   const { specification } = req
-  const collectionSpecifications = json5.parse(specification.json)
+  const collectionSpecifications = JSON.parse(specification.json)
   const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
   req.specification = datasetSpecification
   next()

From c068ae8d9646982449d31412d3818f01ab8ee12e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 10:01:42 +0100
Subject: [PATCH 029/109] change default value to destructuring

---
 src/middleware/issueDetails.middleware.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 8c1dbc25..34c74d82 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -77,12 +77,10 @@ const getIssueField = (text, html, classes) => {
  * @param {*} row
  * @returns {{key: {text: string}, value: { html: string}, classes: string}}
  */
-const processEntryRow = (issueType, issuesByEntryNumber, row) => {
+const processEntryRow = (issueType, issuesByEntryNumber = {}, row) => {
   const { entry_number: entryNumber } = row
   console.assert(entryNumber, 'precessEntryRow(): entry_number not in row')
 
-  issuesByEntryNumber = issuesByEntryNumber || {}
-
   let hasError = false
   let issueIndex
   if (issuesByEntryNumber[entryNumber]) {

From fb6f607b48959597f1b315b94e3cdae0f4783fb4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 15:29:26 +0100
Subject: [PATCH 030/109] change issue details to query on pageNumber again and
 improve error handling

---
 src/middleware/issueDetails.middleware.js | 70 ++++++++++++++++++-----
 src/middleware/issueTable.middleware.js   | 24 +++++++-
 src/routes/organisations.js               |  2 +-
 3 files changed, 81 insertions(+), 15 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 34c74d82..72aa751e 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -4,12 +4,12 @@ import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 
-export const IssueDetailsQueryParams = v.object({
+export const IssueDetailsQueryParams = v.strictObject({
   lpa: v.string(),
   dataset: v.string(),
   issue_type: v.string(),
   issue_field: v.string(),
-  pageNumber: v.optional(v.string()),
+  pageNumber: v.string(),
   resourceId: v.optional(v.string())
 })
 
@@ -21,7 +21,7 @@ const validateIssueDetailsQueryParams = validateQueryParams.bind({
  *
  * Middleware. Updates `req` with `entryData`
  *
- * Requires `pageNumber`, `dataset` and
+ * Requires `dataset` and `entityNumber`
  *
  * @param {*} req
  * @param {*} res
@@ -29,7 +29,8 @@ const validateIssueDetailsQueryParams = validateQueryParams.bind({
  *
  */
 async function fetchEntry (req, res, next) {
-  const { dataset: datasetId, entryNumber } = req.params
+  const { dataset: datasetId } = req.params
+  const { entryNumber } = req
 
   req.entryData = await performanceDbApi.getEntry(
     req.resource.resource,
@@ -103,12 +104,55 @@ const processEntryRow = (issueType, issuesByEntryNumber = {}, row) => {
   return getIssueField(row.field, valueHtml, classes)
 }
 
-/***
- * Middleware. Updates req with `templateParams`
+/**
+ * Middleware. Extracts the entry number from the page number in the request.
+ *
+ * @param {object} req - The request object
+ * @param {object} res - The response object
+ * @param {function} next - The next middleware function
+ *
+ * @throws {Error} If the page number cannot be parsed as an integer
+ * @throws {Error} If the entry number is not found (404)
+ */
+export function getEntryNumberFromPageNumber (req, res, next) {
+  const { issuesByEntryNumber } = req
+  const { pageNumber } = req.params
+
+  const pageNumberAsInt = parseInt(pageNumber)
+  if (isNaN(pageNumberAsInt)) {
+    const error = new Error('page number could not be parsed as an integer')
+    return next(error)
+  }
+
+  const issuesByEntryNumberIndex = pageNumberAsInt - 1
+  const pageNumberToEntryNumberMap = Object.keys(issuesByEntryNumber)
+
+  if (issuesByEntryNumberIndex < 0 || issuesByEntryNumberIndex >= pageNumberToEntryNumberMap.length) {
+    const error = new Error('not found')
+    error.status = 404
+    return next(error)
+  }
+
+  req.entryNumber = pageNumberToEntryNumberMap[issuesByEntryNumberIndex]
+  next()
+}
+
+/**
+ * Middleware. Prepares template parameters for the issue details page.
+ *
+ * @param {object} req - The request object
+ * @param {object} res - The response object (not used)
+ * @param {function} next - The next middleware function
+ *
+ * @summary Extracts relevant data from the request and organizes it into a template parameters object.
+ * @description This middleware function prepares the template parameters for the issue details page.
+ * It extracts the entry data, issue entities count, issues by entry number, error summary, and other relevant data
+ * from the request, and organizes it into a template parameters object that can be used to render the page.
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entryData, issueEntitiesCount, issuesByEntryNumber, errorSummary } = req
-  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField, entryNumber } = req.params
+  const { entryData, issueEntitiesCount, issuesByEntryNumber, errorSummary, entryNumber } = req
+  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField, pageNumber: pageNumberString } = req.params
+  const pageNumber = parseInt(pageNumberString)
 
   const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
 
@@ -139,17 +183,16 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   }
 
   const entryNumbers = Object.keys(issuesByEntryNumber)
-  const pageNumber = entryNumbers.findIndex(currentEntryNumber => currentEntryNumber === entryNumber) + 1
 
   if (pageNumber > 1) {
     paginationObj.previous = {
-      href: `${BaseSubpath}${entryNumbers[pageNumber - 1]}`
+      href: `${BaseSubpath}${pageNumber - 1}`
     }
   }
 
-  if (pageNumber < issueEntitiesCount) {
+  if (pageNumber < entryNumbers.length) {
     paginationObj.next = {
-      href: `${BaseSubpath}${entryNumbers[pageNumber + 1]}`
+      href: `${BaseSubpath}${pageNumber + 1}`
     }
   }
 
@@ -164,7 +207,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
       return {
         type: 'number',
         number: item,
-        href: `${BaseSubpath}${entryNumbers[item - 1]}`,
+        href: `${BaseSubpath}${item}`,
         current: pageNumber === parseInt(item)
       }
     }
@@ -202,6 +245,7 @@ export default [
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchIssues,
   reformatIssuesToBeByEntryNumber,
+  getEntryNumberFromPageNumber,
   parallel([
     fetchEntry,
     fetchEntityCount,
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 527c0ce0..e353b3b3 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -44,6 +44,13 @@ const fetchEntitiesWithIssues = fetchMany({
   dataset: FetchOptions.fromParams
 })
 
+/**
+ * Middleware function to prepare issue table template params
+ *
+ * @param {Request} req - Express request object
+ * @param {Response} res - Express response object
+ * @param {NextFunction} next - Next function in the middleware chain
+ */
 export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
   const { entitiesWithIssues, specification, pagination, errorSummary } = req
@@ -57,7 +64,8 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
       specification.fields.forEach(fieldObject => {
         const { field } = fieldObject
         if (field === 'reference') {
-          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${entity.entry_number}`
+          const pageNumber = index + 1
+          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${pageNumber}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field]}</a>` }
         } else if (entity[field]) {
           columns[field] = { value: entity[field] }
@@ -98,6 +106,20 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
   next()
 }
 
+/**
+ * Creates pagination template parameters for the request.
+ *
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @param {Function} next - The next middleware function in the chain.
+ *
+ * @description
+ * This middleware function extracts pagination-related parameters from the request,
+ * calculates the total number of pages, and creates a pagination object that can be used
+ * to render pagination links in the template.
+ *
+ * @returns {void}
+ */
 export const createPaginationTemplatePrams = (req, res, next) => {
   const { issueEntitiesCount } = req
   const { pageNumber, lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
diff --git a/src/routes/organisations.js b/src/routes/organisations.js
index d3a0221f..6afe148a 100644
--- a/src/routes/organisations.js
+++ b/src/routes/organisations.js
@@ -5,7 +5,7 @@ const router = express.Router()
 
 router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStartedMiddleware)
 router.get('/:lpa/:dataset/overview', OrganisationsController.getDatasetOverviewMiddleware)
-router.get('/:lpa/:dataset/:issue_type/:issue_field/entry/:entryNumber?', OrganisationsController.getIssueDetailsMiddleware)
+router.get('/:lpa/:dataset/:issue_type/:issue_field/entry/:pageNumber?', OrganisationsController.getIssueDetailsMiddleware)
 router.get('/:lpa/:dataset/:issue_type/:issue_field/:pageNumber?', OrganisationsController.getIssueTableMiddleware)
 router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskListMiddleware)
 router.get('/:lpa', OrganisationsController.getOverviewMiddleware)

From 102854d892deced2a330c4a879f2bf31ea8224a2 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 9 Oct 2024 15:43:12 +0100
Subject: [PATCH 031/109] fix tests

---
 test/unit/middleware/issueDetails.middleware.test.js | 9 +++++----
 test/unit/middleware/issueTable.middleware.test.js   | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 938541f0..5252bdc8 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -34,7 +34,7 @@ describe('issueDetails.middleware.js', () => {
         issue_type: 'test-issue-type',
         issue_field: 'test-issue-field',
         resourceId: 'test-resource-id',
-        entryNumber: '10'
+        pageNumber: '1'
       }
       const req = {
         params: requestParams,
@@ -45,6 +45,7 @@ describe('issueDetails.middleware.js', () => {
         dataset,
         entryData,
         issues,
+        entryNumber: 10,
         resource: { resource: requestParams.resourceId },
         issuesByEntryNumber: {
           10: [
@@ -104,7 +105,7 @@ describe('issueDetails.middleware.js', () => {
         pagination: {
           items: [{
             current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/1',
             number: 1,
             type: 'number'
           }]
@@ -134,7 +135,7 @@ describe('issueDetails.middleware.js', () => {
         issue_type: 'test-issue-type',
         issue_field: 'test-issue-field',
         resourceId: 'test-resource-id',
-        entryNumber: '10'
+        pageNumber: '1'
       }
       const req = {
         params: requestParams,
@@ -211,7 +212,7 @@ describe('issueDetails.middleware.js', () => {
         pagination: {
           items: [{
             current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/1',
             number: 1,
             type: 'number'
           }]
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index e9898e38..1a0d3497 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -174,7 +174,7 @@ describe('issueTable.middleware.js', () => {
           {
             columns: {
               reference: {
-                html: `<a href="/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/entry/${req.entitiesWithIssues[0].entry_number}">${req.entitiesWithIssues[0].reference}</a>`
+                html: `<a href="/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/entry/${1}">${req.entitiesWithIssues[0].reference}</a>`
               },
               'start-date': {
                 value: req.entitiesWithIssues[0]['start-date'],

From c30f15fb974c35fe5c7ca6efd061dba4040589e9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 10 Oct 2024 09:48:09 +0100
Subject: [PATCH 032/109] addressing git comments

---
 src/middleware/issueDetails.middleware.js | 3 ++-
 src/middleware/issueTable.middleware.js   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 72aa751e..9904f802 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -21,7 +21,7 @@ const validateIssueDetailsQueryParams = validateQueryParams.bind({
  *
  * Middleware. Updates `req` with `entryData`
  *
- * Requires `dataset` and `entityNumber`
+ * Requires `dataset` and `entryNumber`
  *
  * @param {*} req
  * @param {*} res
@@ -121,6 +121,7 @@ export function getEntryNumberFromPageNumber (req, res, next) {
   const pageNumberAsInt = parseInt(pageNumber)
   if (isNaN(pageNumberAsInt)) {
     const error = new Error('page number could not be parsed as an integer')
+    error.status = 400
     return next(error)
   }
 
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index e353b3b3..6a533bd9 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -74,7 +74,7 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
         }
       })
 
-      let issues
+      let issues = {}
       try {
         issues = JSON.parse(entity.issues)
       } catch (e) {

From f5c254fa91886df86d6a73d5c1bc1c8b20d8d9c6 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 11 Oct 2024 15:26:27 +0100
Subject: [PATCH 033/109] query overhall after entity vs entry talk

---
 src/middleware/common.middleware.js     | 104 ++++++++++++++++++++----
 src/middleware/issueTable.middleware.js |  80 ++++++++----------
 src/services/performanceDbApi.js        |  44 +++++++---
 3 files changed, 156 insertions(+), 72 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 8e1652a4..04e1527d 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -126,25 +126,21 @@ export async function fetchIssueEntitiesCount (req, res, next) {
 }
 
 /**
-*
-* Middleware. Updates `req` with `issues`.
-*
-* Requires `resourceId` in request params or request (in that order).
-*
-* @param {*} req
-* @param {*} res
-* @param {*} next
-*/
+ * Fetches issues from the performance database and updates the request object with the result.
+ *
+ * This middleware requires the `resourceId` to be present in the request params or request object.
+ *
+ * @param {object} req - The HTTP request object
+ * @param {object} res - The HTTP response object
+ * @param {function} next - The next middleware function in the stack
+ *
+ * @throws {Error} If `resourceId` is missing from the request
+ */
 export async function fetchIssues (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  if (!resourceId) {
-    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
-    throw Error('fetchIssues: missing resourceId')
-  }
+  const { dataset, issue_type: issueType, issue_field: issueField, lpa } = req.params
 
   try {
-    req.issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
+    req.issues = await performanceDbApi.getIssues({ organisation: lpa, dataset, issueType, issueField })
     next()
   } catch (error) {
     next(error)
@@ -203,3 +199,79 @@ export function formatErrorSummaryParams (req, res, next) {
   }
   next()
 }
+
+export const getEntryNumbersWithIssues = (req, res, next) => {
+  const { issues } = req
+
+  const entryNumbersWithIssues = [...new Set(issues.map(issue => issue.entry_number))]
+
+  req.entryNumbersWithIssues = entryNumbersWithIssues
+
+  next()
+}
+
+export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
+  query: ({ req, params }) => performanceDbApi.fetchEntityNumbersFromEntryNumbers({ entryNumbers: req.entryNumbersWithIssues, organisationEntity: req.orgInfo.entity }),
+  result: 'entities',
+  dataset: FetchOptions.fromParams
+})
+
+export const extractJsonFieldFromEntities = (req, res, next) => {
+  const { entities } = req
+
+  req.entities = entities.map(entity => {
+    const jsonField = entity.json
+    delete entity.json
+    const parsedJson = JSON.parse(jsonField)
+    entity = { ...entity, ...parsedJson }
+    return entity
+  })
+
+  next()
+}
+
+export const replaceUnderscoreWithHyphenForEntities = (req, res, next) => {
+  const { entities } = req
+
+  entities.forEach(entity => {
+    Object.keys(entity).forEach(key => {
+      if (key.includes('_')) {
+        const newKey = key.replace(/_/g, '-')
+        entity[newKey] = entity[key]
+        delete entity[key]
+      }
+    })
+  })
+
+  next()
+}
+
+export const nestEntityFields = (req, res, next) => {
+  const { entities, specification } = req
+
+  req.entities = entities.map(entity => {
+    specification.fields.forEach(field => {
+      entity[field.field] = { value: entity[field.field] }
+    })
+    return entity
+  })
+
+  next()
+}
+
+export const addIssuesToEntities = (req, res, next) => {
+  const { entities, issues } = req
+
+  req.entitiesWithIssues = entities.map(entity => {
+    const entityIssues = issues.filter(issue => issue.entryNumber === entity.entryNumber)
+
+    entityIssues.forEach(issue => {
+      entity[issue.field].value = issue.value
+      entity[issue.field].issue = issue
+    })
+
+    return entity
+  })
+
+  next()
+}
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 6a533bd9..e4086f06 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,8 +1,27 @@
-import performanceDbApi from '../services/performanceDbApi.js'
-import logger from '../utils/logger.js'
 import { pagination } from '../utils/pagination.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, fetchSpecification, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
-import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
+import {
+  addIssuesToEntities,
+  extractJsonFieldFromEntities,
+  fetchDatasetInfo,
+  fetchEntitiesFromOrganisationAndEntryNumbers,
+  fetchEntityCount,
+  fetchIssueEntitiesCount,
+  fetchIssues,
+  fetchLatestResource,
+  fetchOrgInfo,
+  fetchSpecification,
+  formatErrorSummaryParams,
+  getEntryNumbersWithIssues,
+  isResourceIdNotInParams,
+  logPageError,
+  nestEntityFields,
+  pullOutDatasetSpecification,
+  reformatIssuesToBeByEntryNumber,
+  replaceUnderscoreWithHyphenForEntities,
+  takeResourceIdFromParams,
+  validateQueryParams
+} from './common.middleware.js'
+import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
 const paginationPageLength = 50
@@ -27,23 +46,6 @@ export const setDefaultQueryParams = (req, res, next) => {
   next()
 }
 
-const fetchEntitiesWithIssues = fetchMany({
-  query: ({ req, params }) => {
-    const pagination = {
-      limit: paginationPageLength,
-      offset: paginationPageLength * (params.pageNumber - 1)
-    }
-    return performanceDbApi.entitiesAndIssuesQuery({
-      resource: req.resource.resource,
-      issueType: req.params.issue_type,
-      issueField: req.params.issue_field,
-      pagination
-    })
-  },
-  result: 'entitiesWithIssues',
-  dataset: FetchOptions.fromParams
-})
-
 /**
  * Middleware function to prepare issue table template params
  *
@@ -53,12 +55,12 @@ const fetchEntitiesWithIssues = fetchMany({
  */
 export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
-  const { entitiesWithIssues, specification, pagination, errorSummary } = req
+  const { entities, specification, pagination, errorSummary } = req
 
   const tableParams = {
     columns: specification.fields.map(field => field.field),
     fields: specification.fields.map(field => field.field),
-    rows: entitiesWithIssues.map((entity, index) => {
+    rows: entities.map((entity, index) => {
       const columns = {}
 
       specification.fields.forEach(fieldObject => {
@@ -66,29 +68,14 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
         if (field === 'reference') {
           const pageNumber = index + 1
           const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${pageNumber}`
-          columns[field] = { html: `<a href="${entityLink}">${entity[field]}</a>` }
+          columns[field] = { html: `<a href="${entityLink}">${entity[field].value}</a>`, error: entity[field].issue }
         } else if (entity[field]) {
-          columns[field] = { value: entity[field] }
+          columns[field] = { value: entity[field].value, error: entity[field].issue }
         } else {
           columns[field] = { value: '' }
         }
       })
 
-      let issues = {}
-      try {
-        issues = JSON.parse(entity.issues)
-      } catch (e) {
-        logger.warn('issueTableMiddleware:prepareIssueTableParams - entity issues is not valid json', { entityIssues: entity.issues })
-      }
-
-      Object.entries(issues).forEach(([field, issueType]) => {
-        if (columns[field]) {
-          columns[field].error = { message: issueType }
-        } else {
-          columns[field] = { value: '', error: { message: issueType } }
-        }
-      })
-
       return {
         columns
       }
@@ -177,13 +164,18 @@ export default [
     fetchDatasetInfo
   ]),
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
-  fetchEntitiesWithIssues,
-  fetchIssueEntitiesCount,
   fetchSpecification,
-  fetchIssues,
-  reformatIssuesToBeByEntryNumber,
   pullOutDatasetSpecification,
+  fetchIssues,
+  getEntryNumbersWithIssues,
+  fetchEntitiesFromOrganisationAndEntryNumbers,
+  extractJsonFieldFromEntities,
+  replaceUnderscoreWithHyphenForEntities,
+  nestEntityFields,
+  addIssuesToEntities,
+  fetchIssueEntitiesCount,
   fetchEntityCount,
+  reformatIssuesToBeByEntryNumber,
   formatErrorSummaryParams,
   createPaginationTemplatePrams,
   prepareIssueTableTemplateParams,
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index aa91b8f4..668495b5 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -364,18 +364,28 @@ export default {
      * @param {string} [database="digital-land"] - Database to query (defaults to "digital-land")
      * @returns {Promise<Object>} - Promise resolving to an object with formatted data
      */
-  async getIssues ({
-    resource,
-    issueType,
-    issueField
-  }, database = 'digital-land') {
-    const sql = `
-    SELECT i.field, i.line_number, entry_number, message, issue_type, value
-    FROM issue i
-    WHERE resource = '${resource}'
-    AND issue_type = '${issueType}'
-    AND field = '${issueField}'
-  `
+  async getIssues ({ organisation, dataset, resource, issueType, issueField }, database = 'digital-land') {
+    let sql = `
+      SELECT i.field, i.line_number, entry_number, message, issue_type, value
+      FROM issue i
+      LEFT JOIN reporting_historic_endpoints rhe ON rhe.resource = i.resource
+      WHERE REPLACE(rhe.organisation, '-eng', '') = '${organisation}'
+      AND rhe.pipeline = '${dataset}'
+    `
+
+    if (resource) {
+      sql += ` AND i.resource = '${resource}'`
+    }
+
+    if (issueType) {
+      sql += ` AND i.issue_type = '${issueType}'`
+    }
+
+    if (issueField) {
+      sql += ` AND i.field = '${issueField}'`
+    }
+
+    // (no changes below this line)
 
     const result = await datasette.runQuery(sql, database)
 
@@ -471,5 +481,15 @@ export default {
         LIMIT ${pagination.limit}
         OFFSET ${pagination.offset}
       `
+  },
+
+  fetchEntityNumbersFromEntryNumbers ({ entryNumbers, organisationEntity }) {
+    return /* sql */ `
+      select DISTINCT f.entity, fr.entry_number, fr.resource, e.* from fact f
+      LEFT JOIN fact_resource fr ON f.fact = fr.fact
+      LEFT JOIN entity e ON f.entity = e.entity
+      WHERE e.organisation_entity = ${organisationEntity}
+      AND entry_number in (${entryNumbers.join(', ')})
+    `
   }
 }

From 82044cbf4e218c2a64f35311f2a7ad7a6ead702c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 11 Oct 2024 16:17:28 +0100
Subject: [PATCH 034/109] fix test

---
 .../middleware/issueTable.middleware.test.js   | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 1a0d3497..5fba7672 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -132,12 +132,12 @@ describe('issueTable.middleware.js', () => {
       orgInfo: mockedOrg,
       dataset: mockedDataset,
       errorSummary: mockedErrorSummary,
-      entitiesWithIssues: [
+      entities: [
         {
           entry_number: 10,
-          'start-date': 'start-date',
-          reference: 'reference',
-          issues: '{"start-date": "invalid"}'
+          'start-date': { value: 'start-date', issue: { message: 'invalid', value: 'invalid-start-date' } },
+          reference: { value: 'reference' }
+
         }
       ],
       specification: {
@@ -174,13 +174,15 @@ describe('issueTable.middleware.js', () => {
           {
             columns: {
               reference: {
-                html: `<a href="/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/entry/${1}">${req.entitiesWithIssues[0].reference}</a>`
+                error: undefined,
+                html: `<a href="/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/entry/${1}">${req.entities[0].reference.value}</a>`
               },
               'start-date': {
-                value: req.entitiesWithIssues[0]['start-date'],
                 error: {
-                  message: 'invalid'
-                }
+                  message: 'invalid',
+                  value: 'invalid-start-date'
+                },
+                value: req.entities[0]['start-date'].value
               }
             }
           }

From 1efefb458919e97d7cd19a4e6654f37aa7e47bcb Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 11 Oct 2024 16:44:33 +0100
Subject: [PATCH 035/109] pagination options and refactor

---
 src/middleware/common.middleware.js     | 10 ++++++-
 src/middleware/issueTable.middleware.js |  6 +++-
 src/services/performanceDbApi.js        | 37 ++-----------------------
 3 files changed, 17 insertions(+), 36 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 04e1527d..38a9a943 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -211,11 +211,19 @@ export const getEntryNumbersWithIssues = (req, res, next) => {
 }
 
 export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
-  query: ({ req, params }) => performanceDbApi.fetchEntityNumbersFromEntryNumbers({ entryNumbers: req.entryNumbersWithIssues, organisationEntity: req.orgInfo.entity }),
+  query: ({ req, params }) => performanceDbApi.fetchEntitiesFromEntryNumbers({ entryNumbers: req.entryNumbersWithIssues, organisationEntity: req.orgInfo.entity, pagination: req.pagination }),
   result: 'entities',
   dataset: FetchOptions.fromParams
 })
 
+export const getPaginationOptions = (resultsCount) => (req, res, next) => {
+  const { pageNumber } = req.params
+
+  req.pagination = { offset: pageNumber * resultsCount, limit: resultsCount }
+
+  next()
+}
+
 export const extractJsonFieldFromEntities = (req, res, next) => {
   const { entities } = req
 
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index e4086f06..da708be7 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -12,6 +12,7 @@ import {
   fetchSpecification,
   formatErrorSummaryParams,
   getEntryNumbersWithIssues,
+  getPaginationOptions,
   isResourceIdNotInParams,
   logPageError,
   nestEntityFields,
@@ -24,7 +25,7 @@ import {
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
-const paginationPageLength = 50
+const paginationPageLength = 20
 
 export const IssueTableQueryParams = v.object({
   lpa: v.string(),
@@ -42,6 +43,8 @@ const validateIssueTableQueryParams = validateQueryParams.bind({
 export const setDefaultQueryParams = (req, res, next) => {
   if (!req.params.pageNumber) {
     req.params.pageNumber = 1
+  } else {
+    req.params.pageNumber = parseInt(req.params.pageNumber)
   }
   next()
 }
@@ -166,6 +169,7 @@ export default [
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
+  getPaginationOptions(paginationPageLength),
   fetchIssues,
   getEntryNumbersWithIssues,
   fetchEntitiesFromOrganisationAndEntryNumbers,
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 668495b5..73fb6864 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -450,46 +450,15 @@ export default {
     return result.formattedData[0].entity_count
   },
 
-  entitiesAndIssuesQuery ({ resource, pagination, issueType, issueField }) {
-    return /* sql */ `
-      SELECT
-          e.*,
-          fr.entry_number,
-          '{' || GROUP_CONCAT(
-            '"' || i.field || '": "' || i.issue_type || '"',
-            ',' || CHAR(10)
-          ) || '}' AS issues
-        from
-          entity e
-          LEFT JOIN (
-            SELECT
-              DISTINCT fr.entry_number,
-              f.entity
-            FROM
-              fact_resource fr
-              INNER JOIN fact f ON fr.fact = f.fact
-            WHERE
-              fr.resource = '${resource}'
-          ) fr ON fr.entity = e.entity
-          LEFT JOIN issue i ON i.entry_number = fr.entry_number
-        WHERE i.resource = '${resource}'
-        AND i.issue_type = '${issueType}'
-        AND i.field = '${issueField}'
-
-        GROUP BY
-          (e.entity)
-        LIMIT ${pagination.limit}
-        OFFSET ${pagination.offset}
-      `
-  },
-
-  fetchEntityNumbersFromEntryNumbers ({ entryNumbers, organisationEntity }) {
+  fetchEntitiesFromEntryNumbers ({ entryNumbers, organisationEntity, pagination }) {
     return /* sql */ `
       select DISTINCT f.entity, fr.entry_number, fr.resource, e.* from fact f
       LEFT JOIN fact_resource fr ON f.fact = fr.fact
       LEFT JOIN entity e ON f.entity = e.entity
       WHERE e.organisation_entity = ${organisationEntity}
       AND entry_number in (${entryNumbers.join(', ')})
+      LIMIT ${pagination.limit}
+      OFFSET ${pagination.offset}
     `
   }
 }

From faf449dc2e77b3b273dad4c66c52510ad5cfad9e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 11 Oct 2024 18:13:38 +0100
Subject: [PATCH 036/109] calculate entities with issues instead of entries
 with issues

---
 src/middleware/common.middleware.js     | 16 +++++++++++++---
 src/middleware/issueTable.middleware.js |  8 ++++----
 src/services/performanceDbApi.js        |  6 ++++--
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 38a9a943..1496e58c 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -170,7 +170,7 @@ export async function reformatIssuesToBeByEntryNumber (req, res, next) {
 
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { issuesByEntryNumber, entityCount: entityCountRow, issueEntitiesCount } = req
+  const { issuesByEntryNumber, entityCount: entityCountRow, issues } = req
 
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
@@ -180,7 +180,7 @@ export function formatErrorSummaryParams (req, res, next) {
   let issueItems
 
   if (Object.keys(issuesByEntryNumber).length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true)
     issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
       return {
         html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
@@ -189,7 +189,7 @@ export function formatErrorSummaryParams (req, res, next) {
     })
   } else {
     issueItems = [{
-      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true)
     }]
   }
 
@@ -216,6 +216,16 @@ export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
   dataset: FetchOptions.fromParams
 })
 
+export const paginateEntitiesAndPullOutCount = (req, res, next) => {
+  const { entities, pagination } = req
+
+  req.entitiesWithIssuesCount = entities.length
+
+  req.entities = entities.slice(pagination.offset, pagination.offset + pagination.limit)
+
+  next()
+}
+
 export const getPaginationOptions = (resultsCount) => (req, res, next) => {
   const { pageNumber } = req.params
 
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index da708be7..aedaddd2 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -5,7 +5,6 @@ import {
   fetchDatasetInfo,
   fetchEntitiesFromOrganisationAndEntryNumbers,
   fetchEntityCount,
-  fetchIssueEntitiesCount,
   fetchIssues,
   fetchLatestResource,
   fetchOrgInfo,
@@ -16,6 +15,7 @@ import {
   isResourceIdNotInParams,
   logPageError,
   nestEntityFields,
+  paginateEntitiesAndPullOutCount,
   pullOutDatasetSpecification,
   reformatIssuesToBeByEntryNumber,
   replaceUnderscoreWithHyphenForEntities,
@@ -111,10 +111,10 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
  * @returns {void}
  */
 export const createPaginationTemplatePrams = (req, res, next) => {
-  const { issueEntitiesCount } = req
+  const { entitiesWithIssuesCount } = req
   const { pageNumber, lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
-  const totalPages = Math.ceil(issueEntitiesCount / paginationPageLength)
+  const totalPages = Math.floor(entitiesWithIssuesCount / paginationPageLength)
 
   const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
 
@@ -173,11 +173,11 @@ export default [
   fetchIssues,
   getEntryNumbersWithIssues,
   fetchEntitiesFromOrganisationAndEntryNumbers,
+  paginateEntitiesAndPullOutCount,
   extractJsonFieldFromEntities,
   replaceUnderscoreWithHyphenForEntities,
   nestEntityFields,
   addIssuesToEntities,
-  fetchIssueEntitiesCount,
   fetchEntityCount,
   reformatIssuesToBeByEntryNumber,
   formatErrorSummaryParams,
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 73fb6864..b165d24d 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -457,8 +457,10 @@ export default {
       LEFT JOIN entity e ON f.entity = e.entity
       WHERE e.organisation_entity = ${organisationEntity}
       AND entry_number in (${entryNumbers.join(', ')})
-      LIMIT ${pagination.limit}
-      OFFSET ${pagination.offset}
     `
+    // Can't have pagination here as we need to know the count of all the entities with issues anyway, something for the performance db?
+    //   LIMIT ${pagination.limit}
+    //   OFFSET ${pagination.offset}
+    // `
   }
 }

From b1c8240e8759a59a9afb0642b1915b4a9260ed0c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 12:20:45 +0100
Subject: [PATCH 037/109] now get data by getting all resources and working
 from there

---
 src/middleware/common.middleware.js       | 48 +++++++++++++++++++----
 src/middleware/issueDetails.middleware.js |  4 +-
 src/middleware/issueTable.middleware.js   | 33 ++++++++++------
 src/services/performanceDbApi.js          | 47 ++++++++++++++++++++++
 4 files changed, 111 insertions(+), 21 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 1496e58c..800b3a1b 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -50,6 +50,11 @@ export const fetchLatestResource = fetchOne({
   result: 'resource'
 })
 
+export const fetchActiveResourcesForOrganisationAndDataset = fetchMany({
+  query: ({ params }) => performanceDbApi.activeResourcesForOrganisationAndDatasetQuery(params.lpa, params.dataset),
+  result: 'resources'
+})
+
 export const takeResourceIdFromParams = (req) => {
   logger.debug('skipping resource fetch', { type: types.App, params: req.params })
   req.resource = { resource: req.params.resourceId }
@@ -158,8 +163,8 @@ export async function fetchIssues (req, res, next) {
  * @param {*} next
  */
 export async function reformatIssuesToBeByEntryNumber (req, res, next) {
-  const { issues } = req
-  const issuesByEntryNumber = issues.reduce((acc, current) => {
+  const { issuesWithReferences } = req
+  const issuesByEntryNumber = issuesWithReferences.reduce((acc, current) => {
     acc[current.entry_number] = acc[current.entry_number] || []
     acc[current.entry_number].push(current)
     return acc
@@ -170,7 +175,7 @@ export async function reformatIssuesToBeByEntryNumber (req, res, next) {
 
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { issuesByEntryNumber, entityCount: entityCountRow, issues } = req
+  const { issuesByEntryNumber, entityCount: entityCountRow, issuesWithReferences } = req
 
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
@@ -180,7 +185,7 @@ export function formatErrorSummaryParams (req, res, next) {
   let issueItems
 
   if (Object.keys(issuesByEntryNumber).length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true)
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true)
     issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
       return {
         html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
@@ -189,7 +194,7 @@ export function formatErrorSummaryParams (req, res, next) {
     })
   } else {
     issueItems = [{
-      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true)
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true)
     }]
   }
 
@@ -218,10 +223,13 @@ export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
 
 export const paginateEntitiesAndPullOutCount = (req, res, next) => {
   const { entities, pagination } = req
+  const { pageNumber } = req.params
+
+  const paginationIndex = pageNumber - 1
 
   req.entitiesWithIssuesCount = entities.length
 
-  req.entities = entities.slice(pagination.offset, pagination.offset + pagination.limit)
+  req.entities = entities.slice(pagination.offset * paginationIndex, pagination.offset * paginationIndex + pagination.limit)
 
   next()
 }
@@ -278,10 +286,10 @@ export const nestEntityFields = (req, res, next) => {
 }
 
 export const addIssuesToEntities = (req, res, next) => {
-  const { entities, issues } = req
+  const { entities, issuesWithReferences } = req
 
   req.entitiesWithIssues = entities.map(entity => {
-    const entityIssues = issues.filter(issue => issue.entryNumber === entity.entryNumber)
+    const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
       entity[issue.field].value = issue.value
@@ -293,3 +301,27 @@ export const addIssuesToEntities = (req, res, next) => {
 
   next()
 }
+
+export const hasEntities = (req, res, next) => req.entities !== undefined
+
+export const fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield = fetchMany({
+  query: ({ req, params }) => performanceDbApi.issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery({
+    resources: req.resources.map(resourceObj => resourceObj.resource),
+    dataset: params.dataset,
+    issueType: params.issue_type,
+    issueField: params.issue_field
+  }),
+  result: 'issuesWithReferences',
+  dataset: FetchOptions.fromParams
+})
+
+export const fetchEntitiesFromIssuesWithReferences = fetchMany({
+  query: ({ req }) => performanceDbApi.fetchEntitiesFromReferencesAndOrganisationEntity({
+    references: req.issuesWithReferences.map(issueWithReference => issueWithReference.reference),
+    organisationEntity: req.orgInfo.entity
+  }),
+  result: 'entities',
+  dataset: FetchOptions.fromParams
+})
+
+// export const getReferencesOfIssueEntities
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 9904f802..0ac54fd9 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,5 +1,5 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntitiesFromOrganisationAndEntryNumbers, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, getEntryNumbersWithIssues, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
@@ -245,6 +245,8 @@ export default [
   fetchDatasetInfo,
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchIssues,
+  getEntryNumbersWithIssues,
+  fetchEntitiesFromOrganisationAndEntryNumbers,
   reformatIssuesToBeByEntryNumber,
   getEntryNumberFromPageNumber,
   parallel([
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index aedaddd2..b69ec516 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -3,15 +3,13 @@ import {
   addIssuesToEntities,
   extractJsonFieldFromEntities,
   fetchDatasetInfo,
-  fetchEntitiesFromOrganisationAndEntryNumbers,
   fetchEntityCount,
-  fetchIssues,
   fetchLatestResource,
   fetchOrgInfo,
   fetchSpecification,
   formatErrorSummaryParams,
-  getEntryNumbersWithIssues,
   getPaginationOptions,
+  hasEntities,
   isResourceIdNotInParams,
   logPageError,
   nestEntityFields,
@@ -20,7 +18,10 @@ import {
   reformatIssuesToBeByEntryNumber,
   replaceUnderscoreWithHyphenForEntities,
   takeResourceIdFromParams,
-  validateQueryParams
+  validateQueryParams,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
+  fetchEntitiesFromIssuesWithReferences
 } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -159,6 +160,14 @@ export const getIssueTable = renderTemplate({
   handlerName: 'getIssueTable'
 })
 
+// const getEntitiesWithIssuesMiddlewareChain = [
+//   fetchResourcesFromOrganisationAndDataset,
+//   fetchIssuesFromResourcesDatasetIssuetypefield,
+//   // have a list of issues with resource, and entry number
+//   getReferencesOfIssueEntities,
+//   getEntitiesFromRefernces
+// ]
+
 export default [
   validateIssueTableQueryParams,
   setDefaultQueryParams,
@@ -170,14 +179,14 @@ export default [
   fetchSpecification,
   pullOutDatasetSpecification,
   getPaginationOptions(paginationPageLength),
-  fetchIssues,
-  getEntryNumbersWithIssues,
-  fetchEntitiesFromOrganisationAndEntryNumbers,
-  paginateEntitiesAndPullOutCount,
-  extractJsonFieldFromEntities,
-  replaceUnderscoreWithHyphenForEntities,
-  nestEntityFields,
-  addIssuesToEntities,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
+  fetchEntitiesFromIssuesWithReferences,
+  fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
+  fetchIf(hasEntities, extractJsonFieldFromEntities),
+  fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities),
+  fetchIf(hasEntities, nestEntityFields),
+  fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
   reformatIssuesToBeByEntryNumber,
   formatErrorSummaryParams,
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index b165d24d..b68edba9 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -263,6 +263,20 @@ export default {
     AND rle.pipeline = '${dataset}'`
   },
 
+  activeResourcesForOrganisationAndDatasetQuery: (lpa, dataset) => {
+    return /* sql */`
+      select
+        rhe.endpoint, rhe.endpoint_url, rhe.resource, rhe.status
+      from
+        reporting_historic_endpoints rhe
+      LEFT JOIN resource_organisation ro ON rhe.resource = ro.resource
+      LEFT JOIN organisation o ON REPLACE(ro.organisation, '-eng', '') = o.organisation
+      WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}'
+      AND pipeline = '${dataset}'
+      AND rhe.endpoint_end_date == ''
+    `
+  },
+
   /**
      * Retrieves the latest resource information for a given LPA and dataset.
      *
@@ -364,6 +378,7 @@ export default {
      * @param {string} [database="digital-land"] - Database to query (defaults to "digital-land")
      * @returns {Promise<Object>} - Promise resolving to an object with formatted data
      */
+
   async getIssues ({ organisation, dataset, resource, issueType, issueField }, database = 'digital-land') {
     let sql = `
       SELECT i.field, i.line_number, entry_number, message, issue_type, value
@@ -392,6 +407,20 @@ export default {
     return result.formattedData
   },
 
+  issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery ({ resources, dataset, issueType, issueField }) {
+    return /* sql */ `
+      SELECT DISTINCT i.message, i.value, i.field, i.issue_type, i.entry_number, f.value as reference
+      FROM issue i
+      LEFT JOIN fact_resource fr ON i.entry_number = fr.entry_number AND i.resource = fr.resource
+      LEFT JOIN fact f ON fr.fact = f.fact
+      WHERE i.resource  in ('${resources.join("', '")}')
+      AND dataset  = '${dataset}'
+      AND issue_type  = '${issueType}'
+      AND i.field  = '${issueField}'
+      AND f.field = 'reference'
+    `
+  },
+
   /**
      *
      * @param {*} resourceId
@@ -462,5 +491,23 @@ export default {
     //   LIMIT ${pagination.limit}
     //   OFFSET ${pagination.offset}
     // `
+  },
+
+  fetchEntityFromEntryNumber ({ entryNumber, organisationEntity }) {
+    return /* sql */ `
+      select DISTINCT f.entity, fr.entry_number, fr.resource, e.* from fact f
+      LEFT JOIN fact_resource fr ON f.fact = fr.fact
+      LEFT JOIN entity e ON f.entity = e.entity
+      AND e.organisation_entity = ${organisationEntity}
+      AND entry_number = ${entryNumber}
+    `
+  },
+
+  fetchEntitiesFromReferencesAndOrganisationEntity ({ references, organisationEntity }) {
+    return /* sql */ `
+      select * from entity 
+      WHERE reference in ('${references.join("', '")}')
+      AND organisation_entity = ${organisationEntity}
+    `
   }
 }

From 310e9033a48cf95b2d6499214c1a403b5b1db22c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 13:39:34 +0100
Subject: [PATCH 038/109] have issue details show rows based on spec, and error
 summary based on entities

---
 src/middleware/common.middleware.js       |  12 +-
 src/middleware/issueDetails.middleware.js | 171 ++++++----------------
 src/middleware/issueTable.middleware.js   |  18 +--
 src/services/performanceDbApi.js          |   2 +-
 4 files changed, 58 insertions(+), 145 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 800b3a1b..bf682e0b 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -175,7 +175,7 @@ export async function reformatIssuesToBeByEntryNumber (req, res, next) {
 
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { issuesByEntryNumber, entityCount: entityCountRow, issuesWithReferences } = req
+  const { entityCount: entityCountRow, issuesWithReferences, entities } = req
 
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
@@ -184,12 +184,12 @@ export function formatErrorSummaryParams (req, res, next) {
   let errorHeading
   let issueItems
 
-  if (Object.keys(issuesByEntryNumber).length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true)
-    issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
+  if (entities.length < entityCount) {
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: entities.length, entityCount, field: issueField }, true)
+    issueItems = entities.map((entity, index) => {
       return {
-        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
-        href: `${BaseSubpath}${entryNumber}`
+        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in entity ${entity?.reference?.value || entity?.reference}`,
+        href: `${BaseSubpath}${index + 1}`
       }
     })
   } else {
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 0ac54fd9..034533d8 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,6 +1,26 @@
-import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchEntitiesFromOrganisationAndEntryNumbers, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, getEntryNumbersWithIssues, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
-import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
+import {
+  addIssuesToEntities,
+  extractJsonFieldFromEntities,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchDatasetInfo,
+  fetchEntitiesFromIssuesWithReferences,
+  fetchEntityCount,
+  fetchIssueEntitiesCount,
+  fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
+  fetchLatestResource,
+  fetchOrgInfo,
+  fetchSpecification,
+  formatErrorSummaryParams,
+  hasEntities,
+  isResourceIdNotInParams,
+  logPageError,
+  nestEntityFields,
+  pullOutDatasetSpecification,
+  replaceUnderscoreWithHyphenForEntities,
+  takeResourceIdFromParams,
+  validateQueryParams
+} from './common.middleware.js'
+import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 
@@ -17,30 +37,6 @@ const validateIssueDetailsQueryParams = validateQueryParams.bind({
   schema: IssueDetailsQueryParams
 })
 
-/**
- *
- * Middleware. Updates `req` with `entryData`
- *
- * Requires `dataset` and `entryNumber`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- *
- */
-async function fetchEntry (req, res, next) {
-  const { dataset: datasetId } = req.params
-  const { entryNumber } = req
-
-  req.entryData = await performanceDbApi.getEntry(
-    req.resource.resource,
-    entryNumber,
-    datasetId
-  )
-
-  next()
-}
-
 /**
  *
  * @param {string} errorMessage
@@ -71,73 +67,6 @@ const getIssueField = (text, html, classes) => {
   }
 }
 
-/**
- *
- * @param {*} issueType
- * @param {*} issuesByEntryNumber
- * @param {*} row
- * @returns {{key: {text: string}, value: { html: string}, classes: string}}
- */
-const processEntryRow = (issueType, issuesByEntryNumber = {}, row) => {
-  const { entry_number: entryNumber } = row
-  console.assert(entryNumber, 'precessEntryRow(): entry_number not in row')
-
-  let hasError = false
-  let issueIndex
-  if (issuesByEntryNumber[entryNumber]) {
-    issueIndex = issuesByEntryNumber[entryNumber].findIndex(
-      (issue) => issue.field === row.field
-    )
-    hasError = issueIndex >= 0
-  }
-
-  let valueHtml = ''
-  let classes = ''
-  if (hasError) {
-    const message =
-        issuesByEntryNumber[entryNumber][issueIndex].message || issueType
-    valueHtml += issueErrorMessageHtml(message, null)
-    classes += 'dl-summary-card-list__row--error'
-  }
-  valueHtml += row.value
-
-  return getIssueField(row.field, valueHtml, classes)
-}
-
-/**
- * Middleware. Extracts the entry number from the page number in the request.
- *
- * @param {object} req - The request object
- * @param {object} res - The response object
- * @param {function} next - The next middleware function
- *
- * @throws {Error} If the page number cannot be parsed as an integer
- * @throws {Error} If the entry number is not found (404)
- */
-export function getEntryNumberFromPageNumber (req, res, next) {
-  const { issuesByEntryNumber } = req
-  const { pageNumber } = req.params
-
-  const pageNumberAsInt = parseInt(pageNumber)
-  if (isNaN(pageNumberAsInt)) {
-    const error = new Error('page number could not be parsed as an integer')
-    error.status = 400
-    return next(error)
-  }
-
-  const issuesByEntryNumberIndex = pageNumberAsInt - 1
-  const pageNumberToEntryNumberMap = Object.keys(issuesByEntryNumber)
-
-  if (issuesByEntryNumberIndex < 0 || issuesByEntryNumberIndex >= pageNumberToEntryNumberMap.length) {
-    const error = new Error('not found')
-    error.status = 404
-    return next(error)
-  }
-
-  req.entryNumber = pageNumberToEntryNumberMap[issuesByEntryNumberIndex]
-  next()
-}
-
 /**
  * Middleware. Prepares template parameters for the issue details page.
  *
@@ -151,47 +80,42 @@ export function getEntryNumberFromPageNumber (req, res, next) {
  * from the request, and organizes it into a template parameters object that can be used to render the page.
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entryData, issueEntitiesCount, issuesByEntryNumber, errorSummary, entryNumber } = req
+  const { entities, issueEntitiesCount, errorSummary, specification } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField, pageNumber: pageNumberString } = req.params
   const pageNumber = parseInt(pageNumberString)
 
   const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
 
-  const fields = entryData.map((row) => processEntryRow(issueType, issuesByEntryNumber, row))
-  const entityIssues = issuesByEntryNumber[entryNumber] || []
-  for (const issue of entityIssues) {
-    if (!fields.find((field) => field.key.text === issue.field)) {
-      const errorMessage = issue.message || issueType
-      // TODO: pull the html out of here and into the template
-      const valueHtml = issueErrorMessageHtml(errorMessage, issue.value)
-      const classes = 'dl-summary-card-list__row--error'
+  const entity = entities[pageNumber - 1]
 
-      fields.push(getIssueField(issue.field, valueHtml, classes))
+  const fields = specification.fields.map(({ field }) => {
+    let valueHtml = ''
+    let classes = ''
+    if (entity[field].issue) {
+      valueHtml += issueErrorMessageHtml(entity[field].issue.message, null)
+      classes += 'dl-summary-card-list__row--error'
     }
-  }
+    valueHtml += entity[field].value || ''
+    return getIssueField(field, valueHtml, classes)
+  })
 
-  const geometries = entryData
-    .filter((row) => row.field === 'geometry')
-    .map((row) => row.value)
   const entry = {
-    title: `entry: ${entryNumber}`,
+    title: `entry: ${entity.reference.value}`,
     fields,
-    geometries
+    geometries: [entity.geometry.value]
   }
 
   const paginationObj = {
     items: []
   }
 
-  const entryNumbers = Object.keys(issuesByEntryNumber)
-
   if (pageNumber > 1) {
     paginationObj.previous = {
       href: `${BaseSubpath}${pageNumber - 1}`
     }
   }
 
-  if (pageNumber < entryNumbers.length) {
+  if (pageNumber < entities.length) {
     paginationObj.next = {
       href: `${BaseSubpath}${pageNumber + 1}`
     }
@@ -244,16 +168,17 @@ export default [
   fetchOrgInfo,
   fetchDatasetInfo,
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
-  fetchIssues,
-  getEntryNumbersWithIssues,
-  fetchEntitiesFromOrganisationAndEntryNumbers,
-  reformatIssuesToBeByEntryNumber,
-  getEntryNumberFromPageNumber,
-  parallel([
-    fetchEntry,
-    fetchEntityCount,
-    fetchIssueEntitiesCount
-  ]),
+  fetchSpecification,
+  pullOutDatasetSpecification,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
+  fetchEntitiesFromIssuesWithReferences,
+  fetchIf(hasEntities, extractJsonFieldFromEntities),
+  fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities),
+  fetchIf(hasEntities, nestEntityFields),
+  fetchIf(hasEntities, addIssuesToEntities),
+  fetchEntityCount,
+  fetchIssueEntitiesCount,
   formatErrorSummaryParams,
   prepareIssueDetailsTemplateParams,
   getIssueDetails,
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index b69ec516..937c5ac3 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -15,7 +15,6 @@ import {
   nestEntityFields,
   paginateEntitiesAndPullOutCount,
   pullOutDatasetSpecification,
-  reformatIssuesToBeByEntryNumber,
   replaceUnderscoreWithHyphenForEntities,
   takeResourceIdFromParams,
   validateQueryParams,
@@ -23,7 +22,7 @@ import {
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences
 } from './common.middleware.js'
-import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
+import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
 const paginationPageLength = 20
@@ -160,21 +159,11 @@ export const getIssueTable = renderTemplate({
   handlerName: 'getIssueTable'
 })
 
-// const getEntitiesWithIssuesMiddlewareChain = [
-//   fetchResourcesFromOrganisationAndDataset,
-//   fetchIssuesFromResourcesDatasetIssuetypefield,
-//   // have a list of issues with resource, and entry number
-//   getReferencesOfIssueEntities,
-//   getEntitiesFromRefernces
-// ]
-
 export default [
   validateIssueTableQueryParams,
   setDefaultQueryParams,
-  parallel([
-    fetchOrgInfo,
-    fetchDatasetInfo
-  ]),
+  fetchOrgInfo,
+  fetchDatasetInfo,
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
@@ -188,7 +177,6 @@ export default [
   fetchIf(hasEntities, nestEntityFields),
   fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
-  reformatIssuesToBeByEntryNumber,
   formatErrorSummaryParams,
   createPaginationTemplatePrams,
   prepareIssueTableTemplateParams,
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index b68edba9..1359df0e 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -409,7 +409,7 @@ export default {
 
   issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery ({ resources, dataset, issueType, issueField }) {
     return /* sql */ `
-      SELECT DISTINCT i.message, i.value, i.field, i.issue_type, i.entry_number, f.value as reference
+      SELECT DISTINCT i.message, i.value, i.field, i.issue_type, i.entry_number, i.resource, f.value as reference
       FROM issue i
       LEFT JOIN fact_resource fr ON i.entry_number = fr.entry_number AND i.resource = fr.resource
       LEFT JOIN fact f ON fr.fact = f.fact

From 8c7f2b9a904ceb76d00dfd3853b76258fe18d4e4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 13:42:15 +0100
Subject: [PATCH 039/109] make summary before pagination

---
 src/middleware/issueTable.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 937c5ac3..7921d133 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -172,12 +172,12 @@ export default [
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
   fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
+  formatErrorSummaryParams,
   fetchIf(hasEntities, extractJsonFieldFromEntities),
   fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities),
   fetchIf(hasEntities, nestEntityFields),
   fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
-  formatErrorSummaryParams,
   createPaginationTemplatePrams,
   prepareIssueTableTemplateParams,
   getIssueTable,

From 4a79ca87adbbcda64d75839f841f6f0becb98e95 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 14:04:21 +0100
Subject: [PATCH 040/109] change issue details to get data based on entity but
 also add in issues to summary that dont have an entity

---
 src/middleware/common.middleware.js          | 20 ++++++++++++++++++--
 src/middleware/datasetTaskList.middleware.js |  9 +++------
 src/middleware/issueDetails.middleware.js    |  2 ++
 src/middleware/issueTable.middleware.js      |  4 +++-
 src/services/performanceDbApi.js             | 14 ++++++++++++++
 src/views/organisations/issueTable.html      |  4 +++-
 6 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index bf682e0b..cbcef4e6 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -175,7 +175,7 @@ export async function reformatIssuesToBeByEntryNumber (req, res, next) {
 
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { entityCount: entityCountRow, issuesWithReferences, entities } = req
+  const { entityCount: entityCountRow, issuesWithReferences, issuesWithoutReferences, entities } = req
 
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
@@ -184,7 +184,12 @@ export function formatErrorSummaryParams (req, res, next) {
   let errorHeading
   let issueItems
 
-  if (entities.length < entityCount) {
+  // if the entities length is 0, this means the entry never became an entity, so we shouldn't show the table or links to the entity details page
+  if (entities.length === 0) {
+    issueItems = [{
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithoutReferences.length, entityCount, field: issueField }, true)
+    }]
+  } else if (entities.length < entityCount) {
     errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: entities.length, entityCount, field: issueField }, true)
     issueItems = entities.map((entity, index) => {
       return {
@@ -324,4 +329,15 @@ export const fetchEntitiesFromIssuesWithReferences = fetchMany({
   dataset: FetchOptions.fromParams
 })
 
+export const fetchIssuesWithoutReferences = fetchMany({
+  query: ({ req, params }) => performanceDbApi.fetchIssuesWithoutReferences({
+    resources: req.resources.map(resourceObj => resourceObj.resource),
+    dataset: params.dataset,
+    issueType: params.issue_type,
+    issueField: params.issue_field
+  }),
+  result: 'issuesWithoutReferences',
+  dataset: FetchOptions.fromParams
+})
+
 // export const getReferencesOfIssueEntities
diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index 8a5d9f23..98ac91a2 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -1,5 +1,5 @@
 import { fetchDatasetInfo, isResourceAccessible, isResourceNotAccessible, fetchLatestResource, fetchEntityCount, logPageError, fetchLpaDatasetIssues } from './common.middleware.js'
-import { parallel, fetchOne, fetchIf, onlyIf, renderTemplate } from './middleware.builders.js'
+import { fetchOne, fetchIf, onlyIf, renderTemplate } from './middleware.builders.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { statusToTagClass } from '../filters/filters.js'
 
@@ -116,13 +116,10 @@ export default [
   fetchOrgInfoWithStatGeo,
   fetchDatasetInfo,
   fetchIf(isResourceAccessible, fetchLatestResource),
-  parallel([
-    fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
-    fetchIf(isResourceAccessible, fetchEntityCount)
-  ]),
+  fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
+  fetchIf(isResourceAccessible, fetchEntityCount),
   onlyIf(isResourceAccessible, prepareDatasetTaskListTemplateParams),
   onlyIf(isResourceAccessible, getDatasetTaskList),
-
   onlyIf(isResourceNotAccessible, prepareDatasetTaskListErrorTemplateParams),
   onlyIf(isResourceNotAccessible, getDatasetTaskListError),
   logPageError
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 034533d8..6fc92980 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -6,6 +6,7 @@ import {
   fetchEntitiesFromIssuesWithReferences,
   fetchEntityCount,
   fetchIssueEntitiesCount,
+  fetchIssuesWithoutReferences,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchLatestResource,
   fetchOrgInfo,
@@ -170,6 +171,7 @@ export default [
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
+  fetchIssuesWithoutReferences,
   fetchActiveResourcesForOrganisationAndDataset,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 7921d133..6e5dc20b 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -20,7 +20,8 @@ import {
   validateQueryParams,
   fetchActiveResourcesForOrganisationAndDataset,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
-  fetchEntitiesFromIssuesWithReferences
+  fetchEntitiesFromIssuesWithReferences,
+  fetchIssuesWithoutReferences
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -171,6 +172,7 @@ export default [
   fetchActiveResourcesForOrganisationAndDataset,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
+  fetchIssuesWithoutReferences,
   fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
   formatErrorSummaryParams,
   fetchIf(hasEntities, extractJsonFieldFromEntities),
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 1359df0e..2f5b4338 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -421,6 +421,20 @@ export default {
     `
   },
 
+  fetchIssuesWithoutReferences ({ resources, dataset, issueType, issueField }) {
+    return /* sql */ `
+      SELECT DISTINCT i.message, i.value, i.field, i.issue_type, i.entry_number, i.resource, f.value as reference
+      FROM issue i
+      LEFT JOIN fact_resource fr ON i.entry_number = fr.entry_number AND i.resource = fr.resource
+      LEFT JOIN fact f ON fr.fact = f.fact
+      WHERE i.resource  in ('${resources.join("', '")}')
+      AND dataset  = '${dataset}'
+      AND issue_type  = '${issueType}'
+      AND i.field  = '${issueField}'
+      AND f.field is NULL
+    `
+  },
+
   /**
      *
      * @param {*} resourceId
diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
index adee5cb5..7d05c92b 100644
--- a/src/views/organisations/issueTable.html
+++ b/src/views/organisations/issueTable.html
@@ -74,7 +74,9 @@
 
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-full">
-    {{ table(tableParams) }}
+    {% if tableParams.rows.length > 0 %}
+      {{ table(tableParams) }}
+    {% endif %}
   </div>
 </div>
 

From 041a4d49a8f2efe2f9cc28a926637c9f9733d81b Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 14:58:29 +0100
Subject: [PATCH 041/109] have numbers on the dataset task list taken from all
 resources

---
 src/middleware/common.middleware.js          | 61 +++++++++---------
 src/middleware/datasetTaskList.middleware.js | 31 +++++----
 src/services/performanceDbApi.js             | 66 +++++++++++---------
 3 files changed, 79 insertions(+), 79 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index cbcef4e6..ca7f1741 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -130,28 +130,6 @@ export async function fetchIssueEntitiesCount (req, res, next) {
   next()
 }
 
-/**
- * Fetches issues from the performance database and updates the request object with the result.
- *
- * This middleware requires the `resourceId` to be present in the request params or request object.
- *
- * @param {object} req - The HTTP request object
- * @param {object} res - The HTTP response object
- * @param {function} next - The next middleware function in the stack
- *
- * @throws {Error} If `resourceId` is missing from the request
- */
-export async function fetchIssues (req, res, next) {
-  const { dataset, issue_type: issueType, issue_field: issueField, lpa } = req.params
-
-  try {
-    req.issues = await performanceDbApi.getIssues({ organisation: lpa, dataset, issueType, issueField })
-    next()
-  } catch (error) {
-    next(error)
-  }
-}
-
 /**
  *
  * Middleware. Updates `req` with `issues`.
@@ -309,23 +287,44 @@ export const addIssuesToEntities = (req, res, next) => {
 
 export const hasEntities = (req, res, next) => req.entities !== undefined
 
-export const fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield = fetchMany({
-  query: ({ req, params }) => performanceDbApi.issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery({
+export const fetchEntitiesFromIssuesWithReferences = fetchMany({
+  query: ({ req }) => performanceDbApi.fetchEntitiesFromReferencesAndOrganisationEntity({
+    references: req.issuesWithReferences.map(issueWithReference => issueWithReference.reference),
+    organisationEntity: req.orgInfo.entity
+  }),
+  result: 'entities',
+  dataset: FetchOptions.fromParams
+})
+
+export const fetchIssuesWithCounts = fetchMany({
+  query: ({ req, params }) => performanceDbApi.issuesWithCountsQuery({
+    resources: req.resources.map(resourceObj => resourceObj.resource),
+    dataset: params.dataset,
+    issueType: params.issue_type,
+    issueField: params.issue_field,
+    statusList: ['Error', 'Needs fixing', 'Warning']
+  }),
+  result: 'issuesWithCounts'
+})
+
+export const fetchIssues = fetchMany({
+  query: ({ req, params }) => performanceDbApi.issuesQuery({
     resources: req.resources.map(resourceObj => resourceObj.resource),
     dataset: params.dataset,
     issueType: params.issue_type,
     issueField: params.issue_field
   }),
-  result: 'issuesWithReferences',
-  dataset: FetchOptions.fromParams
+  result: 'issues'
 })
 
-export const fetchEntitiesFromIssuesWithReferences = fetchMany({
-  query: ({ req }) => performanceDbApi.fetchEntitiesFromReferencesAndOrganisationEntity({
-    references: req.issuesWithReferences.map(issueWithReference => issueWithReference.reference),
-    organisationEntity: req.orgInfo.entity
+export const fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield = fetchMany({
+  query: ({ req, params }) => performanceDbApi.issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery({
+    resources: req.resources.map(resourceObj => resourceObj.resource),
+    dataset: params.dataset,
+    issueType: params.issue_type,
+    issueField: params.issue_field
   }),
-  result: 'entities',
+  result: 'issuesWithReferences',
   dataset: FetchOptions.fromParams
 })
 
diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index 98ac91a2..2b8245d8 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -1,5 +1,5 @@
-import { fetchDatasetInfo, isResourceAccessible, isResourceNotAccessible, fetchLatestResource, fetchEntityCount, logPageError, fetchLpaDatasetIssues } from './common.middleware.js'
-import { fetchOne, fetchIf, onlyIf, renderTemplate } from './middleware.builders.js'
+import { fetchDatasetInfo, fetchEntityCount, logPageError, fetchActiveResourcesForOrganisationAndDataset, fetchIssuesWithCounts } from './common.middleware.js'
+import { fetchOne, renderTemplate } from './middleware.builders.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { statusToTagClass } from '../filters/filters.js'
 
@@ -42,13 +42,12 @@ function getStatusTag (status) {
  * @returns { { templateParams: object }}
  */
 export const prepareDatasetTaskListTemplateParams = (req, res, next) => {
-  const { issues, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req
+  const { issuesWithCounts, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
   const { lpa, dataset: datasetId } = params
-  console.assert(req.resourceStatus.resource === req.resource.resource, 'mismatch between resourceStatus and resource data')
   console.assert(typeof entityCount === 'number', 'entityCount should be a number')
 
-  const taskList = issues.map((issue) => {
+  const taskList = issuesWithCounts.map((issue) => {
     return {
       title: {
         text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field })
@@ -105,22 +104,20 @@ export const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
   next()
 }
 
-const getDatasetTaskListError = renderTemplate({
-  templateParams: (req) => req.templateParams,
-  template: 'organisations/http-error.html',
-  handlerName: 'getDatasetTaskListError'
-})
+// const getDatasetTaskListError = renderTemplate({
+//   templateParams: (req) => req.templateParams,
+//   template: 'organisations/http-error.html',
+//   handlerName: 'getDatasetTaskListError'
+// })
 
 export default [
   fetchResourceStatus,
   fetchOrgInfoWithStatGeo,
   fetchDatasetInfo,
-  fetchIf(isResourceAccessible, fetchLatestResource),
-  fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
-  fetchIf(isResourceAccessible, fetchEntityCount),
-  onlyIf(isResourceAccessible, prepareDatasetTaskListTemplateParams),
-  onlyIf(isResourceAccessible, getDatasetTaskList),
-  onlyIf(isResourceNotAccessible, prepareDatasetTaskListErrorTemplateParams),
-  onlyIf(isResourceNotAccessible, getDatasetTaskListError),
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithCounts,
+  fetchEntityCount,
+  prepareDatasetTaskListTemplateParams,
+  getDatasetTaskList,
   logPageError
 ]
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 2f5b4338..9402f97e 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -368,43 +368,47 @@ export default {
     return result.formattedData[0].count
   },
 
-  /**
-     * Retrieves issues from the performance database.
-     *
-     * @param {Object} params - Object with parameters for the query
-     * @param {string} params.resource - Resource to filter issues by
-     * @param {string} params.issueType - Issue type to filter by
-     * @param {string} params.issueField - Field to filter by
-     * @param {string} [database="digital-land"] - Database to query (defaults to "digital-land")
-     * @returns {Promise<Object>} - Promise resolving to an object with formatted data
-     */
-
-  async getIssues ({ organisation, dataset, resource, issueType, issueField }, database = 'digital-land') {
-    let sql = `
-      SELECT i.field, i.line_number, entry_number, message, issue_type, value
-      FROM issue i
-      LEFT JOIN reporting_historic_endpoints rhe ON rhe.resource = i.resource
-      WHERE REPLACE(rhe.organisation, '-eng', '') = '${organisation}'
-      AND rhe.pipeline = '${dataset}'
+  issuesQuery ({ resources, dataset, issueType, issueField }) {
+    let query = `
+      SELECT DISTINCT message, value, field, issue_type, entry_number, resource
+      FROM issue
+      WHERE resource  in ('${resources.join("', '")}')
     `
+    if (dataset) query += ` AND dataset  = '${dataset}'`
+    if (issueType) query += ` AND issue_type  = '${issueType}'`
+    if (issueField) query += ` AND field  = '${issueField}'`
 
-    if (resource) {
-      sql += ` AND i.resource = '${resource}'`
-    }
-
-    if (issueType) {
-      sql += ` AND i.issue_type = '${issueType}'`
-    }
+    return query
+  },
 
-    if (issueField) {
-      sql += ` AND i.field = '${issueField}'`
-    }
+  issuesWithCountsQuery  ({ resources, dataset, issueType, issueField, statusList }) {
+    let query = `
+      SELECT 
+        field, 
+        i.issue_type, 
+        CASE
+        WHEN COUNT(
+          CASE
+            WHEN it.severity == 'error' THEN 1
+            ELSE null
+          END
+        ) > 0 THEN 'Needs fixing'
+        ELSE 'Live'
+        END AS status, 
+        count(line_number) as num_issues
+      FROM issue i
+      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+      WHERE resource  in ('${resources.join("', '")}')
+    `
+    if (dataset) query += ` AND dataset  = '${dataset}'`
+    if (issueType) query += ` AND i.issue_type  = '${issueType}'`
+    if (issueField) query += ` AND i.field  = '${issueField}'`
 
-    // (no changes below this line)
+    query += ' GROUP BY i.field, i.issue_type'
 
-    const result = await datasette.runQuery(sql, database)
+    if (statusList) query += ` HAVING status in ('${statusList.join("', '")}')`
 
-    return result.formattedData
+    return query
   },
 
   issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery ({ resources, dataset, issueType, issueField }) {

From bbf92ae32214420d8ba7613c0f5a6ab4934df74d Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 16:04:43 +0100
Subject: [PATCH 042/109] datasetTaskList: look at all resources

---
 src/middleware/common.middleware.js          |  2 +-
 src/middleware/datasetOverview.middleware.js | 13 ++++++-------
 src/middleware/datasetTaskList.middleware.js |  1 +
 src/services/performanceDbApi.js             |  4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index ca7f1741..743eaa1c 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -94,7 +94,7 @@ export function validateQueryParams (req, res, next) {
 }
 
 export const fetchLpaDatasetIssues = fetchMany({
-  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
+  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resources, params.dataset),
   result: 'issues'
 })
 
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index 3a2ed4e6..ec430096 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,5 +1,5 @@
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
-import { fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
+import { fetchActiveResourcesForOrganisationAndDataset, fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchIf, fetchMany, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
 
 const fetchColumnSummary = fetchMany({
@@ -110,11 +110,10 @@ const getDatasetOverview = renderTemplate(
 export default [
   fetchOrgInfo,
   fetchDatasetInfo,
-  parallel([
-    fetchColumnSummary,
-    fetchResourceStatus,
-    fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams)
-  ]),
+  fetchColumnSummary,
+  fetchResourceStatus,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
   fetchSpecification,
   pullOutDatasetSpecification,
diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index 2b8245d8..beb08141 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -104,6 +104,7 @@ export const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
   next()
 }
 
+// ToDo: do we need to add this back in to the middleware chain?
 // const getDatasetTaskListError = renderTemplate({
 //   templateParams: (req) => req.templateParams,
 //   template: 'organisations/http-error.html',
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 9402f97e..eccc14a5 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -45,7 +45,7 @@ function getAllRowsMessages () {
 
 // ===========================================
 
-const datasetIssuesQuery = (resource, datasetId) => {
+const datasetIssuesQuery = (resources, datasetId) => {
   return /* sql */ `
     SELECT
     i.field,
@@ -69,7 +69,7 @@ const datasetIssuesQuery = (resource, datasetId) => {
   LEFT JOIN
     issue_type it ON i.issue_type = it.issue_type
   WHERE
-      i.resource = '${resource}'
+      i.resource in ('${resources.join("', '")}')
       AND i.dataset = '${datasetId}'
       AND (it.severity == 'error')
   GROUP BY i.issue_type, i.field

From 99c7dfa32bf0687190cfec7e718b8cc0278c653b Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 14 Oct 2024 16:11:33 +0100
Subject: [PATCH 043/109] remove unused functions from common.middleware

---
 src/middleware/common.middleware.js          | 55 +-------------------
 src/middleware/datasetOverview.middleware.js |  8 ++-
 2 files changed, 8 insertions(+), 55 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 743eaa1c..53f1f9b1 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -93,11 +93,6 @@ export function validateQueryParams (req, res, next) {
   }
 }
 
-export const fetchLpaDatasetIssues = fetchMany({
-  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resources, params.dataset),
-  result: 'issues'
-})
-
 export const fetchSpecification = fetchOne({
   query: ({ req }) => `select * from specification WHERE specification = '${req.dataset.collection}'`,
   result: 'specification'
@@ -130,27 +125,6 @@ export async function fetchIssueEntitiesCount (req, res, next) {
   next()
 }
 
-/**
- *
- * Middleware. Updates `req` with `issues`.
- *
- * Requires `issues` in request.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-export async function reformatIssuesToBeByEntryNumber (req, res, next) {
-  const { issuesWithReferences } = req
-  const issuesByEntryNumber = issuesWithReferences.reduce((acc, current) => {
-    acc[current.entry_number] = acc[current.entry_number] || []
-    acc[current.entry_number].push(current)
-    return acc
-  }, {})
-  req.issuesByEntryNumber = issuesByEntryNumber
-  next()
-}
-
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
   const { entityCount: entityCountRow, issuesWithReferences, issuesWithoutReferences, entities } = req
@@ -188,22 +162,7 @@ export function formatErrorSummaryParams (req, res, next) {
   next()
 }
 
-export const getEntryNumbersWithIssues = (req, res, next) => {
-  const { issues } = req
-
-  const entryNumbersWithIssues = [...new Set(issues.map(issue => issue.entry_number))]
-
-  req.entryNumbersWithIssues = entryNumbersWithIssues
-
-  next()
-}
-
-export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
-  query: ({ req, params }) => performanceDbApi.fetchEntitiesFromEntryNumbers({ entryNumbers: req.entryNumbersWithIssues, organisationEntity: req.orgInfo.entity, pagination: req.pagination }),
-  result: 'entities',
-  dataset: FetchOptions.fromParams
-})
-
+// as we want the number of entities with issues anyway, we do the pagination here instead of after. need this count in the performance db ideally
 export const paginateEntitiesAndPullOutCount = (req, res, next) => {
   const { entities, pagination } = req
   const { pageNumber } = req.params
@@ -307,16 +266,6 @@ export const fetchIssuesWithCounts = fetchMany({
   result: 'issuesWithCounts'
 })
 
-export const fetchIssues = fetchMany({
-  query: ({ req, params }) => performanceDbApi.issuesQuery({
-    resources: req.resources.map(resourceObj => resourceObj.resource),
-    dataset: params.dataset,
-    issueType: params.issue_type,
-    issueField: params.issue_field
-  }),
-  result: 'issues'
-})
-
 export const fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield = fetchMany({
   query: ({ req, params }) => performanceDbApi.issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery({
     resources: req.resources.map(resourceObj => resourceObj.resource),
@@ -338,5 +287,3 @@ export const fetchIssuesWithoutReferences = fetchMany({
   result: 'issuesWithoutReferences',
   dataset: FetchOptions.fromParams
 })
-
-// export const getReferencesOfIssueEntities
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index ec430096..e20b0cb6 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,6 +1,7 @@
-import { fetchActiveResourcesForOrganisationAndDataset, fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
+import { fetchActiveResourcesForOrganisationAndDataset, fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, fetchSpecification, isResourceAccessible, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, takeResourceIdFromParams } from './common.middleware.js'
 import { fetchIf, fetchMany, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
+import performanceDbApi from '../services/performanceDbApi.js'
 
 const fetchColumnSummary = fetchMany({
   query: ({ params }) => `
@@ -107,6 +108,11 @@ const getDatasetOverview = renderTemplate(
   }
 )
 
+export const fetchLpaDatasetIssues = fetchMany({
+  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resources, params.dataset),
+  result: 'issues'
+})
+
 export default [
   fetchOrgInfo,
   fetchDatasetInfo,

From ccb6ca82d1a80d91bc4920b6f5e1c67a7478ea0e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 11:21:21 +0100
Subject: [PATCH 044/109] add tests for common.middleware

---
 src/middleware/common.middleware.js           |  10 +-
 .../common.middleware.test.js.snap            |  11 +
 .../unit/middleware/common.middleware.test.js | 366 +++++++++++++++++-
 .../datasetTaskList.middleware.test.js        |   2 +-
 .../issueDetails.middleware.test.js           |  40 +-
 .../middleware/issueTable.middleware.test.js  |   2 +-
 vite.config.js                                |   3 +-
 7 files changed, 413 insertions(+), 21 deletions(-)
 create mode 100644 test/unit/middleware/__snapshots__/common.middleware.test.js.snap

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 53f1f9b1..8963deb9 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -177,9 +177,11 @@ export const paginateEntitiesAndPullOutCount = (req, res, next) => {
 }
 
 export const getPaginationOptions = (resultsCount) => (req, res, next) => {
-  const { pageNumber } = req.params
+  let { pageNumber } = req.params
+
+  pageNumber = pageNumber || 1
 
-  req.pagination = { offset: pageNumber * resultsCount, limit: resultsCount }
+  req.pagination = { offset: (pageNumber - 1) * resultsCount, limit: resultsCount }
 
   next()
 }
@@ -189,6 +191,10 @@ export const extractJsonFieldFromEntities = (req, res, next) => {
 
   req.entities = entities.map(entity => {
     const jsonField = entity.json
+    if (!jsonField || jsonField === '') {
+      logger.info(`common.middleware/extractJsonField: No json field for entity ${entity.toString()}`)
+      return entity
+    }
     delete entity.json
     const parsedJson = JSON.parse(jsonField)
     entity = { ...entity, ...parsedJson }
diff --git a/test/unit/middleware/__snapshots__/common.middleware.test.js.snap b/test/unit/middleware/__snapshots__/common.middleware.test.js.snap
new file mode 100644
index 00000000..c771bffa
--- /dev/null
+++ b/test/unit/middleware/__snapshots__/common.middleware.test.js.snap
@@ -0,0 +1,11 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`formatErrorSummaryParams > formats error summary params with all entities 1`] = `"10 issue of type testIssueType"`;
+
+exports[`formatErrorSummaryParams > formats error summary params with no entities 1`] = `"1 issue of type testIssueType"`;
+
+exports[`formatErrorSummaryParams > formats error summary params with some entities 1`] = `"2 issue of type testIssueType"`;
+
+exports[`formatErrorSummaryParams > formats error summary params with some entities 2`] = `"1 issue of type testIssueType in entity 1"`;
+
+exports[`formatErrorSummaryParams > formats error summary params with some entities 3`] = `"/organisations/testLpa/testDataset/testIssueType/testIssueField/entry/1"`;
diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 2c2e24d2..8b4ce82c 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, expect, vi } from 'vitest'
-import { formatErrorSummaryParams, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
+import { addIssuesToEntities, extractJsonFieldFromEntities, formatErrorSummaryParams, getPaginationOptions, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, nestEntityFields, paginateEntitiesAndPullOutCount, pullOutDatasetSpecification, replaceUnderscoreWithHyphenForEntities, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
 import logger from '../../../src/utils/logger'
 
 vi.mock('../../../src/utils/logger')
@@ -62,32 +62,43 @@ describe('resource middleware', () => {
   })
 })
 
-describe('reformatIssuesToBeByEntryNumber', () => {
-  it('reformats the issues by entry number', async () => {
-    const req = { issues: [{ entry_number: '1', issue: 'testIssue' }, { entry_number: '1', issue: 'testIssue2' }, { entry_number: '2', issue: 'testIssue3' }] }
+describe('formatErrorSummaryParams', () => {
+  it('formats error summary params with no entities', () => {
+    const req = {
+      params: { lpa: 'testLpa', dataset: 'testDataset', issue_type: 'testIssueType', issue_field: 'testIssueField' },
+      entityCount: { entity_count: 10 },
+      issuesWithReferences: [],
+      issuesWithoutReferences: [{ field: 'field1', issue_type: 'type1', message: 'message1', reference: { value: '1' } }],
+      entities: []
+    }
     const res = {}
     const next = vi.fn()
-    await reformatIssuesToBeByEntryNumber(req, res, next)
-    expect(req.issuesByEntryNumber).toBeDefined()
-    expect(req.issuesByEntryNumber['1']).toHaveLength(2)
-    expect(req.issuesByEntryNumber['2']).toHaveLength(1)
+    formatErrorSummaryParams(req, res, next)
+    expect(req.errorSummary).toBeDefined()
+    expect(req.errorSummary.heading).toBeUndefined()
+    expect(req.errorSummary.items).toHaveLength(1)
+    expect(req.errorSummary.items[0].html).toMatchSnapshot()
   })
-})
 
-describe('formatErrorSummaryParams', () => {
-  it('formats the error summary params', async () => {
+  it('formats error summary params with some entities', () => {
     const req = {
       params: { lpa: 'testLpa', dataset: 'testDataset', issue_type: 'testIssueType', issue_field: 'testIssueField' },
-      issuesByEntryNumber: { 1: [{ issue: 'testIssue' }], 2: [{ issue: 'testIssue2' }] },
       entityCount: { entity_count: 10 },
-      issueEntitiesCount: 5
+      issuesWithReferences: [{ field: 'field1', issue_type: 'type1', message: 'message1', reference: { value: '1' } }],
+      issuesWithoutReferences: [{ field: 'field2', issue_type: 'type2', message: 'message2', reference: { value: '2' } }],
+      entities: [
+        { reference: { value: '1' } },
+        { reference: { value: '2' } }
+      ]
     }
     const res = {}
     const next = vi.fn()
     formatErrorSummaryParams(req, res, next)
     expect(req.errorSummary).toBeDefined()
-    expect(req.errorSummary.heading).toBeDefined()
+    expect(req.errorSummary.heading).toMatchSnapshot()
     expect(req.errorSummary.items).toHaveLength(2)
+    expect(req.errorSummary.items[0].html).toMatchSnapshot()
+    expect(req.errorSummary.items[0].href).toMatchSnapshot()
   })
 })
 
@@ -118,3 +129,330 @@ describe('pullOutDatasetSpecification', () => {
     expect(reqWithSpecification.specification).toEqual({ dataset: 'mock-dataset', foo: 'bar' })
   })
 })
+
+describe('pagination', () => {
+  describe('getPaginationOptions', () => {
+    it('sets pagination options correctly', () => {
+      const resultsCount = 10
+      const getPaginationOptionsMiddleware = getPaginationOptions(resultsCount)
+      const req = { params: { pageNumber: 2 } }
+      const res = {}
+      const next = vi.fn()
+
+      getPaginationOptionsMiddleware(req, res, next)
+      expect(req.pagination).toBeDefined()
+      expect(req.pagination.offset).toBe(10)
+      expect(req.pagination.limit).toBe(10)
+    })
+
+    it('calls next function', () => {
+      const resultsCount = 10
+      const getPaginationOptionsMiddleware = getPaginationOptions(resultsCount)
+      const req = { params: { pageNumber: 1 } }
+      const res = {}
+      const next = vi.fn()
+
+      getPaginationOptionsMiddleware(req, res, next)
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+
+    it('handles default pageNumber as 1', () => {
+      const resultsCount = 10
+      const getPaginationOptionsMiddleware = getPaginationOptions(resultsCount)
+      const req = { params: {} }
+      const res = {}
+      const next = vi.fn()
+
+      getPaginationOptionsMiddleware(req, res, next)
+      expect(req.pagination).toBeDefined()
+      expect(req.pagination.offset).toBe(0)
+      expect(req.pagination.limit).toBe(10)
+    })
+  })
+
+  describe('paginateEntitiesAndPullOutCount', () => {
+    it('sets entitiesWithIssuesCount to the total number of entities', () => {
+      const req = { entities: [{}, {}, {}], params: { pageNumber: 1 }, pagination: { offset: 0, limit: 2 } }
+      const res = {}
+      const next = vi.fn()
+
+      paginateEntitiesAndPullOutCount(req, res, next)
+      expect(req.entitiesWithIssuesCount).toBe(3)
+    })
+
+    it('paginates entities correctly', () => {
+      const req = { entities: [{}, {}, {}, {}, {}, {}], params: { pageNumber: 2 }, pagination: { offset: 2, limit: 2 } }
+      const res = {}
+      const next = vi.fn()
+
+      paginateEntitiesAndPullOutCount(req, res, next)
+      expect(req.entities).toHaveLength(2)
+      expect(req.entities).toEqual([{}, {}])
+    })
+
+    it('calls next function', () => {
+      const req = { entities: [{}, {}, {}], params: { pageNumber: 1 }, pagination: { offset: 0, limit: 2 } }
+      const res = {}
+      const next = vi.fn()
+
+      paginateEntitiesAndPullOutCount(req, res, next)
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+  })
+})
+
+describe('extractJsonFieldFromEntities', () => {
+  it('extracts and parses json field from entities', () => {
+    const req = {
+      entities: [
+        { id: 1, json: '{"field1": "value1", "field2": "value2"}' },
+        { id: 2, json: '{"field3": "value3", "field4": "value4"}' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    extractJsonFieldFromEntities(req, res, next)
+
+    expect(req.entities).toHaveLength(2)
+    expect(req.entities[0]).toEqual({
+      id: 1,
+      field1: 'value1',
+      field2: 'value2'
+    })
+    expect(req.entities[1]).toEqual({
+      id: 2,
+      field3: 'value3',
+      field4: 'value4'
+    })
+  })
+
+  it('calls next function', () => {
+    const req = {
+      entities: [
+        { id: 1, json: '{"field1": "value1", "field2": "value2"}' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    extractJsonFieldFromEntities(req, res, next)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+
+  it('handles entities with no json field', () => {
+    const req = {
+      entities: [
+        { id: 1 }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    extractJsonFieldFromEntities(req, res, next)
+    expect(req.entities).toHaveLength(1)
+    expect(req.entities[0]).toEqual({ id: 1 })
+  })
+
+  it('handles entities with empty json field', () => {
+    const req = {
+      entities: [
+        { id: 1 }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    extractJsonFieldFromEntities(req, res, next)
+    expect(req.entities).toHaveLength(1)
+    expect(req.entities[0]).toEqual({ id: 1 })
+  })
+})
+
+describe('replaceUnderscoreWithHyphenForEntities', () => {
+  it('replaces underscore with hyphen in entity keys', () => {
+    const req = {
+      entities: [
+        { id: 1, foo_bar: 'value1', baz_qux: 'value2' },
+        { id: 2, quux_foo: 'value3', bar_baz: 'value4' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    replaceUnderscoreWithHyphenForEntities(req, res, next)
+
+    expect(req.entities).toHaveLength(2)
+    expect(req.entities[0]).toEqual({
+      id: 1,
+      'foo-bar': 'value1',
+      'baz-qux': 'value2'
+    })
+    expect(req.entities[1]).toEqual({
+      id: 2,
+      'quux-foo': 'value3',
+      'bar-baz': 'value4'
+    })
+  })
+
+  it('calls next function', () => {
+    const req = {
+      entities: [
+        { id: 1, foo_bar: 'value1' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    replaceUnderscoreWithHyphenForEntities(req, res, next)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+
+  it('handles entities with no underscore in keys', () => {
+    const req = {
+      entities: [
+        { id: 1, foobar: 'value1' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    replaceUnderscoreWithHyphenForEntities(req, res, next)
+    expect(req.entities).toHaveLength(1)
+    expect(req.entities[0]).toEqual({ id: 1, foobar: 'value1' })
+  })
+})
+
+describe('nestEntityFields', () => {
+  it('nests entity fields correctly', () => {
+    const req = {
+      entities: [
+        { id: 1, name: 'John', age: 30 },
+        { id: 2, name: 'Jane', age: 25 }
+      ],
+      specification: {
+        fields: [
+          { field: 'name' },
+          { field: 'age' }
+        ]
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    nestEntityFields(req, res, next)
+
+    expect(req.entities).toHaveLength(2)
+    expect(req.entities[0]).toEqual({
+      id: 1,
+      name: { value: 'John' },
+      age: { value: 30 }
+    })
+    expect(req.entities[1]).toEqual({
+      id: 2,
+      name: { value: 'Jane' },
+      age: { value: 25 }
+    })
+  })
+
+  it('calls next function', () => {
+    const req = {
+      entities: [
+        { id: 1, name: 'John' }
+      ],
+      specification: {
+        fields: [
+          { field: 'name' }
+        ]
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    nestEntityFields(req, res, next)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+
+  it('handles entities with no specification fields', () => {
+    const req = {
+      entities: [
+        { id: 1, name: 'John' }
+      ],
+      specification: {
+        fields: []
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    nestEntityFields(req, res, next)
+    expect(req.entities).toHaveLength(1)
+    expect(req.entities[0]).toEqual({ id: 1, name: 'John' })
+  })
+})
+
+describe('addIssuesToEntities', () => {
+  it('adds issues to entities correctly', () => {
+    const req = {
+      entities: [
+        { entryNumber: 1, name: { value: 'John' }, age: { value: 30 } },
+        { entryNumber: 2, name: { value: 'Jane' }, age: { value: 25 } }
+      ],
+      issuesWithReferences: [
+        { entryNumber: 1, field: 'name', value: 'nameIssueValue' },
+        { entryNumber: 1, field: 'age', value: 'ageIssueValue' },
+        { entryNumber: 2, field: 'name', value: 'nameIssueValue2' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addIssuesToEntities(req, res, next)
+
+    expect(req.entitiesWithIssues).toHaveLength(2)
+    expect(req.entitiesWithIssues[0]).toEqual({
+      entryNumber: 1,
+      name: { value: 'nameIssueValue', issue: { entryNumber: 1, field: 'name', value: 'nameIssueValue' } },
+      age: { value: 'ageIssueValue', issue: { entryNumber: 1, field: 'age', value: 'ageIssueValue' } }
+    })
+    expect(req.entitiesWithIssues[1]).toEqual({
+      entryNumber: 2,
+      name: { value: 'nameIssueValue2', issue: { entryNumber: 2, field: 'name', value: 'nameIssueValue2' } },
+      age: { value: 25 }
+    })
+  })
+
+  it('calls next function', () => {
+    const req = {
+      entities: [
+        { entryNumber: 1, name: { value: 'John' } }
+      ],
+      issuesWithReferences: [
+        { entryNumber: 1, field: 'name', value: 'Invalid name' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addIssuesToEntities(req, res, next)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+
+  it('handles entities with no issues', () => {
+    const req = {
+      entities: [
+        { entryNumber: 1, name: { value: 'John' } }
+      ],
+      issuesWithReferences: []
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addIssuesToEntities(req, res, next)
+    expect(req.entitiesWithIssues).toHaveLength(1)
+    expect(req.entitiesWithIssues[0]).toEqual({
+      entryNumber: 1,
+      name: { value: 'John' }
+    })
+  })
+})
diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js
index d176617b..62238940 100644
--- a/test/unit/middleware/datasetTaskList.middleware.test.js
+++ b/test/unit/middleware/datasetTaskList.middleware.test.js
@@ -19,7 +19,7 @@ describe('datasetTaskList.middleware.js', () => {
         orgInfo: { name: 'Example Organisation', organisation: 'ORG' },
         dataset: { name: 'Example Dataset' },
         resource: { resource: 'mock-resource' },
-        issues: [
+        issuesWithCounts: [
           {
             issue: 'Example issue 1',
             issue_type: 'Example issue type 1',
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 5252bdc8..26bae623 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -45,7 +45,7 @@ describe('issueDetails.middleware.js', () => {
         dataset,
         entryData,
         issues,
-        entryNumber: 10,
+        pageNumber: 10,
         resource: { resource: requestParams.resourceId },
         issuesByEntryNumber: {
           10: [
@@ -67,6 +67,22 @@ describe('issueDetails.middleware.js', () => {
               href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
             }
           ]
+        },
+        entities: [
+          {
+            reference: '1',
+            'start-date': 'invalid'
+          }
+        ],
+        specification: {
+          fields: [
+            {
+              field: 'reference'
+            },
+            {
+              field: 'start-date'
+            }
+          ]
         }
       }
       v.parse(IssueDetailsQueryParams, req.params)
@@ -140,7 +156,7 @@ describe('issueDetails.middleware.js', () => {
       const req = {
         params: requestParams,
         // middleware supplies the below
-        entryNumber: 10,
+        pageNumber: 2,
         entityCount: { entity_count: 3 },
         issueEntitiesCount: 1,
         orgInfo,
@@ -164,6 +180,26 @@ describe('issueDetails.middleware.js', () => {
               message: 'mock message'
             }
           ]
+        },
+        entities: [
+          {
+            reference: '1',
+            startDate: 'invalid'
+          },
+          {
+            reference: '2',
+            startDate: 'invalid'
+          }
+        ],
+        specification: {
+          fields: [
+            {
+              field: 'reference'
+            },
+            {
+              field: 'start-date'
+            }
+          ]
         }
       }
 
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 5fba7672..66d25325 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -46,7 +46,7 @@ describe('issueTable.middleware.js', () => {
     it('should correctly set next when there is more than one page', () => {
       const req = {
         params: { pageNumber: 1, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
-        issueEntitiesCount: 60
+        entities: { length: 60 }
       }
       const res = {}
       const next = vi.fn()
diff --git a/vite.config.js b/vite.config.js
index 2783b64d..ba4a6ed5 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -14,7 +14,8 @@ export default defineConfig({
       // you can include other reporters, but 'json-summary' is required, json is recommended
       reporter: ['text', 'json-summary', 'json'],
       // If you want a coverage reports even if your tests are failing, include the reportOnFailure option
-      reportOnFailure: true
+      reportOnFailure: true,
+      snapshotDir: 'snapshots'
     }
   }
 })

From c25ee0bf7c9298f9d3b1cb8e335ecead089fed19 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 11:30:12 +0100
Subject: [PATCH 045/109] fix pagination tests

---
 src/middleware/issueTable.middleware.js        |  4 ++--
 .../middleware/issueTable.middleware.test.js   | 18 ++++++++++++------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 6e5dc20b..0c02e091 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -111,7 +111,7 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
  *
  * @returns {void}
  */
-export const createPaginationTemplatePrams = (req, res, next) => {
+export const createPaginationTemplateParams = (req, res, next) => {
   const { entitiesWithIssuesCount } = req
   const { pageNumber, lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
@@ -180,7 +180,7 @@ export default [
   fetchIf(hasEntities, nestEntityFields),
   fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
-  createPaginationTemplatePrams,
+  createPaginationTemplateParams,
   prepareIssueTableTemplateParams,
   getIssueTable,
   logPageError
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 66d25325..8ab5a2d7 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, vi, expect } from 'vitest'
-import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, createPaginationTemplatePrams } from '../../../src/middleware/issueTable.middleware.js'
+import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, createPaginationTemplateParams } from '../../../src/middleware/issueTable.middleware.js'
 // import { pagination } from '../../../src/utils/pagination.js'
 
 import mocker from '../../utils/mocker.js'
@@ -45,15 +45,21 @@ describe('issueTable.middleware.js', () => {
   describe('createPaginationTemplatePrams', () => {
     it('should correctly set next when there is more than one page', () => {
       const req = {
-        params: { pageNumber: 1, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
-        entities: { length: 60 }
+        params: {
+          pageNumber: 1,
+          lpa: 'some-lpa',
+          dataset: 'some-dataset-id',
+          issue_type: 'some-issue-type',
+          issue_field: 'some-issue-field'
+        },
+        entitiesWithIssuesCount: 200
       }
       const res = {}
       const next = vi.fn()
 
       const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
 
-      createPaginationTemplatePrams(req, res, next)
+      createPaginationTemplateParams(req, res, next)
 
       expect(req.pagination.previous).not.toBeDefined()
       expect(req.pagination.next).toBeDefined()
@@ -72,7 +78,7 @@ describe('issueTable.middleware.js', () => {
 
       const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
 
-      createPaginationTemplatePrams(req, res, next)
+      createPaginationTemplateParams(req, res, next)
 
       expect(req.pagination.next).not.toBeDefined()
       expect(req.pagination.previous).toBeDefined()
@@ -91,7 +97,7 @@ describe('issueTable.middleware.js', () => {
 
       const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
 
-      createPaginationTemplatePrams(req, res, next)
+      createPaginationTemplateParams(req, res, next)
 
       expect(req.pagination.items).toEqual([
         {

From 745e6fd4b7ff3dc03dcd900c800043a5dbb10c0e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 12:38:06 +0100
Subject: [PATCH 046/109] move createPaginationTemplateParamsMiddleware to
 common

---
 src/middleware/common.middleware.js       | 56 +++++++++++++++++
 src/middleware/issueDetails.middleware.js | 60 ++++++-------------
 src/middleware/issueTable.middleware.js   | 73 +++++------------------
 3 files changed, 89 insertions(+), 100 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 8963deb9..c98a8bb2 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -3,6 +3,7 @@ import { types } from '../utils/logging.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { fetchOne, FetchOptions, FetchOneFallbackPolicy, fetchMany } from './middleware.builders.js'
 import * as v from 'valibot'
+import { pagination } from '../utils/pagination.js'
 
 /**
  * Middleware. Set `req.handlerName` to a string that will identify
@@ -186,6 +187,61 @@ export const getPaginationOptions = (resultsCount) => (req, res, next) => {
   next()
 }
 
+/**
+ * Creates pagination template parameters for the request.
+ *
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @param {Function} next - The next middleware function in the chain.
+ *
+ * @description
+ * This middleware function extracts pagination-related parameters from the request,
+ * calculates the total number of pages, and creates a pagination object that can be used
+ * to render pagination links in the template.
+ *
+ * @returns {void}
+ */
+export const createPaginationTemplateParams = (req, res, next) => {
+  const { resultsCount, urlSubPath, paginationPageLength } = req
+  const { pageNumber } = req.params
+
+  const totalPages = Math.floor(resultsCount / paginationPageLength)
+
+  const paginationObj = {}
+  if (pageNumber > 1) {
+    paginationObj.previous = {
+      href: `${urlSubPath}${pageNumber - 1}`
+    }
+  }
+
+  if (pageNumber < totalPages) {
+    paginationObj.next = {
+      href: `${urlSubPath}${pageNumber + 1}`
+    }
+  }
+
+  paginationObj.items = pagination(totalPages, pageNumber).map(item => {
+    if (item === '...') {
+      return {
+        type: 'ellipsis',
+        ellipsis: true,
+        href: '#'
+      }
+    } else {
+      return {
+        type: 'number',
+        number: item,
+        href: `${urlSubPath}${item}`,
+        current: pageNumber === parseInt(item)
+      }
+    }
+  })
+
+  req.pagination = paginationObj
+
+  next()
+}
+
 export const extractJsonFieldFromEntities = (req, res, next) => {
   const { entities } = req
 
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 6fc92980..c9f4c7af 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,5 +1,6 @@
 import {
   addIssuesToEntities,
+  createPaginationTemplateParams,
   extractJsonFieldFromEntities,
   fetchActiveResourcesForOrganisationAndDataset,
   fetchDatasetInfo,
@@ -23,7 +24,6 @@ import {
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
-import { pagination } from '../utils/pagination.js'
 
 export const IssueDetailsQueryParams = v.strictObject({
   lpa: v.string(),
@@ -44,7 +44,7 @@ const validateIssueDetailsQueryParams = validateQueryParams.bind({
  * @param {{value: string}?} issue
  * @returns {string}
  */
-const issueErrorMessageHtml = (errorMessage, issue) =>
+export const issueErrorMessageHtml = (errorMessage, issue) =>
     `<p class="govuk-error-message">${errorMessage}</p>${
       issue ? issue.value ?? '' : ''
     }`
@@ -56,7 +56,7 @@ const issueErrorMessageHtml = (errorMessage, issue) =>
  * @param {*} classes
  * @returns {{key: {text: string}, value: { html: string}, classes: string}}
  */
-const getIssueField = (text, html, classes) => {
+export const getIssueField = (text, html, classes = '') => {
   return {
     key: {
       text
@@ -68,6 +68,17 @@ const getIssueField = (text, html, classes) => {
   }
 }
 
+export const setPagePaginationOptions = (req, res, next) => {
+  const { issueEntitiesCount } = req
+  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+
+  req.resultsCount = issueEntitiesCount.length
+  req.urlSubPath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
+  req.paginationPageLength = 1
+
+  next()
+}
+
 /**
  * Middleware. Prepares template parameters for the issue details page.
  *
@@ -81,12 +92,10 @@ const getIssueField = (text, html, classes) => {
  * from the request, and organizes it into a template parameters object that can be used to render the page.
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entities, issueEntitiesCount, errorSummary, specification } = req
-  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField, pageNumber: pageNumberString } = req.params
+  const { entities, issueEntitiesCount, errorSummary, specification, pagination } = req
+  const { issue_type: issueType, issue_field: issueField, pageNumber: pageNumberString } = req.params
   const pageNumber = parseInt(pageNumberString)
 
-  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
-
   const entity = entities[pageNumber - 1]
 
   const fields = specification.fields.map(({ field }) => {
@@ -106,39 +115,6 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     geometries: [entity.geometry.value]
   }
 
-  const paginationObj = {
-    items: []
-  }
-
-  if (pageNumber > 1) {
-    paginationObj.previous = {
-      href: `${BaseSubpath}${pageNumber - 1}`
-    }
-  }
-
-  if (pageNumber < entities.length) {
-    paginationObj.next = {
-      href: `${BaseSubpath}${pageNumber + 1}`
-    }
-  }
-
-  paginationObj.items = pagination(issueEntitiesCount, pageNumber).map(item => {
-    if (item === '...') {
-      return {
-        type: 'ellipsis',
-        ellipsis: true,
-        href: '#'
-      }
-    } else {
-      return {
-        type: 'number',
-        number: item,
-        href: `${BaseSubpath}${item}`,
-        current: pageNumber === parseInt(item)
-      }
-    }
-  })
-
   // schema: OrgIssueDetails
   req.templateParams = {
     organisation: req.orgInfo,
@@ -147,7 +123,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     entry,
     issueType,
     issueField,
-    pagination: paginationObj,
+    pagination,
     issueEntitiesCount
   }
 
@@ -182,6 +158,8 @@ export default [
   fetchEntityCount,
   fetchIssueEntitiesCount,
   formatErrorSummaryParams,
+  setPagePaginationOptions,
+  createPaginationTemplateParams,
   prepareIssueDetailsTemplateParams,
   getIssueDetails,
   logPageError
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 0c02e091..19709f2a 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,4 +1,3 @@
-import { pagination } from '../utils/pagination.js'
 import {
   addIssuesToEntities,
   extractJsonFieldFromEntities,
@@ -21,7 +20,8 @@ import {
   fetchActiveResourcesForOrganisationAndDataset,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
-  fetchIssuesWithoutReferences
+  fetchIssuesWithoutReferences,
+  createPaginationTemplateParams
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -50,6 +50,17 @@ export const setDefaultQueryParams = (req, res, next) => {
   next()
 }
 
+export const setPagePageOptions = (pageLength) => (req, res, next) => {
+  const { entitiesWithIssuesCount } = req
+  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+
+  req.resultsCount = entitiesWithIssuesCount
+  req.urlSubPath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
+  req.paginationPageLength = pageLength
+
+  next()
+}
+
 /**
  * Middleware function to prepare issue table template params
  *
@@ -97,63 +108,6 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
   next()
 }
 
-/**
- * Creates pagination template parameters for the request.
- *
- * @param {Object} req - The request object.
- * @param {Object} res - The response object.
- * @param {Function} next - The next middleware function in the chain.
- *
- * @description
- * This middleware function extracts pagination-related parameters from the request,
- * calculates the total number of pages, and creates a pagination object that can be used
- * to render pagination links in the template.
- *
- * @returns {void}
- */
-export const createPaginationTemplateParams = (req, res, next) => {
-  const { entitiesWithIssuesCount } = req
-  const { pageNumber, lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-
-  const totalPages = Math.floor(entitiesWithIssuesCount / paginationPageLength)
-
-  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
-
-  const paginationObj = {}
-  if (pageNumber > 1) {
-    paginationObj.previous = {
-      href: `${BaseSubpath}${pageNumber - 1}`
-    }
-  }
-
-  if (pageNumber < totalPages) {
-    paginationObj.next = {
-      href: `${BaseSubpath}${pageNumber + 1}`
-    }
-  }
-
-  paginationObj.items = pagination(totalPages, pageNumber).map(item => {
-    if (item === '...') {
-      return {
-        type: 'ellipsis',
-        ellipsis: true,
-        href: '#'
-      }
-    } else {
-      return {
-        type: 'number',
-        number: item,
-        href: `${BaseSubpath}${item}`,
-        current: pageNumber === parseInt(item)
-      }
-    }
-  })
-
-  req.pagination = paginationObj
-
-  next()
-}
-
 export const getIssueTable = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/issueTable.html',
@@ -180,6 +134,7 @@ export default [
   fetchIf(hasEntities, nestEntityFields),
   fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
+  setPagePageOptions(paginationPageLength),
   createPaginationTemplateParams,
   prepareIssueTableTemplateParams,
   getIssueTable,

From d8c1de6fa26dd44cf2c4881bf8643214830b389e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 12:38:26 +0100
Subject: [PATCH 047/109] move tests for paginationTemplateParams

---
 .../unit/middleware/common.middleware.test.js | 50 +++++++++-
 .../middleware/issueTable.middleware.test.js  | 92 ++++---------------
 2 files changed, 65 insertions(+), 77 deletions(-)

diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 8b4ce82c..1527fc07 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, expect, vi } from 'vitest'
-import { addIssuesToEntities, extractJsonFieldFromEntities, formatErrorSummaryParams, getPaginationOptions, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, nestEntityFields, paginateEntitiesAndPullOutCount, pullOutDatasetSpecification, replaceUnderscoreWithHyphenForEntities, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
+import { addIssuesToEntities, createPaginationTemplateParams, extractJsonFieldFromEntities, formatErrorSummaryParams, getPaginationOptions, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, nestEntityFields, paginateEntitiesAndPullOutCount, pullOutDatasetSpecification, replaceUnderscoreWithHyphenForEntities, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
 import logger from '../../../src/utils/logger'
 
 vi.mock('../../../src/utils/logger')
@@ -199,6 +199,54 @@ describe('pagination', () => {
       expect(next).toHaveBeenCalledTimes(1)
     })
   })
+
+  describe('createPaginationTemplateParams', () => {
+    it('creates pagination object with correct parameters', () => {
+      const req = {
+        resultsCount: 100,
+        urlSubPath: '/api/results/',
+        paginationPageLength: 10,
+        params: { pageNumber: 2 }
+      }
+      const res = {}
+      const next = vi.fn()
+
+      createPaginationTemplateParams(req, res, next)
+
+      expect(req.pagination).toEqual({
+        previous: { href: '/api/results/1' },
+        next: { href: '/api/results/3' },
+        items: [
+          { type: 'number', number: 1, href: '/api/results/1', current: false },
+          { type: 'number', number: 2, href: '/api/results/2', current: true },
+          { type: 'number', number: 3, href: '/api/results/3', current: false },
+          { type: 'ellipsis', ellipsis: true, href: '#' },
+          { type: 'number', number: 10, href: '/api/results/10', current: false }
+        ]
+      })
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+
+    it('handles edge cases for pagination', () => {
+      const req = {
+        resultsCount: 10,
+        urlSubPath: '/api/results/',
+        paginationPageLength: 10,
+        params: { pageNumber: 1 }
+      }
+      const res = {}
+      const next = vi.fn()
+
+      createPaginationTemplateParams(req, res, next)
+
+      expect(req.pagination).toEqual({
+        items: [
+          { type: 'number', number: 1, href: '/api/results/1', current: true }
+        ]
+      })
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+  })
 })
 
 describe('extractJsonFieldFromEntities', () => {
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 8ab5a2d7..cb1e00fc 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, vi, expect } from 'vitest'
-import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, createPaginationTemplateParams } from '../../../src/middleware/issueTable.middleware.js'
+import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, setPagePageOptions } from '../../../src/middleware/issueTable.middleware.js'
 // import { pagination } from '../../../src/utils/pagination.js'
 
 import mocker from '../../utils/mocker.js'
@@ -42,88 +42,28 @@ describe('issueTable.middleware.js', () => {
     })
   })
 
-  describe('createPaginationTemplatePrams', () => {
-    it('should correctly set next when there is more than one page', () => {
+  describe('setPagePageOptions', () => {
+    it('sets request parameters for pagination', () => {
+      const pageLength = 20
+      const setPagePageOptionsMiddleware = setPagePageOptions(pageLength)
       const req = {
+        entitiesWithIssuesCount: 50,
         params: {
-          pageNumber: 1,
-          lpa: 'some-lpa',
-          dataset: 'some-dataset-id',
-          issue_type: 'some-issue-type',
-          issue_field: 'some-issue-field'
-        },
-        entitiesWithIssuesCount: 200
-      }
-      const res = {}
-      const next = vi.fn()
-
-      const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
-
-      createPaginationTemplateParams(req, res, next)
-
-      expect(req.pagination.previous).not.toBeDefined()
-      expect(req.pagination.next).toBeDefined()
-      expect(req.pagination.next).toEqual({
-        href: `${BaseSubpath}${2}`
-      })
-    })
-
-    it('should correct set previous when the current pageNumber is greater than 1', () => {
-      const req = {
-        params: { pageNumber: 2, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
-        issueEntitiesCount: 60
-      }
-      const res = {}
-      const next = vi.fn()
-
-      const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
-
-      createPaginationTemplateParams(req, res, next)
-
-      expect(req.pagination.next).not.toBeDefined()
-      expect(req.pagination.previous).toBeDefined()
-      expect(req.pagination.previous).toEqual({
-        href: `${BaseSubpath}${1}`
-      })
-    })
-
-    it('should correctly set the items', () => {
-      const req = {
-        params: { pageNumber: 4, lpa: 'lpa', dataset: 'datasetId', issue_type: 'issueType', issue_field: 'issueField' },
-        issueEntitiesCount: 60
+          lpa: 'lpa-123',
+          dataset: 'dataset-456',
+          issue_type: 'issue-type',
+          issue_field: 'issue-field'
+        }
       }
       const res = {}
       const next = vi.fn()
 
-      const BaseSubpath = `/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/`
+      setPagePageOptionsMiddleware(req, res, next)
 
-      createPaginationTemplateParams(req, res, next)
-
-      expect(req.pagination.items).toEqual([
-        {
-          current: false,
-          href: `${BaseSubpath}1`,
-          number: 1,
-          type: 'number'
-        },
-        {
-          ellipsis: true,
-          href: '#',
-          type: 'ellipsis'
-        },
-        {
-          current: true,
-          href: `${BaseSubpath}4`,
-          number: 4,
-          type: 'number'
-        },
-        {
-          current: false,
-          href: `${BaseSubpath}5`,
-          number: 5,
-          type: 'number'
-        }
-      ])
+      expect(req.resultsCount).toBe(50)
+      expect(req.urlSubPath).toBe('/organisations/lpa-123/dataset-456/issue-type/issue-field/')
+      expect(req.paginationPageLength).toBe(pageLength)
+      expect(next).toHaveBeenCalledTimes(1)
     })
   })
 

From c47c95caa532656caf9436a490a91b803b8d55a4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 12:50:43 +0100
Subject: [PATCH 048/109] added new tests to issueDetails.middleware

---
 src/middleware/issueDetails.middleware.js     |   5 +
 .../issueDetails.middleware.test.js           | 357 ++++++------------
 2 files changed, 122 insertions(+), 240 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index c9f4c7af..1b17a7fa 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -101,6 +101,11 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   const fields = specification.fields.map(({ field }) => {
     let valueHtml = ''
     let classes = ''
+    if (!entity[field]) {
+      entity[field] = {
+        value: ''
+      }
+    }
     if (entity[field].issue) {
       valueHtml += issueErrorMessageHtml(entity[field].issue.message, null)
       classes += 'dl-summary-card-list__row--error'
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 26bae623..de468a4d 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -1,262 +1,139 @@
-import { describe, it, vi, expect } from 'vitest'
-import * as v from 'valibot'
+import { describe, it, vi, expect, beforeEach } from 'vitest'
 
-import performanceDbApi from '../../../src/services/performanceDbApi.js'
-import { getIssueDetails, IssueDetailsQueryParams, prepareIssueDetailsTemplateParams } from '../../../src/middleware/issueDetails.middleware.js'
+import { getIssueDetails, getIssueField, issueErrorMessageHtml, prepareIssueDetailsTemplateParams } from '../../../src/middleware/issueDetails.middleware.js'
 import mocker from '../../utils/mocker.js'
 import { DatasetNameField, errorSummaryField, OrgField } from '../../../src/routes/schemas.js'
 
 vi.mock('../../../src/services/performanceDbApi.js')
 
 describe('issueDetails.middleware.js', () => {
-  const orgInfo = { name: 'mock lpa', organisation: 'ORG' }
-  const dataset = { name: 'mock dataset', dataset: 'mock-dataset', collection: 'mock-collection' }
-  const entryData = [
-    {
-      field: 'start-date',
-      value: '02-02-2022',
-      entry_number: 10
-    }
-  ]
-  const issues = [
-    {
-      entry_number: 10,
-      field: 'start-date',
-      value: '02-02-2022'
-    }
-  ]
-
-  describe('prepareIssueDetailsTemplateParams', () => {
-    it('should correctly set the template params', async () => {
-      const requestParams = {
-        lpa: 'test-lpa',
-        dataset: 'test-dataset',
-        issue_type: 'test-issue-type',
-        issue_field: 'test-issue-field',
-        resourceId: 'test-resource-id',
-        pageNumber: '1'
-      }
-      const req = {
-        params: requestParams,
-        // middleware supplies the below
-        entityCount: { entity_count: 3 },
-        issueEntitiesCount: 1,
-        orgInfo,
-        dataset,
-        entryData,
-        issues,
-        pageNumber: 10,
-        resource: { resource: requestParams.resourceId },
-        issuesByEntryNumber: {
-          10: [
-            {
-              field: 'start-date',
-              value: '02-02-2022',
-              line_number: 1,
-              entry_number: 10,
-              message: 'mock message',
-              issue_type: 'mock type'
-            }
-          ]
-        },
-        errorSummary: {
-          heading: 'mockHeading',
-          items: [
-            {
-              html: 'mock task message 1 in record 10',
-              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
-            }
-          ]
-        },
-        entities: [
-          {
-            reference: '1',
-            'start-date': 'invalid'
-          }
-        ],
-        specification: {
-          fields: [
-            {
-              field: 'reference'
-            },
-            {
-              field: 'start-date'
-            }
-          ]
-        }
-      }
-      v.parse(IssueDetailsQueryParams, req.params)
+  describe('issueErrorMessageHtml', () => {
+    it('should return an HTML string with the error message and issue value', () => {
+      const errorMessage = 'Mock error message'
+      const issue = { value: '02-02-2022' }
+      const result = issueErrorMessageHtml(errorMessage, issue)
+      expect(result).toBe(`<p class="govuk-error-message">${errorMessage}</p>02-02-2022`)
+    })
 
-      issues.forEach(issue => {
-        vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
-      })
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
+    it('should return an HTML string with only the error message if issue is null', () => {
+      const errorMessage = 'Mock error message'
+      const result = issueErrorMessageHtml(errorMessage, null)
+      expect(result).toBe(`<p class="govuk-error-message">${errorMessage}</p>`)
+    })
 
-      prepareIssueDetailsTemplateParams(req, {}, () => {})
+    it('should return an HTML string with only the error message if issue.value is null', () => {
+      const errorMessage = 'Mock error message'
+      const issue = { value: null }
+      const result = issueErrorMessageHtml(errorMessage, issue)
+      expect(result).toBe(`<p class="govuk-error-message">${errorMessage}</p>`)
+    })
+  })
 
-      const expectedTempalteParams = {
-        organisation: {
-          name: 'mock lpa',
-          organisation: 'ORG'
-        },
-        dataset: {
-          name: 'mock dataset',
-          dataset: 'mock-dataset',
-          collection: 'mock-collection'
-        },
-        errorSummary: req.errorSummary,
-        entry: {
-          title: 'entry: 10',
-          fields: [
-            {
-              key: { text: 'start-date' },
-              value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
-              classes: 'dl-summary-card-list__row--error'
-            }
-          ],
-          geometries: []
-        },
-        issueType: 'test-issue-type',
-        issueField: 'test-issue-field',
-        pagination: {
-          items: [{
-            current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/1',
-            number: 1,
-            type: 'number'
-          }]
-        },
-        issueEntitiesCount: 1
-      }
+  describe('getIssueField', () => {
+    it('should return an object with key, value, and classes properties', () => {
+      const text = 'Mock text'
+      const html = '<p>Mock html</p>'
+      const classes = 'mock-classes'
+      const result = getIssueField(text, html, classes)
+      expect(result).toEqual({
+        key: { text },
+        value: { html },
+        classes
+      })
+    })
 
-      expect(req.templateParams).toEqual(expectedTempalteParams)
+    it('should return an object with default classes if classes is not provided', () => {
+      const text = 'Mock text'
+      const html = '<p>Mock html</p>'
+      const result = getIssueField(text, html)
+      expect(result).toEqual({
+        key: { text },
+        value: { html },
+        classes: ''
+      })
     })
+  })
 
-    it('should correctly set the template params with the correct geometry params', async () => {
-      const entryData = [
-        {
-          field: 'start-date',
-          value: '02-02-2022',
-          entry_number: 10
-        },
-        {
-          field: 'geometry',
-          value: 'POINT(0 0)',
-          entry_number: 10
-        }
-      ]
-      const requestParams = {
-        lpa: 'test-lpa',
-        dataset: 'test-dataset',
-        issue_type: 'test-issue-type',
-        issue_field: 'test-issue-field',
-        resourceId: 'test-resource-id',
+  describe('prepareIssueDetailsTemplateParams', () => {
+    const req = {
+      entities: [
+        { reference: { value: 'entry-1' }, geometry: { value: 'geom-1' }, field1: { value: 'val-1', issue: { message: 'error' } } },
+        { reference: { value: 'entry-2' }, geometry: { value: 'geom-2' }, field2: { value: 'val-2' } }
+      ],
+      issueEntitiesCount: 2,
+      errorSummary: 'Mock error summary',
+      specification: {
+        fields: [
+          { field: 'reference', label: 'Reference' },
+          { field: 'geometry', label: 'Geometry' },
+          { field: 'field1', label: 'Field 1' },
+          { field: 'field2', label: 'Field 2' },
+          { field: 'field3', label: 'Field 3' }
+        ]
+      },
+      params: {
+        lpa: 'lpa-1',
+        dataset: 'dataset-1',
+        issue_type: 'issue-type-1',
+        issue_field: 'issue-field-1',
         pageNumber: '1'
-      }
-      const req = {
-        params: requestParams,
-        // middleware supplies the below
-        pageNumber: 2,
-        entityCount: { entity_count: 3 },
-        issueEntitiesCount: 1,
-        orgInfo,
-        dataset,
-        entryData,
-        issues,
-        resource: { resource: requestParams.resourceId },
-        errorSummary: {
-          heading: 'mock heading',
-          items: [
-            {
-              html: 'mock task message 1 in record 10',
-              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/10'
-            }
-          ]
-        },
-        issuesByEntryNumber: {
-          10: [
-            {
-              field: 'start-date',
-              message: 'mock message'
-            }
-          ]
-        },
-        entities: [
-          {
-            reference: '1',
-            startDate: 'invalid'
-          },
-          {
-            reference: '2',
-            startDate: 'invalid'
-          }
-        ],
-        specification: {
-          fields: [
-            {
-              field: 'reference'
-            },
-            {
-              field: 'start-date'
-            }
-          ]
-        }
-      }
+      },
+      orgInfo: { name: 'Org Name' },
+      dataset: { name: 'Dataset Name' },
+      pagination: 'paginationObject'
+    }
 
-      v.parse(IssueDetailsQueryParams, req.params)
+    const res = {}
+    const next = vi.fn()
 
-      issues.forEach(issue => {
-        vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
-      })
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
+    beforeEach(() => {
+      req.templateParams = {}
+    })
 
-      prepareIssueDetailsTemplateParams(req, {}, () => {})
+    it('should set templateParams on the request object', () => {
+      prepareIssueDetailsTemplateParams(req, res, next)
+      expect(req.templateParams).toBeDefined()
+    })
 
-      const expectedTemplateParams = {
-        organisation: {
-          name: 'mock lpa',
-          organisation: 'ORG'
-        },
-        dataset: {
-          name: 'mock dataset',
-          dataset: 'mock-dataset',
-          collection: 'mock-collection'
-        },
-        errorSummary: req.errorSummary,
-        entry: {
-          title: 'entry: 10',
-          fields: [
-            {
-              key: { text: 'start-date' },
-              value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
-              classes: 'dl-summary-card-list__row--error'
-            },
-            {
-              classes: '',
-              key: {
-                text: 'geometry'
-              },
-              value: {
-                html: 'POINT(0 0)'
-              }
-            }
-          ],
-          geometries: ['POINT(0 0)']
-        },
-        issueType: 'test-issue-type',
-        issueField: 'test-issue-field',
-        pagination: {
-          items: [{
-            current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/entry/1',
-            number: 1,
-            type: 'number'
-          }]
-        },
-        issueEntitiesCount: 1
-      }
+    it('should set organisation, dataset, and errorSummary on templateParams', () => {
+      prepareIssueDetailsTemplateParams(req, res, next)
+      expect(req.templateParams.organisation).toEqual(req.orgInfo)
+      expect(req.templateParams.dataset).toEqual(req.dataset)
+      expect(req.templateParams.errorSummary).toBe(req.errorSummary)
+    })
+
+    it('should set entry on templateParams with correct fields', () => {
+      prepareIssueDetailsTemplateParams(req, res, next)
+      expect(req.templateParams.entry).toBeDefined()
+      expect(req.templateParams.entry.title).toBe('entry: entry-1')
+      expect(req.templateParams.entry.fields).toHaveLength(5)
+      expect(req.templateParams.entry.geometries).toEqual(['geom-1'])
+      expect(req.templateParams.entry.fields[0].key.text).toBe('reference')
+      expect(req.templateParams.entry.fields[0].value.html).toBe('entry-1')
+      expect(req.templateParams.entry.fields[0].classes).toBe('')
+      expect(req.templateParams.entry.fields[1].key.text).toBe('geometry')
+      expect(req.templateParams.entry.fields[1].value.html).toBe('geom-1')
+      expect(req.templateParams.entry.fields[1].classes).toBe('')
+      expect(req.templateParams.entry.fields[2].key.text).toBe('field1')
+      expect(req.templateParams.entry.fields[2].value.html).toBe('<p class="govuk-error-message">error</p>val-1')
+      expect(req.templateParams.entry.fields[2].classes).toBe('dl-summary-card-list__row--error')
+      expect(req.templateParams.entry.fields[3].key.text).toBe('field2')
+      expect(req.templateParams.entry.fields[3].value.html).toBe('')
+      expect(req.templateParams.entry.fields[3].classes).toBe('')
+      expect(req.templateParams.entry.fields[4].key.text).toBe('field3')
+      expect(req.templateParams.entry.fields[4].value.html).toBe('')
+      expect(req.templateParams.entry.fields[4].classes).toBe('')
+    })
+
+    it('should set pagination on templateParams with correct items', () => {
+      prepareIssueDetailsTemplateParams(req, res, next)
+      expect(req.templateParams.pagination).toEqual('paginationObject')
+    })
 
-      expect(req.templateParams).toEqual(expectedTemplateParams)
+    it('should call next function', () => {
+      const nextSpy = vi.fn()
+      prepareIssueDetailsTemplateParams(req, res, nextSpy)
+      expect(nextSpy).toHaveBeenCalledTimes(1)
     })
   })
 

From 15ff1a345f87b974ae782c34a62cea4daa1a8415 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 13:01:15 +0100
Subject: [PATCH 049/109] make sure to pass in array of resourceids

---
 src/middleware/datasetOverview.middleware.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index aec1856a..ab2797bd 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -152,7 +152,10 @@ const getDatasetOverview = renderTemplate(
 )
 
 export const fetchLpaDatasetIssues = fetchMany({
-  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resources, params.dataset),
+  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(
+    req.resources.map(resource => resource.resource),
+    params.dataset
+  ),
   result: 'issues'
 })
 

From 3defdeb82d1f5517ce7dcfb94fb80726c887be13 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 13:06:08 +0100
Subject: [PATCH 050/109] fix dataset details page

---
 src/middleware/issueDetails.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 1b17a7fa..ac8b20ae 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -152,8 +152,8 @@ export default [
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
-  fetchIssuesWithoutReferences,
   fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithoutReferences,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
   fetchIf(hasEntities, extractJsonFieldFromEntities),

From a868cab335f10631e90b83e09446d60d16c5e7e8 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 13:15:33 +0100
Subject: [PATCH 051/109] make sure links are correct from table view

---
 src/middleware/issueTable.middleware.js       | 12 ++++++--
 .../middleware/issueTable.middleware.test.js  | 29 +++++++++++++++++--
 2 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 19709f2a..1ebf73dd 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -50,6 +50,14 @@ export const setDefaultQueryParams = (req, res, next) => {
   next()
 }
 
+export const addEntityPageNumberToEntity = (req, res, next) => {
+  const { entities } = req
+
+  req.entities = entities.map((entity, index) => ({ ...entity, entityPageNumber: index + 1 }))
+
+  next()
+}
+
 export const setPagePageOptions = (pageLength) => (req, res, next) => {
   const { entitiesWithIssuesCount } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
@@ -81,8 +89,7 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
       specification.fields.forEach(fieldObject => {
         const { field } = fieldObject
         if (field === 'reference') {
-          const pageNumber = index + 1
-          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${pageNumber}`
+          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${entity.entityPageNumber}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field].value}</a>`, error: entity[field].issue }
         } else if (entity[field]) {
           columns[field] = { value: entity[field].value, error: entity[field].issue }
@@ -127,6 +134,7 @@ export default [
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
   fetchIssuesWithoutReferences,
+  fetchIf(hasEntities, addEntityPageNumberToEntity),
   fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
   formatErrorSummaryParams,
   fetchIf(hasEntities, extractJsonFieldFromEntities),
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index cb1e00fc..25b892f9 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, vi, expect } from 'vitest'
-import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, setPagePageOptions } from '../../../src/middleware/issueTable.middleware.js'
+import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, setPagePageOptions, addEntityPageNumberToEntity } from '../../../src/middleware/issueTable.middleware.js'
 // import { pagination } from '../../../src/utils/pagination.js'
 
 import mocker from '../../utils/mocker.js'
@@ -42,6 +42,29 @@ describe('issueTable.middleware.js', () => {
     })
   })
 
+  describe('addEntityPageNumberToEntity', () => {
+    it('adds entityPageNumber to each entity', () => {
+      const req = {
+        entities: [
+          { id: 1, name: 'Entity 1' },
+          { id: 2, name: 'Entity 2' },
+          { id: 3, name: 'Entity 3' }
+        ]
+      }
+      const res = {}
+      const next = vi.fn()
+
+      addEntityPageNumberToEntity(req, res, next)
+
+      expect(req.entities).toEqual([
+        { id: 1, name: 'Entity 1', entityPageNumber: 1 },
+        { id: 2, name: 'Entity 2', entityPageNumber: 2 },
+        { id: 3, name: 'Entity 3', entityPageNumber: 3 }
+      ])
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+  })
+
   describe('setPagePageOptions', () => {
     it('sets request parameters for pagination', () => {
       const pageLength = 20
@@ -82,8 +105,8 @@ describe('issueTable.middleware.js', () => {
         {
           entry_number: 10,
           'start-date': { value: 'start-date', issue: { message: 'invalid', value: 'invalid-start-date' } },
-          reference: { value: 'reference' }
-
+          reference: { value: 'reference' },
+          entityPageNumber: 1
         }
       ],
       specification: {

From 36f5b7661f8dc9d2ac8869af5b78455e74a13c08 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 13:37:23 +0100
Subject: [PATCH 052/109] fix geometries issue

---
 src/middleware/issueDetails.middleware.js | 2 +-
 src/routes/schemas.js                     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index ac8b20ae..04162283 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -117,7 +117,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   const entry = {
     title: `entry: ${entity.reference.value}`,
     fields,
-    geometries: [entity.geometry.value]
+    geometries: entity.geometry.value
   }
 
   // schema: OrgIssueDetails
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 6748701c..6470904d 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -183,7 +183,7 @@ export const OrgIssueDetails = v.strictObject({
       value: v.strictObject({ html: v.string() }),
       classes: v.string()
     })),
-    geometries: v.optional(v.array(v.string()))
+    geometries: v.optional(v.string())
   }),
   pagination: paginationParams,
   issueEntitiesCount: v.integer(),

From 189a0df03fead09c1638bba3dcd0e3b6e461e6be Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 14:34:06 +0100
Subject: [PATCH 053/109] have the table view sticky

---
 src/assets/scss/index.scss      | 42 +++++++++++++++++++++++++++++++++
 src/views/components/table.html |  8 +++----
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss
index 0ea0697e..126f9b9d 100644
--- a/src/assets/scss/index.scss
+++ b/src/assets/scss/index.scss
@@ -170,3 +170,45 @@ code * {
 .padding-top {
   padding-top: 40px;
 }
+
+.dl-scrollable {
+  max-width: 100%;
+  overflow-x: auto;
+  padding: 0px;
+}
+
+
+.dl-table {
+  overflow-y: auto;
+  overflow-x: hidden;
+  max-height: 80vh;
+  width: 100%;
+  margin-bottom: 0px;
+
+  th, td {
+    padding: govuk-spacing(1) govuk-spacing(2);
+  }
+}
+
+
+.dl-table__head {
+  position: sticky;
+  top: 0;
+  z-index: 1;
+
+  box-shadow: 0 2px 2px -1px rgba(0, 0, 0, 0.4);
+  tr {
+    background-color: govuk-colour("light-grey");
+  }
+}
+
+
+.dl-table__header {
+  border: 1px solid $govuk-border-colour;
+  text-wrap: nowrap;
+  background-color: govuk-colour("light-grey");
+}
+
+.dl-table__cell--error {
+  border-left: govuk-spacing(1) solid $govuk-error-colour;
+}
diff --git a/src/views/components/table.html b/src/views/components/table.html
index 92afd37e..aef223e4 100644
--- a/src/views/components/table.html
+++ b/src/views/components/table.html
@@ -1,12 +1,12 @@
 {% from 'govuk/components/inset-text/macro.njk' import govukInsetText %}
 
 {% macro table(params) %}
-    <div class="app-scrollable-container">
-        <table class="govuk-table">
-            <thead class="govuk-table__head">
+    <div class="app-scrollable-container dl-scrollable">
+        <table class="govuk-table dl-table">
+            <thead class="govuk-table__head dl-table__head">
                 <tr class="govuk-table__row">
                     {% for column in params.columns %}
-                        <th scope="col" class="govuk-table__header">{{ column | prettifyColumnName}}</th>
+                        <th scope="col" class="govuk-table__header dl-table__header">{{ column | prettifyColumnName}}</th>
                     {% endfor %}
                 </tr>
             </thead>

From 31842bdc9f9c97b46d50ada42cda59c1455c5a33 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 14:47:46 +0100
Subject: [PATCH 054/109] fixed tests

---
 test/unit/issueDetailsPage.test.js                   | 4 ++--
 test/unit/middleware/issueDetails.middleware.test.js | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/unit/issueDetailsPage.test.js b/test/unit/issueDetailsPage.test.js
index d963968e..a3655334 100644
--- a/test/unit/issueDetailsPage.test.js
+++ b/test/unit/issueDetailsPage.test.js
@@ -183,7 +183,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
         errorSummary: params.errorSummary,
         entry: {
           ...params.entry,
-          geometries: ['POINT(0 0)']
+          geometries: 'POINT(0 0)'
         },
         issueType: params.issueType,
         issueField: params.issueField
@@ -206,7 +206,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
         errorSummary: params.errorSummary,
         entry: {
           ...params.entry,
-          geometries: []
+          geometries: ''
         },
         issueType: params.issueType,
         issueField: params.issueField
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index de468a4d..7a2afddf 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -107,7 +107,7 @@ describe('issueDetails.middleware.js', () => {
       expect(req.templateParams.entry).toBeDefined()
       expect(req.templateParams.entry.title).toBe('entry: entry-1')
       expect(req.templateParams.entry.fields).toHaveLength(5)
-      expect(req.templateParams.entry.geometries).toEqual(['geom-1'])
+      expect(req.templateParams.entry.geometries).toEqual('geom-1')
       expect(req.templateParams.entry.fields[0].key.text).toBe('reference')
       expect(req.templateParams.entry.fields[0].value.html).toBe('entry-1')
       expect(req.templateParams.entry.fields[0].classes).toBe('')
@@ -166,7 +166,7 @@ describe('issueDetails.middleware.js', () => {
                 }
               }
             ],
-            geometries: ['POINT(0 0)']
+            geometries: 'POINT(0 0)'
           },
           issueType: 'test-issue-type',
           issueField: 'test-issue-field',

From 92b2ff1bf4872f55b08b011c3846ce1ee34d903b Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 14:53:06 +0100
Subject: [PATCH 055/109] add back button to issue details

---
 src/views/organisations/issueDetails.html | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/views/organisations/issueDetails.html b/src/views/organisations/issueDetails.html
index 2d11fa9b..cfd30252 100644
--- a/src/views/organisations/issueDetails.html
+++ b/src/views/organisations/issueDetails.html
@@ -4,6 +4,7 @@
 {% from "govuk/components/summary-list/macro.njk" import govukSummaryList %}
 {% from "govuk/components/pagination/macro.njk" import govukPagination %}
 {% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %}
+{% from "govuk/components/back-link/macro.njk" import govukBackLink %}
 
 {% set serviceType = 'Submit'%}
 
@@ -61,6 +62,11 @@
       errorList: errorSummary.items
     }) }}
 
+    {{ govukBackLink({
+      text: "Back",
+      href: '/organisations/' + organisation.organisation + '/' + dataset.dataset + '/' + issueType + '/' + issueField
+    }) }}
+
     {% if entry.geometries and entry.geometries.length %}
       <div
         id="map"

From 3dc46a74c2fe9c5f8c41c5dd0c6512abf065872f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 15 Oct 2024 16:22:36 +0100
Subject: [PATCH 056/109] pagination change

---
 src/middleware/common.middleware.js       | 3 ++-
 src/middleware/issueDetails.middleware.js | 2 +-
 src/middleware/issueTable.middleware.js   | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index d790d54f..fd2f8864 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -203,7 +203,8 @@ export const getPaginationOptions = (resultsCount) => (req, res, next) => {
  */
 export const createPaginationTemplateParams = (req, res, next) => {
   const { resultsCount, urlSubPath, paginationPageLength } = req
-  const { pageNumber } = req.params
+  let { pageNumber } = req.params
+  pageNumber = parseInt(pageNumber)
 
   const totalPages = Math.floor(resultsCount / paginationPageLength)
 
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 95d102ab..a7765277 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -72,7 +72,7 @@ export const setPagePaginationOptions = (req, res, next) => {
   const { issueEntitiesCount } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
-  req.resultsCount = issueEntitiesCount.length
+  req.resultsCount = issueEntitiesCount
   req.urlSubPath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
   req.paginationPageLength = 1
 
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 1ebf73dd..2174e7fd 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -1,4 +1,5 @@
 import {
+  validateQueryParams,
   addIssuesToEntities,
   extractJsonFieldFromEntities,
   fetchDatasetInfo,
@@ -16,7 +17,6 @@ import {
   pullOutDatasetSpecification,
   replaceUnderscoreWithHyphenForEntities,
   takeResourceIdFromParams,
-  validateQueryParams,
   fetchActiveResourcesForOrganisationAndDataset,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
@@ -37,7 +37,7 @@ export const IssueTableQueryParams = v.object({
   resourceId: v.optional(v.string())
 })
 
-const validateIssueTableQueryParams = validateQueryParams.bind({
+const validateIssueTableQueryParams = validateQueryParams({
   schema: IssueTableQueryParams
 })
 

From fe6c627b016b06e2fb3335451561dd84a5a123cd Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 16 Oct 2024 14:58:10 +0100
Subject: [PATCH 057/109] use dataset-field where possible

---
 src/middleware/common.middleware.js     | 17 +++++++++++++----
 src/middleware/issueTable.middleware.js | 10 +++++-----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index fd2f8864..8ef83b25 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -282,7 +282,14 @@ export const nestEntityFields = (req, res, next) => {
 
   req.entities = entities.map(entity => {
     specification.fields.forEach(field => {
-      entity[field.field] = { value: entity[field.field] }
+      if ('dataset-field' in field) {
+        entity[field['dataset-field']] = { value: entity[field['dataset-field']] }
+      } else if (field.field in entity) {
+        entity[field.field] = { value: entity[field.field] }
+      } else {
+        logger.warn(`Common.middleware:nestEntityFields - ${field.field} has no 'dataset-field' property set in specification`)
+        entity[field.field] = { value: '' }
+      }
     })
     return entity
   })
@@ -291,14 +298,16 @@ export const nestEntityFields = (req, res, next) => {
 }
 
 export const addIssuesToEntities = (req, res, next) => {
-  const { entities, issuesWithReferences } = req
+  const { entities, issuesWithReferences, specification } = req
 
   req.entitiesWithIssues = entities.map(entity => {
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
-      entity[issue.field].value = issue.value
-      entity[issue.field].issue = issue
+      const specificationEntry = specification.fields.find(field => field.field === issue.field)
+      const datasetField = specificationEntry['dataset-field'] || specificationEntry.field
+      entity[datasetField].value = issue.value
+      entity[datasetField].issue = issue
     })
 
     return entity
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 2174e7fd..e564b249 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -80,14 +80,14 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
   const { entities, specification, pagination, errorSummary } = req
 
+  const columnHeaders = [...new Set(specification.fields.map(field => field['dataset-field'] || field.field))]
+
   const tableParams = {
-    columns: specification.fields.map(field => field.field),
-    fields: specification.fields.map(field => field.field),
+    columns: columnHeaders,
+    fields: columnHeaders,
     rows: entities.map((entity, index) => {
       const columns = {}
-
-      specification.fields.forEach(fieldObject => {
-        const { field } = fieldObject
+      columnHeaders.forEach(field => {
         if (field === 'reference') {
           const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${entity.entityPageNumber}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field].value}</a>`, error: entity[field].issue }

From e23a76730b1500beffc85ee1ed13fc8d7561924c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 09:27:09 +0100
Subject: [PATCH 058/109] add special cases for brownfield land

---
 src/middleware/common.middleware.js     | 41 ++++++++++++++++++-------
 src/middleware/issueTable.middleware.js |  6 +++-
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 8ef83b25..6a78e2f0 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -281,15 +281,9 @@ export const nestEntityFields = (req, res, next) => {
   const { entities, specification } = req
 
   req.entities = entities.map(entity => {
-    specification.fields.forEach(field => {
-      if ('dataset-field' in field) {
-        entity[field['dataset-field']] = { value: entity[field['dataset-field']] }
-      } else if (field.field in entity) {
-        entity[field.field] = { value: entity[field.field] }
-      } else {
-        logger.warn(`Common.middleware:nestEntityFields - ${field.field} has no 'dataset-field' property set in specification`)
-        entity[field.field] = { value: '' }
-      }
+    const columnHeaders = [...new Set(specification.fields.map(field => field['dataset-field'] || field.field))]
+    columnHeaders.forEach(field => {
+      entity[field] = { value: entity[field] }
     })
     return entity
   })
@@ -304,8 +298,13 @@ export const addIssuesToEntities = (req, res, next) => {
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
-      const specificationEntry = specification.fields.find(field => field.field === issue.field)
-      const datasetField = specificationEntry['dataset-field'] || specificationEntry.field
+      let datasetField
+      if (issue.field === 'GeoX,GeoY') { // special case for brownfield land
+        datasetField = 'point'
+      } else {
+        const specificationEntry = specification.fields.find(field => field.field === issue.field)
+        datasetField = specificationEntry ? specificationEntry['dataset-field'] : specificationEntry?.field || issue.field
+      }
       entity[datasetField].value = issue.value
       entity[datasetField].issue = issue
     })
@@ -363,3 +362,23 @@ export const fetchIssuesWithoutReferences = fetchMany({
 export function validateQueryParams (context) {
   return validateQueryParamsFn.bind(context)
 }
+
+export const fetchFieldMappings = fetchMany({
+  query: () => 'select * from transform',
+  result: 'fieldMappings'
+})
+
+export const addDatabaseFieldToSpecification = (req, res, next) => {
+  const { specification, fieldMappings } = req
+
+  req.specification.fields = specification.fields.map(fieldObj => {
+    if (['GeoX', 'GeoY'].includes(fieldObj.field)) { // special case for brownfield land
+      return { 'dataset-field': 'Point', ...fieldObj }
+    }
+
+    const databaseField = fieldMappings.find(mapping => mapping.field === fieldObj.field) || fieldObj.field
+    return { 'dataset-field': databaseField.replacement_field, ...fieldObj }
+  })
+
+  next()
+}
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index e564b249..c2a40ee6 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -21,7 +21,9 @@ import {
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
   fetchIssuesWithoutReferences,
-  createPaginationTemplateParams
+  createPaginationTemplateParams,
+  fetchFieldMappings,
+  addDatabaseFieldToSpecification
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -129,6 +131,8 @@ export default [
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
+  fetchFieldMappings,
+  addDatabaseFieldToSpecification,
   getPaginationOptions(paginationPageLength),
   fetchActiveResourcesForOrganisationAndDataset,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,

From 0434788ceb6cb674f870fe02c15a2310b7be0f07 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 09:37:35 +0100
Subject: [PATCH 059/109] pull out dataset_field in issues middleware to own
 middleware

---
 src/middleware/common.middleware.js     | 32 ++++++++++++++++---------
 src/middleware/issueTable.middleware.js |  4 +++-
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 6a78e2f0..72091582 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -291,22 +291,32 @@ export const nestEntityFields = (req, res, next) => {
   next()
 }
 
+export const addDatasetFieldsToIssues = (req, res, next) => {
+  const { issuesWithReferences, specification } = req
+
+  req.issuesWithReferences = issuesWithReferences.map(issue => {
+    let datasetField
+    if (issue.field === 'GeoX,GeoY') { // special case for brownfield land
+      datasetField = 'point'
+    } else {
+      const specificationEntry = specification.fields.find(field => field.field === issue.field)
+      datasetField = specificationEntry ? specificationEntry['dataset-field'] : specificationEntry?.field || issue.field
+    }
+    return { ...issue, datasetField }
+  })
+
+  next()
+}
+
 export const addIssuesToEntities = (req, res, next) => {
-  const { entities, issuesWithReferences, specification } = req
+  const { entities, issuesWithReferences } = req
 
   req.entitiesWithIssues = entities.map(entity => {
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
-      let datasetField
-      if (issue.field === 'GeoX,GeoY') { // special case for brownfield land
-        datasetField = 'point'
-      } else {
-        const specificationEntry = specification.fields.find(field => field.field === issue.field)
-        datasetField = specificationEntry ? specificationEntry['dataset-field'] : specificationEntry?.field || issue.field
-      }
-      entity[datasetField].value = issue.value
-      entity[datasetField].issue = issue
+      entity[issue.datasetField].value = issue.value
+      entity[issue.datasetField].issue = issue
     })
 
     return entity
@@ -373,7 +383,7 @@ export const addDatabaseFieldToSpecification = (req, res, next) => {
 
   req.specification.fields = specification.fields.map(fieldObj => {
     if (['GeoX', 'GeoY'].includes(fieldObj.field)) { // special case for brownfield land
-      return { 'dataset-field': 'Point', ...fieldObj }
+      return { 'dataset-field': 'point', ...fieldObj }
     }
 
     const databaseField = fieldMappings.find(mapping => mapping.field === fieldObj.field) || fieldObj.field
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index c2a40ee6..f3e64613 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -23,7 +23,8 @@ import {
   fetchIssuesWithoutReferences,
   createPaginationTemplateParams,
   fetchFieldMappings,
-  addDatabaseFieldToSpecification
+  addDatabaseFieldToSpecification,
+  addDatasetFieldsToIssues
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -138,6 +139,7 @@ export default [
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
   fetchIssuesWithoutReferences,
+  addDatasetFieldsToIssues,
   fetchIf(hasEntities, addEntityPageNumberToEntity),
   fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
   formatErrorSummaryParams,

From 1be8653274a5408f30fa46d679bdf51e7c879ca0 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 09:59:07 +0100
Subject: [PATCH 060/109] catch no field mapping found

---
 src/middleware/common.middleware.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 72091582..06e67d27 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -386,8 +386,9 @@ export const addDatabaseFieldToSpecification = (req, res, next) => {
       return { 'dataset-field': 'point', ...fieldObj }
     }
 
-    const databaseField = fieldMappings.find(mapping => mapping.field === fieldObj.field) || fieldObj.field
-    return { 'dataset-field': databaseField.replacement_field, ...fieldObj }
+    const fieldMapping = fieldMappings.find(mapping => mapping.field === fieldObj.field)
+    const databaseField = fieldMapping?.replacement_field || fieldObj.field
+    return { 'dataset-field': databaseField, ...fieldObj }
   })
 
   next()

From 565dc14b3556ff7ec28e74eac46e3d7255412a09 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 09:59:36 +0100
Subject: [PATCH 061/109] pass through entity value for issue if no issue value
 is present

---
 src/middleware/common.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 06e67d27..de11645f 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -315,7 +315,7 @@ export const addIssuesToEntities = (req, res, next) => {
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
-      entity[issue.datasetField].value = issue.value
+      entity[issue.datasetField].value = issue.value || entity[issue.datasetField].value
       entity[issue.datasetField].issue = issue
     })
 

From 965b7715db08c517eb1a641569a8ba778372bb80 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 10:00:38 +0100
Subject: [PATCH 062/109] set default value to empty if none is provided

---
 src/middleware/common.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index de11645f..555f3d0e 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -315,7 +315,7 @@ export const addIssuesToEntities = (req, res, next) => {
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
-      entity[issue.datasetField].value = issue.value || entity[issue.datasetField].value
+      entity[issue.datasetField].value = issue.value || entity[issue.datasetField].value || ''
       entity[issue.datasetField].issue = issue
     })
 

From 4222fed95c7886c5bcbdacdeea1a9c3a5ef10a36 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 10:03:01 +0100
Subject: [PATCH 063/109] add some more tests

---
 .../unit/middleware/common.middleware.test.js | 202 +++++++++++++++++-
 1 file changed, 194 insertions(+), 8 deletions(-)

diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 1527fc07..cd7444e8 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, expect, vi } from 'vitest'
-import { addIssuesToEntities, createPaginationTemplateParams, extractJsonFieldFromEntities, formatErrorSummaryParams, getPaginationOptions, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, nestEntityFields, paginateEntitiesAndPullOutCount, pullOutDatasetSpecification, replaceUnderscoreWithHyphenForEntities, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
+import { addDatabaseFieldToSpecification, addDatasetFieldsToIssues, addIssuesToEntities, createPaginationTemplateParams, extractJsonFieldFromEntities, formatErrorSummaryParams, getPaginationOptions, isResourceAccessible, isResourceIdNotInParams, isResourceNotAccessible, logPageError, nestEntityFields, paginateEntitiesAndPullOutCount, pullOutDatasetSpecification, replaceUnderscoreWithHyphenForEntities, takeResourceIdFromParams } from '../../../src/middleware/common.middleware'
 import logger from '../../../src/utils/logger'
 
 vi.mock('../../../src/utils/logger')
@@ -439,6 +439,99 @@ describe('nestEntityFields', () => {
   })
 })
 
+describe('addDatasetFieldsToIssues', () => {
+  it('adds dataset field to issues', () => {
+    const req = {
+      issuesWithReferences: [
+        { entryNumber: 1, field: 'name', value: 'nameIssueValue' },
+        { entryNumber: 2, field: 'age', value: 'ageIssueValue' }
+      ],
+      specification: {
+        fields: [
+          { field: 'name', 'dataset-field': 'fullName' },
+          { field: 'age', 'dataset-field': 'Age' }
+        ]
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatasetFieldsToIssues(req, res, next)
+    expect(req.issuesWithReferences).toHaveLength(2)
+    expect(req.issuesWithReferences[0]).toEqual({
+      entryNumber: 1,
+      field: 'name',
+      value: 'nameIssueValue',
+      datasetField: 'fullName'
+    })
+    expect(req.issuesWithReferences[1]).toEqual({
+      entryNumber: 2,
+      field: 'age',
+      value: 'ageIssueValue',
+      datasetField: 'Age'
+    })
+  })
+
+  it('handles special case for GeoX,GeoY field', () => {
+    const req = {
+      issuesWithReferences: [
+        { entryNumber: 1, field: 'GeoX,GeoY', value: ' GeoX,GeoY issue value' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatasetFieldsToIssues(req, res, next)
+    expect(req.issuesWithReferences).toHaveLength(1)
+    expect(req.issuesWithReferences[0]).toEqual({
+      entryNumber: 1,
+      field: 'GeoX,GeoY',
+      value: ' GeoX,GeoY issue value',
+      datasetField: 'point'
+    })
+  })
+
+  it('handles issues with no matching specification field', () => {
+    const req = {
+      issuesWithReferences: [
+        { entryNumber: 1, field: 'unknownField', value: 'unknownFieldValue' }
+      ],
+      specification: {
+        fields: []
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatasetFieldsToIssues(req, res, next)
+    expect(req.issuesWithReferences).toHaveLength(1)
+    expect(req.issuesWithReferences[0]).toEqual({
+      entryNumber: 1,
+      field: 'unknownField',
+      value: 'unknownFieldValue',
+      datasetField: 'unknownField'
+    })
+  })
+
+  it('calls next function', () => {
+    const req = {
+      issuesWithReferences: [
+        { entryNumber: 1, field: 'name', value: 'nameIssueValue' }
+      ],
+      specification: {
+        fields: [
+          { field: 'name', 'dataset-field': 'fullName' }
+        ]
+      }
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatasetFieldsToIssues(req, res, next)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+})
+
 describe('addIssuesToEntities', () => {
   it('adds issues to entities correctly', () => {
     const req = {
@@ -447,9 +540,9 @@ describe('addIssuesToEntities', () => {
         { entryNumber: 2, name: { value: 'Jane' }, age: { value: 25 } }
       ],
       issuesWithReferences: [
-        { entryNumber: 1, field: 'name', value: 'nameIssueValue' },
-        { entryNumber: 1, field: 'age', value: 'ageIssueValue' },
-        { entryNumber: 2, field: 'name', value: 'nameIssueValue2' }
+        { entryNumber: 1, field: 'name', datasetField: 'name', value: 'nameIssueValue' },
+        { entryNumber: 1, field: 'age', datasetField: 'age', value: 'ageIssueValue' },
+        { entryNumber: 2, field: 'name', datasetField: 'name', value: 'nameIssueValue2' }
       ]
     }
     const res = {}
@@ -460,12 +553,12 @@ describe('addIssuesToEntities', () => {
     expect(req.entitiesWithIssues).toHaveLength(2)
     expect(req.entitiesWithIssues[0]).toEqual({
       entryNumber: 1,
-      name: { value: 'nameIssueValue', issue: { entryNumber: 1, field: 'name', value: 'nameIssueValue' } },
-      age: { value: 'ageIssueValue', issue: { entryNumber: 1, field: 'age', value: 'ageIssueValue' } }
+      name: { value: 'nameIssueValue', issue: { entryNumber: 1, field: 'name', datasetField: 'name', value: 'nameIssueValue' } },
+      age: { value: 'ageIssueValue', issue: { entryNumber: 1, field: 'age', datasetField: 'age', value: 'ageIssueValue' } }
     })
     expect(req.entitiesWithIssues[1]).toEqual({
       entryNumber: 2,
-      name: { value: 'nameIssueValue2', issue: { entryNumber: 2, field: 'name', value: 'nameIssueValue2' } },
+      name: { value: 'nameIssueValue2', issue: { entryNumber: 2, field: 'name', datasetField: 'name', value: 'nameIssueValue2' } },
       age: { value: 25 }
     })
   })
@@ -476,7 +569,7 @@ describe('addIssuesToEntities', () => {
         { entryNumber: 1, name: { value: 'John' } }
       ],
       issuesWithReferences: [
-        { entryNumber: 1, field: 'name', value: 'Invalid name' }
+        { entryNumber: 1, field: 'name', value: 'Invalid name', datasetField: 'name' }
       ]
     }
     const res = {}
@@ -504,3 +597,96 @@ describe('addIssuesToEntities', () => {
     })
   })
 })
+
+describe('addDatabaseFieldToSpecification', () => {
+  it('adds database field to specification fields', () => {
+    const req = {
+      specification: {
+        fields: [
+          { field: 'name' },
+          { field: 'address' }
+        ]
+      },
+      fieldMappings: [
+        { field: 'name', replacement_field: 'full_name' },
+        { field: 'address', replacement_field: 'physical_address' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatabaseFieldToSpecification(req, res, next)
+    expect(req.specification.fields).toHaveLength(2)
+    expect(req.specification.fields[0]).toEqual({
+      field: 'name',
+      'dataset-field': 'full_name'
+    })
+    expect(req.specification.fields[1]).toEqual({
+      field: 'address',
+      'dataset-field': 'physical_address'
+    })
+  })
+
+  it('handles special case for GeoX and GeoY fields', () => {
+    const req = {
+      specification: {
+        fields: [
+          { field: 'GeoX' },
+          { field: 'GeoY' }
+        ]
+      },
+      fieldMappings: []
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatabaseFieldToSpecification(req, res, next)
+    expect(req.specification.fields).toHaveLength(2)
+    expect(req.specification.fields[0]).toEqual({
+      field: 'GeoX',
+      'dataset-field': 'point'
+    })
+    expect(req.specification.fields[1]).toEqual({
+      field: 'GeoY',
+      'dataset-field': 'point'
+    })
+  })
+
+  it('handles fields with no matching field mapping', () => {
+    const req = {
+      specification: {
+        fields: [
+          { field: 'unknownField' }
+        ]
+      },
+      fieldMappings: []
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatabaseFieldToSpecification(req, res, next)
+    expect(req.specification.fields).toHaveLength(1)
+    expect(req.specification.fields[0]).toEqual({
+      field: 'unknownField',
+      'dataset-field': 'unknownField'
+    })
+  })
+
+  it('calls next function', () => {
+    const req = {
+      specification: {
+        fields: [
+          { field: 'name' }
+        ]
+      },
+      fieldMappings: [
+        { field: 'name', replacement_field: 'full_name' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    addDatabaseFieldToSpecification(req, res, next)
+    expect(next).toHaveBeenCalledTimes(1)
+  })
+})

From 0c0fc1a9f6b6f2f062ba690ddff70b2a80103eb3 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 11:23:49 +0100
Subject: [PATCH 064/109] improved page number valibot validation

---
 src/middleware/common.middleware.js       |  4 +---
 src/middleware/issueDetails.middleware.js | 12 +++++++++---
 src/middleware/issueTable.middleware.js   |  4 +---
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 555f3d0e..b3097173 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -178,9 +178,7 @@ export const paginateEntitiesAndPullOutCount = (req, res, next) => {
 }
 
 export const getPaginationOptions = (resultsCount) => (req, res, next) => {
-  let { pageNumber } = req.params
-
-  pageNumber = pageNumber || 1
+  const { pageNumber } = req.params
 
   req.pagination = { offset: (pageNumber - 1) * resultsCount, limit: resultsCount }
 
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index a7765277..2f9c1ea9 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -30,7 +30,7 @@ export const IssueDetailsQueryParams = v.strictObject({
   dataset: v.string(),
   issue_type: v.string(),
   issue_field: v.string(),
-  pageNumber: v.string(),
+  pageNumber: v.pipe(v.string(), v.transform(parseInt), v.number(), v.integer(), v.minValue(1)),
   resourceId: v.optional(v.string())
 })
 
@@ -93,8 +93,14 @@ export const setPagePaginationOptions = (req, res, next) => {
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
   const { entities, issueEntitiesCount, errorSummary, specification, pagination } = req
-  const { issue_type: issueType, issue_field: issueField, pageNumber: pageNumberString } = req.params
-  const pageNumber = parseInt(pageNumberString)
+  const { issue_type: issueType, issue_field: issueField, pageNumber } = req.params
+
+  if (pageNumber > entities.length) {
+    const error = new Error('pageNumber out of bounds')
+    error.status = 400
+    next(error)
+    return
+  }
 
   const entity = entities[pageNumber - 1]
 
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index f3e64613..ec6d041d 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -36,7 +36,7 @@ export const IssueTableQueryParams = v.object({
   dataset: v.string(),
   issue_type: v.string(),
   issue_field: v.string(),
-  pageNumber: v.optional(v.string()),
+  pageNumber: v.optional(v.pipe(v.string(), v.transform(parseInt), v.number(), v.integer(), v.minValue(1))),
   resourceId: v.optional(v.string())
 })
 
@@ -47,8 +47,6 @@ const validateIssueTableQueryParams = validateQueryParams({
 export const setDefaultQueryParams = (req, res, next) => {
   if (!req.params.pageNumber) {
     req.params.pageNumber = 1
-  } else {
-    req.params.pageNumber = parseInt(req.params.pageNumber)
   }
   next()
 }

From 356de1f700094babae8ca40ebc84450d116d82fe Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 11:24:12 +0100
Subject: [PATCH 065/109] change database-field to databaseField

---
 src/middleware/common.middleware.js           |  8 ++---
 src/middleware/issueDetails.middleware.js     | 22 +++++++++-----
 src/middleware/issueTable.middleware.js       |  2 +-
 .../unit/middleware/common.middleware.test.js | 16 +++++-----
 .../middleware/issueTable.middleware.test.js  | 29 +------------------
 5 files changed, 28 insertions(+), 49 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index b3097173..85521384 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -279,7 +279,7 @@ export const nestEntityFields = (req, res, next) => {
   const { entities, specification } = req
 
   req.entities = entities.map(entity => {
-    const columnHeaders = [...new Set(specification.fields.map(field => field['dataset-field'] || field.field))]
+    const columnHeaders = [...new Set(specification.fields.map(field => field.datasetField || field.field))]
     columnHeaders.forEach(field => {
       entity[field] = { value: entity[field] }
     })
@@ -298,7 +298,7 @@ export const addDatasetFieldsToIssues = (req, res, next) => {
       datasetField = 'point'
     } else {
       const specificationEntry = specification.fields.find(field => field.field === issue.field)
-      datasetField = specificationEntry ? specificationEntry['dataset-field'] : specificationEntry?.field || issue.field
+      datasetField = specificationEntry ? specificationEntry.datasetField : specificationEntry?.field || issue.field
     }
     return { ...issue, datasetField }
   })
@@ -381,12 +381,12 @@ export const addDatabaseFieldToSpecification = (req, res, next) => {
 
   req.specification.fields = specification.fields.map(fieldObj => {
     if (['GeoX', 'GeoY'].includes(fieldObj.field)) { // special case for brownfield land
-      return { 'dataset-field': 'point', ...fieldObj }
+      return { datasetField: 'point', ...fieldObj }
     }
 
     const fieldMapping = fieldMappings.find(mapping => mapping.field === fieldObj.field)
     const databaseField = fieldMapping?.replacement_field || fieldObj.field
-    return { 'dataset-field': databaseField, ...fieldObj }
+    return { datasetField: databaseField, ...fieldObj }
   })
 
   next()
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 2f9c1ea9..6a648ec7 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,4 +1,6 @@
 import {
+  addDatabaseFieldToSpecification,
+  addDatasetFieldsToIssues,
   addIssuesToEntities,
   createPaginationTemplateParams,
   extractJsonFieldFromEntities,
@@ -6,6 +8,7 @@ import {
   fetchDatasetInfo,
   fetchEntitiesFromIssuesWithReferences,
   fetchEntityCount,
+  fetchFieldMappings,
   fetchIssueEntitiesCount,
   fetchIssuesWithoutReferences,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
@@ -104,20 +107,20 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
 
   const entity = entities[pageNumber - 1]
 
-  const fields = specification.fields.map(({ field }) => {
+  const fields = specification.fields.map(({ datasetField }) => {
     let valueHtml = ''
     let classes = ''
-    if (!entity[field]) {
-      entity[field] = {
+    if (!entity[datasetField]) {
+      entity[datasetField] = {
         value: ''
       }
     }
-    if (entity[field].issue) {
-      valueHtml += issueErrorMessageHtml(entity[field].issue.message, null)
+    if (entity[datasetField].issue) {
+      valueHtml += issueErrorMessageHtml(entity[datasetField].issue.message, null)
       classes += 'dl-summary-card-list__row--error'
     }
-    valueHtml += entity[field].value || ''
-    return getIssueField(field, valueHtml, classes)
+    valueHtml += entity[datasetField].value || ''
+    return getIssueField(datasetField, valueHtml, classes)
   })
 
   const entry = {
@@ -158,10 +161,13 @@ export default [
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
+  fetchFieldMappings,
+  addDatabaseFieldToSpecification,
   fetchActiveResourcesForOrganisationAndDataset,
-  fetchIssuesWithoutReferences,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
   fetchEntitiesFromIssuesWithReferences,
+  fetchIssuesWithoutReferences,
+  addDatasetFieldsToIssues,
   fetchIf(hasEntities, extractJsonFieldFromEntities),
   fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities),
   fetchIf(hasEntities, nestEntityFields),
diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index ec6d041d..406270f5 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -81,7 +81,7 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
   const { entities, specification, pagination, errorSummary } = req
 
-  const columnHeaders = [...new Set(specification.fields.map(field => field['dataset-field'] || field.field))]
+  const columnHeaders = [...new Set(specification.fields.map(field => field.datasetField || field.field))]
 
   const tableParams = {
     columns: columnHeaders,
diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index cd7444e8..06c88454 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -448,8 +448,8 @@ describe('addDatasetFieldsToIssues', () => {
       ],
       specification: {
         fields: [
-          { field: 'name', 'dataset-field': 'fullName' },
-          { field: 'age', 'dataset-field': 'Age' }
+          { field: 'name', datasetField: 'fullName' },
+          { field: 'age', datasetField: 'Age' }
         ]
       }
     }
@@ -520,7 +520,7 @@ describe('addDatasetFieldsToIssues', () => {
       ],
       specification: {
         fields: [
-          { field: 'name', 'dataset-field': 'fullName' }
+          { field: 'name', datasetField: 'fullName' }
         ]
       }
     }
@@ -619,11 +619,11 @@ describe('addDatabaseFieldToSpecification', () => {
     expect(req.specification.fields).toHaveLength(2)
     expect(req.specification.fields[0]).toEqual({
       field: 'name',
-      'dataset-field': 'full_name'
+      datasetField: 'full_name'
     })
     expect(req.specification.fields[1]).toEqual({
       field: 'address',
-      'dataset-field': 'physical_address'
+      datasetField: 'physical_address'
     })
   })
 
@@ -644,11 +644,11 @@ describe('addDatabaseFieldToSpecification', () => {
     expect(req.specification.fields).toHaveLength(2)
     expect(req.specification.fields[0]).toEqual({
       field: 'GeoX',
-      'dataset-field': 'point'
+      datasetField: 'point'
     })
     expect(req.specification.fields[1]).toEqual({
       field: 'GeoY',
-      'dataset-field': 'point'
+      datasetField: 'point'
     })
   })
 
@@ -668,7 +668,7 @@ describe('addDatabaseFieldToSpecification', () => {
     expect(req.specification.fields).toHaveLength(1)
     expect(req.specification.fields[0]).toEqual({
       field: 'unknownField',
-      'dataset-field': 'unknownField'
+      datasetField: 'unknownField'
     })
   })
 
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 25b892f9..d31b9c94 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, vi, expect } from 'vitest'
-import { prepareIssueTableTemplateParams, IssueTableQueryParams, setDefaultQueryParams, setPagePageOptions, addEntityPageNumberToEntity } from '../../../src/middleware/issueTable.middleware.js'
+import { prepareIssueTableTemplateParams, IssueTableQueryParams, setPagePageOptions, addEntityPageNumberToEntity } from '../../../src/middleware/issueTable.middleware.js'
 // import { pagination } from '../../../src/utils/pagination.js'
 
 import mocker from '../../utils/mocker.js'
@@ -15,33 +15,6 @@ vi.mock('../../../src/utils/pagination.js', () => {
 })
 
 describe('issueTable.middleware.js', () => {
-  describe('setDefaultQueryParams', () => {
-    it('sets the page number when none is set', () => {
-      const req = {
-        params: {}
-      }
-      const next = vi.fn()
-
-      setDefaultQueryParams(req, {}, next)
-
-      expect(req.params.pageNumber).toEqual(1)
-      expect(next).toHaveBeenCalledOnce()
-    })
-
-    it('sets does not change the page number when one is set', () => {
-      const req = {
-        params: {
-          pageNumber: 2
-        }
-      }
-      const next = vi.fn()
-
-      setDefaultQueryParams(req, {}, next)
-      expect(req.params.pageNumber).toEqual(2)
-      expect(next).toHaveBeenCalledOnce()
-    })
-  })
-
   describe('addEntityPageNumberToEntity', () => {
     it('adds entityPageNumber to each entity', () => {
       const req = {

From 8e9f85a7e30b356e37ddab84ec0327a8e09055f4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 12:51:51 +0100
Subject: [PATCH 066/109] remove commented method

---
 src/middleware/datasetTaskList.middleware.js | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index d81341eb..3b86fc3f 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -105,13 +105,6 @@ export const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
   next()
 }
 
-// ToDo: do we need to add this back in to the middleware chain?
-// const getDatasetTaskListError = renderTemplate({
-//   templateParams: (req) => req.templateParams,
-//   template: 'organisations/http-error.html',
-//   handlerName: 'getDatasetTaskListError'
-// })
-
 const validateParams = validateQueryParams({
   schema: v.object({
     lpa: v.string(),

From fe7f7eb82c27e0b9a68a167153d7c3475f82ca66 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 12:59:31 +0100
Subject: [PATCH 067/109] remove unneeded test

---
 test/unit/middleware/common.middleware.test.js | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 06c88454..4be75e48 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -155,19 +155,6 @@ describe('pagination', () => {
       getPaginationOptionsMiddleware(req, res, next)
       expect(next).toHaveBeenCalledTimes(1)
     })
-
-    it('handles default pageNumber as 1', () => {
-      const resultsCount = 10
-      const getPaginationOptionsMiddleware = getPaginationOptions(resultsCount)
-      const req = { params: {} }
-      const res = {}
-      const next = vi.fn()
-
-      getPaginationOptionsMiddleware(req, res, next)
-      expect(req.pagination).toBeDefined()
-      expect(req.pagination.offset).toBe(0)
-      expect(req.pagination.limit).toBe(10)
-    })
   })
 
   describe('paginateEntitiesAndPullOutCount', () => {

From 8424fadd657302571782c697ad1dadf1006c8d55 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 13:01:09 +0100
Subject: [PATCH 068/109] fix test

---
 test/unit/middleware/issueDetails.middleware.test.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 7a2afddf..2adfaa3d 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -64,11 +64,11 @@ describe('issueDetails.middleware.js', () => {
       errorSummary: 'Mock error summary',
       specification: {
         fields: [
-          { field: 'reference', label: 'Reference' },
-          { field: 'geometry', label: 'Geometry' },
-          { field: 'field1', label: 'Field 1' },
-          { field: 'field2', label: 'Field 2' },
-          { field: 'field3', label: 'Field 3' }
+          { field: 'reference', datasetField: 'reference', label: 'Reference' },
+          { field: 'geometry', datasetField: 'geometry', label: 'Geometry' },
+          { field: 'field1', datasetField: 'field1', label: 'Field 1' },
+          { field: 'field2', datasetField: 'field2', label: 'Field 2' },
+          { field: 'field3', datasetField: 'field3', label: 'Field 3' }
         ]
       },
       params: {

From 5453c8cba6f48ec885edd4fda30db8047423309b Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 13:30:24 +0100
Subject: [PATCH 069/109] exclude built javascript code from coverage

---
 package-lock.json | 128 +++++++++++++++++++---------------------------
 package.json      |   2 +-
 vite.config.js    |   5 +-
 3 files changed, 58 insertions(+), 77 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index f703815f..cd5bd938 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -46,7 +46,7 @@
         "@playwright/test": "^1.39.0",
         "@testcontainers/localstack": "^10.7.2",
         "@types/node": "^20.8.9",
-        "@vitest/coverage-v8": "^2.1.2",
+        "@vitest/coverage-v8": "^2.1.3",
         "@wiremock/wiremock-testcontainers-node": "^0.0.1",
         "concurrently": "^8.2.2",
         "husky": "^9.0.11",
@@ -3361,11 +3361,10 @@
       "license": "ISC"
     },
     "node_modules/@vitest/coverage-v8": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-2.1.2.tgz",
-      "integrity": "sha512-b7kHrFrs2urS0cOk5N10lttI8UdJ/yP3nB4JYTREvR5o18cR99yPpK4gK8oQgI42BVv0ILWYUSYB7AXkAUDc0g==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-2.1.3.tgz",
+      "integrity": "sha512-2OJ3c7UPoFSmBZwqD2VEkUw6A/tzPF0LmW0ZZhhB8PFxuc+9IBG/FaSM+RLEenc7ljzFvGN+G0nGQoZnh7sy2A==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "@ampproject/remapping": "^2.3.0",
         "@bcoe/v8-coverage": "^0.2.3",
@@ -3384,8 +3383,8 @@
         "url": "https://opencollective.com/vitest"
       },
       "peerDependencies": {
-        "@vitest/browser": "2.1.2",
-        "vitest": "2.1.2"
+        "@vitest/browser": "2.1.3",
+        "vitest": "2.1.3"
       },
       "peerDependenciesMeta": {
         "@vitest/browser": {
@@ -3419,14 +3418,13 @@
       "license": "MIT"
     },
     "node_modules/@vitest/expect": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.2.tgz",
-      "integrity": "sha512-FEgtlN8mIUSEAAnlvn7mP8vzaWhEaAEvhSXCqrsijM7K6QqjB11qoRZYEd4AKSCDz8p0/+yH5LzhZ47qt+EyPg==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.3.tgz",
+      "integrity": "sha512-SNBoPubeCJhZ48agjXruCI57DvxcsivVDdWz+SSsmjTT4QN/DfHk3zB/xKsJqMs26bLZ/pNRLnCf0j679i0uWQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "@vitest/spy": "2.1.2",
-        "@vitest/utils": "2.1.2",
+        "@vitest/spy": "2.1.3",
+        "@vitest/utils": "2.1.3",
         "chai": "^5.1.1",
         "tinyrainbow": "^1.2.0"
       },
@@ -3435,13 +3433,12 @@
       }
     },
     "node_modules/@vitest/mocker": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.2.tgz",
-      "integrity": "sha512-ExElkCGMS13JAJy+812fw1aCv2QO/LBK6CyO4WOPAzLTmve50gydOlWhgdBJPx2ztbADUq3JVI0C5U+bShaeEA==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.3.tgz",
+      "integrity": "sha512-eSpdY/eJDuOvuTA3ASzCjdithHa+GIF1L4PqtEELl6Qa3XafdMLBpBlZCIUCX2J+Q6sNmjmxtosAG62fK4BlqQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "@vitest/spy": "^2.1.0-beta.1",
+        "@vitest/spy": "2.1.3",
         "estree-walker": "^3.0.3",
         "magic-string": "^0.30.11"
       },
@@ -3449,7 +3446,7 @@
         "url": "https://opencollective.com/vitest"
       },
       "peerDependencies": {
-        "@vitest/spy": "2.1.2",
+        "@vitest/spy": "2.1.3",
         "msw": "^2.3.5",
         "vite": "^5.0.0"
       },
@@ -3463,11 +3460,10 @@
       }
     },
     "node_modules/@vitest/pretty-format": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.2.tgz",
-      "integrity": "sha512-FIoglbHrSUlOJPDGIrh2bjX1sNars5HbxlcsFKCtKzu4+5lpsRhOCVcuzp0fEhAGHkPZRIXVNzPcpSlkoZ3LuA==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.3.tgz",
+      "integrity": "sha512-XH1XdtoLZCpqV59KRbPrIhFCOO0hErxrQCMcvnQete3Vibb9UeIOX02uFPfVn3Z9ZXsq78etlfyhnkmIZSzIwQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "tinyrainbow": "^1.2.0"
       },
@@ -3476,13 +3472,12 @@
       }
     },
     "node_modules/@vitest/runner": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.2.tgz",
-      "integrity": "sha512-UCsPtvluHO3u7jdoONGjOSil+uON5SSvU9buQh3lP7GgUXHp78guN1wRmZDX4wGK6J10f9NUtP6pO+SFquoMlw==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.3.tgz",
+      "integrity": "sha512-JGzpWqmFJ4fq5ZKHtVO3Xuy1iF2rHGV4d/pdzgkYHm1+gOzNZtqjvyiaDGJytRyMU54qkxpNzCx+PErzJ1/JqQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "@vitest/utils": "2.1.2",
+        "@vitest/utils": "2.1.3",
         "pathe": "^1.1.2"
       },
       "funding": {
@@ -3490,13 +3485,12 @@
       }
     },
     "node_modules/@vitest/snapshot": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.2.tgz",
-      "integrity": "sha512-xtAeNsZ++aRIYIUsek7VHzry/9AcxeULlegBvsdLncLmNCR6tR8SRjn8BbDP4naxtccvzTqZ+L1ltZlRCfBZFA==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.3.tgz",
+      "integrity": "sha512-qWC2mWc7VAXmjAkEKxrScWHWFyCQx/cmiZtuGqMi+WwqQJ2iURsVY4ZfAK6dVo6K2smKRU6l3BPwqEBvhnpQGg==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "@vitest/pretty-format": "2.1.2",
+        "@vitest/pretty-format": "2.1.3",
         "magic-string": "^0.30.11",
         "pathe": "^1.1.2"
       },
@@ -3505,11 +3499,10 @@
       }
     },
     "node_modules/@vitest/spy": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.2.tgz",
-      "integrity": "sha512-GSUi5zoy+abNRJwmFhBDC0yRuVUn8WMlQscvnbbXdKLXX9dE59YbfwXxuJ/mth6eeqIzofU8BB5XDo/Ns/qK2A==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.3.tgz",
+      "integrity": "sha512-Nb2UzbcUswzeSP7JksMDaqsI43Sj5+Kry6ry6jQJT4b5gAK+NS9NED6mDb8FlMRCX8m5guaHCDZmqYMMWRy5nQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "tinyspy": "^3.0.0"
       },
@@ -3518,13 +3511,12 @@
       }
     },
     "node_modules/@vitest/utils": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.2.tgz",
-      "integrity": "sha512-zMO2KdYy6mx56btx9JvAqAZ6EyS3g49krMPPrgOp1yxGZiA93HumGk+bZ5jIZtOg5/VBYl5eBmGRQHqq4FG6uQ==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.3.tgz",
+      "integrity": "sha512-xpiVfDSg1RrYT0tX6czgerkpcKFmFOF/gCr30+Mve5V2kewCy4Prn1/NDMSRwaSmT7PRaOF83wu+bEtsY1wrvA==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "@vitest/pretty-format": "2.1.2",
+        "@vitest/pretty-format": "2.1.3",
         "loupe": "^3.1.1",
         "tinyrainbow": "^1.2.0"
       },
@@ -4297,7 +4289,6 @@
       "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz",
       "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">=12"
       }
@@ -4806,7 +4797,6 @@
       "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz",
       "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">=8"
       }
@@ -4892,7 +4882,6 @@
       "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.1.tgz",
       "integrity": "sha512-pT1ZgP8rPNqUgieVaEY+ryQr6Q4HXNg8Ei9UnLUrjN4IA7dvQC5JB+/kxVcPNDHyBcc/26CXPkbNzq3qwrOEKA==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "assertion-error": "^2.0.1",
         "check-error": "^2.1.1",
@@ -4937,7 +4926,6 @@
       "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz",
       "integrity": "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">= 16"
       }
@@ -5633,7 +5621,6 @@
       "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz",
       "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">=6"
       }
@@ -7037,7 +7024,6 @@
       "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz",
       "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "@types/estree": "^1.0.0"
       }
@@ -9668,8 +9654,7 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz",
       "integrity": "sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==",
-      "dev": true,
-      "license": "MIT"
+      "dev": true
     },
     "node_modules/lowercase-keys": {
       "version": "2.0.0",
@@ -10733,15 +10718,13 @@
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz",
       "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==",
-      "dev": true,
-      "license": "MIT"
+      "dev": true
     },
     "node_modules/pathval": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.0.tgz",
       "integrity": "sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">= 14.16"
       }
@@ -13291,7 +13274,6 @@
       "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz",
       "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==",
       "dev": true,
-      "license": "MIT",
       "engines": {
         "node": ">=14.0.0"
       }
@@ -13851,11 +13833,10 @@
       }
     },
     "node_modules/vite-node": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.2.tgz",
-      "integrity": "sha512-HPcGNN5g/7I2OtPjLqgOtCRu/qhVvBxTUD3qzitmL0SrG1cWFzxzhMDWussxSbrRYWqnKf8P2jiNhPMSN+ymsQ==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.3.tgz",
+      "integrity": "sha512-I1JadzO+xYX887S39Do+paRePCKoiDrWRRjp9kkG5he0t7RXNvPAJPCQSJqbGN4uCrFFeS3Kj3sLqY8NMYBEdA==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "cac": "^6.7.14",
         "debug": "^4.3.6",
@@ -13877,7 +13858,6 @@
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
       "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "ms": "^2.1.3"
       },
@@ -13894,23 +13874,21 @@
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
-      "dev": true,
-      "license": "MIT"
+      "dev": true
     },
     "node_modules/vitest": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.2.tgz",
-      "integrity": "sha512-veNjLizOMkRrJ6xxb+pvxN6/QAWg95mzcRjtmkepXdN87FNfxAss9RKe2far/G9cQpipfgP2taqg0KiWsquj8A==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.3.tgz",
+      "integrity": "sha512-Zrxbg/WiIvUP2uEzelDNTXmEMJXuzJ1kCpbDvaKByFA9MNeO95V+7r/3ti0qzJzrxdyuUw5VduN7k+D3VmVOSA==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
-        "@vitest/expect": "2.1.2",
-        "@vitest/mocker": "2.1.2",
-        "@vitest/pretty-format": "^2.1.2",
-        "@vitest/runner": "2.1.2",
-        "@vitest/snapshot": "2.1.2",
-        "@vitest/spy": "2.1.2",
-        "@vitest/utils": "2.1.2",
+        "@vitest/expect": "2.1.3",
+        "@vitest/mocker": "2.1.3",
+        "@vitest/pretty-format": "^2.1.3",
+        "@vitest/runner": "2.1.3",
+        "@vitest/snapshot": "2.1.3",
+        "@vitest/spy": "2.1.3",
+        "@vitest/utils": "2.1.3",
         "chai": "^5.1.1",
         "debug": "^4.3.6",
         "magic-string": "^0.30.11",
@@ -13921,7 +13899,7 @@
         "tinypool": "^1.0.0",
         "tinyrainbow": "^1.2.0",
         "vite": "^5.0.0",
-        "vite-node": "2.1.2",
+        "vite-node": "2.1.3",
         "why-is-node-running": "^2.3.0"
       },
       "bin": {
@@ -13936,8 +13914,8 @@
       "peerDependencies": {
         "@edge-runtime/vm": "*",
         "@types/node": "^18.0.0 || >=20.0.0",
-        "@vitest/browser": "2.1.2",
-        "@vitest/ui": "2.1.2",
+        "@vitest/browser": "2.1.3",
+        "@vitest/ui": "2.1.3",
         "happy-dom": "*",
         "jsdom": "*"
       },
diff --git a/package.json b/package.json
index afd8f8be..ce8bb3d7 100644
--- a/package.json
+++ b/package.json
@@ -43,7 +43,7 @@
     "@playwright/test": "^1.39.0",
     "@testcontainers/localstack": "^10.7.2",
     "@types/node": "^20.8.9",
-    "@vitest/coverage-v8": "^2.1.2",
+    "@vitest/coverage-v8": "^2.1.3",
     "@wiremock/wiremock-testcontainers-node": "^0.0.1",
     "concurrently": "^8.2.2",
     "husky": "^9.0.11",
diff --git a/vite.config.js b/vite.config.js
index ba4a6ed5..e3775575 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -15,7 +15,10 @@ export default defineConfig({
       reporter: ['text', 'json-summary', 'json'],
       // If you want a coverage reports even if your tests are failing, include the reportOnFailure option
       reportOnFailure: true,
-      snapshotDir: 'snapshots'
+      snapshotDir: 'snapshots',
+      exclude: [
+        './public/*'
+      ]
     }
   }
 })

From f2729b07f9cc57e4f6edfc0341b1a852b9cfa1e4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 14:15:35 +0100
Subject: [PATCH 070/109] get entitiesWithIssueCount from all active resources
 and not just the latest

---
 src/middleware/common.middleware.js       | 19 -------------------
 src/middleware/issueDetails.middleware.js | 14 ++++----------
 src/services/performanceDbApi.js          |  4 ++--
 3 files changed, 6 insertions(+), 31 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 85521384..51bf7dca 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -107,25 +107,6 @@ export const pullOutDatasetSpecification = (req, res, next) => {
   next()
 }
 
-/**
- *
- * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
- *
- * Requires `req.resource.resource`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-export async function fetchIssueEntitiesCount (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  console.assert(resourceId, 'missng resource id')
-  const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId)
-  req.issueEntitiesCount = parseInt(issueEntitiesCount)
-  next()
-}
-
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
   const { entityCount: entityCountRow, issuesWithReferences, issuesWithoutReferences, entities } = req
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 6a648ec7..dbbc47b6 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -9,20 +9,16 @@ import {
   fetchEntitiesFromIssuesWithReferences,
   fetchEntityCount,
   fetchFieldMappings,
-  fetchIssueEntitiesCount,
   fetchIssuesWithoutReferences,
   fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
-  fetchLatestResource,
   fetchOrgInfo,
   fetchSpecification,
   formatErrorSummaryParams,
   hasEntities,
-  isResourceIdNotInParams,
   logPageError,
   nestEntityFields,
   pullOutDatasetSpecification,
   replaceUnderscoreWithHyphenForEntities,
-  takeResourceIdFromParams,
   validateQueryParams
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
@@ -72,10 +68,10 @@ export const getIssueField = (text, html, classes = '') => {
 }
 
 export const setPagePaginationOptions = (req, res, next) => {
-  const { issueEntitiesCount } = req
+  const { entities } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
-  req.resultsCount = issueEntitiesCount
+  req.resultsCount = entities.length
   req.urlSubPath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
   req.paginationPageLength = 1
 
@@ -95,7 +91,7 @@ export const setPagePaginationOptions = (req, res, next) => {
  * from the request, and organizes it into a template parameters object that can be used to render the page.
  */
 export function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entities, issueEntitiesCount, errorSummary, specification, pagination } = req
+  const { entities, errorSummary, specification, pagination } = req
   const { issue_type: issueType, issue_field: issueField, pageNumber } = req.params
 
   if (pageNumber > entities.length) {
@@ -138,7 +134,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     issueType,
     issueField,
     pagination,
-    issueEntitiesCount
+    issueEntitiesCount: entities.length
   }
 
   next()
@@ -158,7 +154,6 @@ export default [
   validateIssueDetailsQueryParams,
   fetchOrgInfo,
   fetchDatasetInfo,
-  fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchSpecification,
   pullOutDatasetSpecification,
   fetchFieldMappings,
@@ -173,7 +168,6 @@ export default [
   fetchIf(hasEntities, nestEntityFields),
   fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
-  fetchIssueEntitiesCount,
   formatErrorSummaryParams,
   setPagePaginationOptions,
   createPaginationTemplateParams,
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index eccc14a5..9f1ca08f 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -351,14 +351,14 @@ export default {
      * @returns {Promise<number>} Count of entities with issues
      */
   async getEntitiesWithIssuesCount ({
-    resource,
+    resources,
     issueType,
     issueField
   }, database = 'digital-land') {
     const sql = `
     SELECT count(DISTINCT entry_number) as count
     FROM issue
-    WHERE resource = '${resource}'
+    WHERE resource in ('${resources.join("',' ")}')
     AND issue_type = '${issueType}'
     AND field = '${issueField}'
   `

From 7401876fa0043e3db0256de7972d54770e6710f9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 14:19:38 +0100
Subject: [PATCH 071/109] check for null end_dates too

---
 src/services/performanceDbApi.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 9f1ca08f..3fff79ae 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -273,7 +273,7 @@ export default {
       LEFT JOIN organisation o ON REPLACE(ro.organisation, '-eng', '') = o.organisation
       WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}'
       AND pipeline = '${dataset}'
-      AND rhe.endpoint_end_date == ''
+      AND (rhe.endpoint_end_date == '' OR rhe.endpoint_end_date is NULL)
     `
   },
 

From 6f0776fad854daf812ee14dc0a47e2577c5e3f57 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 14:45:20 +0100
Subject: [PATCH 072/109] account for one to many field -> dataset field
 mappings

---
 src/middleware/common.middleware.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 51bf7dca..3ff7c0c0 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -360,14 +360,20 @@ export const fetchFieldMappings = fetchMany({
 export const addDatabaseFieldToSpecification = (req, res, next) => {
   const { specification, fieldMappings } = req
 
-  req.specification.fields = specification.fields.map(fieldObj => {
+  req.specification.fields = specification.fields.flatMap(fieldObj => {
     if (['GeoX', 'GeoY'].includes(fieldObj.field)) { // special case for brownfield land
       return { datasetField: 'point', ...fieldObj }
     }
 
-    const fieldMapping = fieldMappings.find(mapping => mapping.field === fieldObj.field)
-    const databaseField = fieldMapping?.replacement_field || fieldObj.field
-    return { datasetField: databaseField, ...fieldObj }
+    const fieldMapping = fieldMappings.filter(mapping => mapping.field === fieldObj.field)
+
+    // sometimes a field maps to more than one dataset field, so we need to account for that
+    const specificationEntriesWithDatasetFields = []
+    fieldMapping.forEach(mapping => {
+      const databaseField = mapping?.replacement_field || fieldObj.field
+      specificationEntriesWithDatasetFields.push({ datasetField: databaseField, ...fieldObj })
+    })
+    return specificationEntriesWithDatasetFields
   })
 
   next()

From cd6c29cc68b7ae8404c3d04ca60bc505c1ee7d50 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 17 Oct 2024 15:18:24 +0100
Subject: [PATCH 073/109] populate with regular fields if dataset field isn't
 present

---
 src/middleware/common.middleware.js | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 3ff7c0c0..2ef35f49 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -365,14 +365,17 @@ export const addDatabaseFieldToSpecification = (req, res, next) => {
       return { datasetField: 'point', ...fieldObj }
     }
 
-    const fieldMapping = fieldMappings.filter(mapping => mapping.field === fieldObj.field)
+    const fieldMappingsForField = fieldMappings.filter(mapping => mapping.field === fieldObj.field)
+
+    const datasetFields = fieldMappingsForField.map(mapping => mapping.replacement_field).filter(Boolean)
+
+    if (datasetFields.length === 0) {
+      // no dataset fields found, add the field anyway with datasetField set to the same value as fieldObj.field
+      return { datasetField: fieldObj.field, ...fieldObj }
+    }
 
     // sometimes a field maps to more than one dataset field, so we need to account for that
-    const specificationEntriesWithDatasetFields = []
-    fieldMapping.forEach(mapping => {
-      const databaseField = mapping?.replacement_field || fieldObj.field
-      specificationEntriesWithDatasetFields.push({ datasetField: databaseField, ...fieldObj })
-    })
+    const specificationEntriesWithDatasetFields = datasetFields.map(datasetField => ({ datasetField, ...fieldObj }))
     return specificationEntriesWithDatasetFields
   })
 

From 3ea9e1eda9f697c2ce742cc462c075fa05b64344 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 18 Oct 2024 09:57:39 +0100
Subject: [PATCH 074/109] check resource end date too

---
 src/services/performanceDbApi.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 3fff79ae..74be9c81 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -274,6 +274,7 @@ export default {
       WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}'
       AND pipeline = '${dataset}'
       AND (rhe.endpoint_end_date == '' OR rhe.endpoint_end_date is NULL)
+      AND (rhe.resource_end_date == '' OR rhe.resource_end_date is NULL)
     `
   },
 

From a9f96c50ce315d953867bd850aa8800d3adeac92 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 18 Oct 2024 10:45:08 +0100
Subject: [PATCH 075/109] make sure to get entity count early on so it can be
 used in the error summary

---
 src/middleware/issueTable.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 406270f5..9730a86f 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -140,12 +140,12 @@ export default [
   addDatasetFieldsToIssues,
   fetchIf(hasEntities, addEntityPageNumberToEntity),
   fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
+  fetchEntityCount,
   formatErrorSummaryParams,
   fetchIf(hasEntities, extractJsonFieldFromEntities),
   fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities),
   fetchIf(hasEntities, nestEntityFields),
   fetchIf(hasEntities, addIssuesToEntities),
-  fetchEntityCount,
   setPagePageOptions(paginationPageLength),
   createPaginationTemplateParams,
   prepareIssueTableTemplateParams,

From 60acdd8e6e87c9fcdfa7754bb4576c7b7bb1d211 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 18 Oct 2024 11:38:11 +0100
Subject: [PATCH 076/109] make sure to look at total issues and not pagination
 entities when creating the error summary title

---
 src/middleware/common.middleware.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 2ef35f49..a2ddfc26 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -118,13 +118,15 @@ export function formatErrorSummaryParams (req, res, next) {
   let errorHeading
   let issueItems
 
+  const totalIssues = issuesWithReferences.length + issuesWithoutReferences.length
+
   // if the entities length is 0, this means the entry never became an entity, so we shouldn't show the table or links to the entity details page
   if (entities.length === 0) {
     issueItems = [{
       html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithoutReferences.length, entityCount, field: issueField }, true)
     }]
-  } else if (entities.length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: entities.length, entityCount, field: issueField }, true)
+  } else if (totalIssues < entityCount) {
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: totalIssues, entityCount, field: issueField }, true)
     issueItems = entities.map((entity, index) => {
       return {
         html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in entity ${entity?.reference?.value || entity?.reference}`,
@@ -133,7 +135,7 @@ export function formatErrorSummaryParams (req, res, next) {
     })
   } else {
     issueItems = [{
-      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true)
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: totalIssues, entityCount, field: issueField }, true)
     }]
   }
 

From 24edfae0f379be042529e52b9ffe242b1c466cf4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 18 Oct 2024 15:47:10 +0100
Subject: [PATCH 077/109] make sure that we only show each field once

---
 src/middleware/issueDetails.middleware.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index dbbc47b6..2c11a861 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -103,7 +103,9 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
 
   const entity = entities[pageNumber - 1]
 
-  const fields = specification.fields.map(({ datasetField }) => {
+  const datasetFields = [...new Set(specification.fields.map(({ datasetField }) => datasetField))]
+
+  const fields = datasetFields.map(datasetField => {
     let valueHtml = ''
     let classes = ''
     if (!entity[datasetField]) {

From 8644d54f7a98cf5d7b577c00beabead2a0830be9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:00:06 +0100
Subject: [PATCH 078/109] nested loop refactor

---
 test/unit/sharedTests/tableTests.js | 33 +++++++++++++++++------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/test/unit/sharedTests/tableTests.js b/test/unit/sharedTests/tableTests.js
index 992b971c..3f268ae6 100644
--- a/test/unit/sharedTests/tableTests.js
+++ b/test/unit/sharedTests/tableTests.js
@@ -23,20 +23,27 @@ export const runTableTests = (tableParams, document) => {
     it('Renders the correct row content', () => {
       tableParams.rows.forEach((rowData, i) => {
         const columns = rows[i].children
-        expect(columns.length).toEqual(Object.keys(rowData.columns).length)
-
-        Object.values(rowData.columns).forEach((field, j) => {
-          if (field.value) {
-            expect(columns[j].textContent).toContain(field.value)
-          } else if (field.html) {
-            expect(columns[j].innerHTML).toContain(field.html)
-          }
-
-          if (field.error) {
-            expect(columns[j].textContent).toContain(prettifyColumnName(field.error.message))
-          }
-        })
+        checkRowContent(columns, rowData)
       })
     })
   })
 }
+
+function checkRowContent (columns, rowData) {
+  expect(columns.length).toEqual(Object.keys(rowData.columns).length)
+
+  Object.values(rowData.columns).forEach((field, j) => {
+    if (field.value) {
+      expect(columns[j].textContent).toContain(field.value)
+    } else if (field.html) {
+      expect(columns[j].innerHTML).toContain(field.html)
+    }
+
+    if (field.error) {
+      expect(columns[j].textContent).toContain(prettifyColumnName(field.error.message))
+    }
+  })
+
+  // Check for unexpected additional columns
+  expect(columns.length).toEqual(Object.keys(rowData.columns).length)
+}

From 16894687aa778e9b7f15f082c2e73f8690386fd6 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:14:33 +0100
Subject: [PATCH 079/109] improve and standardise test seed generation

---
 test/unit/dataset-details.test.js                    | 4 ++--
 test/unit/datasetTaskListPage.test.js                | 4 ++--
 test/unit/findPage.test.js                           | 4 ++--
 test/unit/get-startedPage.test.js                    | 4 ++--
 test/unit/http-errorPage.test.js                     | 4 ++--
 test/unit/issueDetailsPage.test.js                   | 4 ++--
 test/unit/lpaOverviewPage.test.js                    | 4 ++--
 test/unit/views/organisations/issueTablePage.test.js | 4 ++--
 test/utils/mocker.js                                 | 4 ++++
 9 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/test/unit/dataset-details.test.js b/test/unit/dataset-details.test.js
index 2ef9b39d..35e5684d 100644
--- a/test/unit/dataset-details.test.js
+++ b/test/unit/dataset-details.test.js
@@ -8,7 +8,7 @@ import { testValidationErrorMessage } from './validation-tests.js'
 import { render } from '../../src/utils/custom-renderer.js'
 import * as v from 'valibot'
 
-import mock from '../utils/mocker.js'
+import mock, { getSeed } from '../utils/mocker.js'
 import { DatasetDetails } from '../../src/routes/schemas.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
@@ -40,7 +40,7 @@ function errorTestFn ({
   }
 }
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`dataset details View (seed: ${seed})`, () => {
   const params = mock(DatasetDetails, seed)
diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js
index d696cd45..42112ae1 100644
--- a/test/unit/datasetTaskListPage.test.js
+++ b/test/unit/datasetTaskListPage.test.js
@@ -2,12 +2,12 @@ import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { runGenericPageTests } from './sharedTests/generic-page.js'
 import jsdom from 'jsdom'
-import mocker from '../utils/mocker.js'
+import mocker, { getSeed } from '../utils/mocker.js'
 import { OrgDatasetTaskList } from '../../src/routes/schemas.js'
 
 const nunjucks = setupNunjucks({})
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`Dataset Task List Page (seed: ${seed})`, () => {
   const params = mocker(OrgDatasetTaskList, seed)
diff --git a/test/unit/findPage.test.js b/test/unit/findPage.test.js
index 0f41b6c9..296f95bf 100644
--- a/test/unit/findPage.test.js
+++ b/test/unit/findPage.test.js
@@ -3,12 +3,12 @@ import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import jsdom from 'jsdom'
 import { runGenericPageTests } from './sharedTests/generic-page.js'
 import config from '../../config/index.js'
-import mock from '../utils/mocker.js'
+import mock, { getSeed } from '../utils/mocker.js'
 import { OrgFindPage } from '../../src/routes/schemas.js'
 
 const nunjucks = setupNunjucks({})
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`Organisations Find Page (seed: ${seed})`, () => {
   const params = mock(OrgFindPage, seed)
diff --git a/test/unit/get-startedPage.test.js b/test/unit/get-startedPage.test.js
index 3936e5b2..7ccf6655 100644
--- a/test/unit/get-startedPage.test.js
+++ b/test/unit/get-startedPage.test.js
@@ -3,13 +3,13 @@
 import { describe, it, expect } from 'vitest'
 import { runGenericPageTests } from './sharedTests/generic-page.js'
 import jsdom from 'jsdom'
-import mocker from '../utils/mocker.js'
+import mocker, { getSeed } from '../utils/mocker.js'
 import { OrgGetStarted } from '../../src/routes/schemas.js'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 
 const nunjucks = setupNunjucks({})
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`Get Started Page (seed: ${seed})`, () => {
   const params = mocker(OrgGetStarted, seed)
diff --git a/test/unit/http-errorPage.test.js b/test/unit/http-errorPage.test.js
index 9a90878d..1f2ffb6e 100644
--- a/test/unit/http-errorPage.test.js
+++ b/test/unit/http-errorPage.test.js
@@ -2,14 +2,14 @@ import { describe, it, expect } from 'vitest'
 import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
 import { runGenericPageTests } from './sharedTests/generic-page.js'
-import mock from '../utils/mocker.js'
+import mock, { getSeed } from '../utils/mocker.js'
 import { OrgEndpointError } from '../../src/routes/schemas.js'
 
 const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
 
 const dateRegex = /\d{1,2} \w{3,9} \d{4} at \d{1,2}(?::\d{2})?(?:am|pm)/g
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`http-error.html(seed: ${seed})`, () => {
   const params = mock(OrgEndpointError, seed)
diff --git a/test/unit/issueDetailsPage.test.js b/test/unit/issueDetailsPage.test.js
index a3655334..83567e72 100644
--- a/test/unit/issueDetailsPage.test.js
+++ b/test/unit/issueDetailsPage.test.js
@@ -4,11 +4,11 @@ import { JSDOM } from 'jsdom'
 import { runGenericPageTests } from './sharedTests/generic-page.js'
 import config from '../../config/index.js'
 import { OrgIssueDetails } from '../../src/routes/schemas.js'
-import mocker from '../utils/mocker.js'
+import mocker, { getSeed } from '../utils/mocker.js'
 
 const nunjucks = setupNunjucks({})
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`issueDetails.html(seed: ${seed})`, () => {
   const params = mocker(OrgIssueDetails, seed)
diff --git a/test/unit/lpaOverviewPage.test.js b/test/unit/lpaOverviewPage.test.js
index 09585faf..32be860b 100644
--- a/test/unit/lpaOverviewPage.test.js
+++ b/test/unit/lpaOverviewPage.test.js
@@ -3,13 +3,13 @@ import { setupNunjucks } from '../../src/serverSetup/nunjucks.js'
 import { runGenericPageTests } from './sharedTests/generic-page.js'
 import jsdom from 'jsdom'
 import { makeDatasetSlugToReadableNameFilter } from '../../src/filters/makeDatasetSlugToReadableNameFilter.js'
-import mocker from '../utils/mocker.js'
+import mocker, { getSeed } from '../utils/mocker.js'
 import { datasetStatusEnum, OrgOverviewPage } from '../../src/routes/schemas.js'
 
 const datasetNameMapping = new Map()
 const nunjucks = setupNunjucks({ datasetNameMapping })
 
-const seed = new Date().getTime()
+const seed = getSeed()
 
 describe(`LPA Overview Page (seed: ${seed})`, () => {
   const params = mocker(OrgOverviewPage, seed)
diff --git a/test/unit/views/organisations/issueTablePage.test.js b/test/unit/views/organisations/issueTablePage.test.js
index 9abeae3a..d097cf1d 100644
--- a/test/unit/views/organisations/issueTablePage.test.js
+++ b/test/unit/views/organisations/issueTablePage.test.js
@@ -3,7 +3,7 @@ import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
 import { JSDOM } from 'jsdom'
 import { runGenericPageTests } from '../../sharedTests/generic-page.js'
 import config from '../../../../config/index.js'
-import mocker from '../../../utils/mocker.js'
+import mocker, { getSeed } from '../../../utils/mocker.js'
 import { OrgIssueTable } from '../../../../src/routes/schemas.js'
 import { runTableTests } from '../../sharedTests/tableTests.js'
 
@@ -75,5 +75,5 @@ const runTestsWithSeed = (seed) => {
   })
 }
 
-const seed = new Date().getTime()
+const seed = getSeed()
 runTestsWithSeed(seed)
diff --git a/test/utils/mocker.js b/test/utils/mocker.js
index 17341961..cdcec132 100644
--- a/test/utils/mocker.js
+++ b/test/utils/mocker.js
@@ -108,3 +108,7 @@ const mockTableParams = (tableParams, schema) => {
 
   return tableParams
 }
+
+export const getSeed = () => {
+  return process.env.TEST_SEED || new Date().getTime()
+}

From 0c741500465f3c5b0503027c8e29955bc086b521 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:24:09 +0100
Subject: [PATCH 080/109] make sure map on table page is showing

---
 src/views/organisations/issueTable.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
index 7d05c92b..7b6094ad 100644
--- a/src/views/organisations/issueTable.html
+++ b/src/views/organisations/issueTable.html
@@ -59,7 +59,7 @@
       errorList: errorSummary.items
     }) }}
 
-    {% if false and entry.geometries and entry.geometries.length %}
+    {% if entry.geometries and entry.geometries.length %}
       <div
         id="map"
         class="app-map"

From 9c997c8ba89af7ad5bacc3dee7ecd7b2612a048a Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:43:04 +0100
Subject: [PATCH 081/109] make sure map is shown on issue table page

---
 src/middleware/issueTable.middleware.js       | 23 ++++++++++-
 src/routes/schemas.js                         |  3 +-
 src/views/organisations/issueTable.html       |  6 +--
 .../middleware/issueTable.middleware.test.js  | 40 ++++++++++++++++++-
 4 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 9730a86f..6826b6ef 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -70,6 +70,23 @@ export const setPagePageOptions = (pageLength) => (req, res, next) => {
   next()
 }
 
+export const getGeometriesFromEntities = (req, res, next) => {
+  const { entities } = req
+
+  const geometries = entities.map(entity => {
+    if (entity && entity.geometry && entity.geometry.value) {
+      return entity.geometry.value
+    } else if (entity && entity.point && entity.point.value) {
+      return entity.point.value
+    } else {
+      return null
+    }
+  }).filter(geometry => geometry !== null)
+
+  req.geometries = geometries
+  next()
+}
+
 /**
  * Middleware function to prepare issue table template params
  *
@@ -79,7 +96,7 @@ export const setPagePageOptions = (pageLength) => (req, res, next) => {
  */
 export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
-  const { entities, specification, pagination, errorSummary } = req
+  const { entities, specification, pagination, errorSummary, geometries } = req
 
   const columnHeaders = [...new Set(specification.fields.map(field => field.datasetField || field.field))]
 
@@ -111,7 +128,8 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
     errorSummary,
     issueType,
     tableParams,
-    pagination
+    pagination,
+    geometries
   }
   next()
 }
@@ -148,6 +166,7 @@ export default [
   fetchIf(hasEntities, addIssuesToEntities),
   setPagePageOptions(paginationPageLength),
   createPaginationTemplateParams,
+  getGeometriesFromEntities,
   prepareIssueTableTemplateParams,
   getIssueTable,
   logPageError
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 80ce5179..dcd120df 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -196,7 +196,8 @@ export const OrgIssueTable = v.strictObject({
   errorSummary: errorSummaryField,
   issueType: NonEmptyString,
   tableParams,
-  pagination: paginationParams
+  pagination: paginationParams,
+  geometries: v.array(v.string())
 })
 
 export const CheckAnswers = v.strictObject({
diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html
index 7b6094ad..1d13068c 100644
--- a/src/views/organisations/issueTable.html
+++ b/src/views/organisations/issueTable.html
@@ -59,7 +59,7 @@
       errorList: errorSummary.items
     }) }}
 
-    {% if entry.geometries and entry.geometries.length %}
+    {% if geometries and geometries.length %}
       <div
         id="map"
         class="app-map"
@@ -107,12 +107,12 @@ <h3 class="govuk-heading-m">
 
 {% block scripts %}
   {{ super() }}
-  {% if entry.geometries and entry.geometries.length %}
+  {% if geometries and geometries.length %}
     <link href='https://unpkg.com/maplibre-gl@latest/dist/maplibre-gl.css' rel='stylesheet' />
     <script>
       window.serverContext = {
         containerId: 'map',
-        geometries: {{ entry.geometries | dump | safe }},
+        geometries: {{ geometries | dump | safe }},
         mapType: "static"
       }
     </script>
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index d31b9c94..1d18dcc8 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -1,5 +1,5 @@
 import { describe, it, vi, expect } from 'vitest'
-import { prepareIssueTableTemplateParams, IssueTableQueryParams, setPagePageOptions, addEntityPageNumberToEntity } from '../../../src/middleware/issueTable.middleware.js'
+import { prepareIssueTableTemplateParams, IssueTableQueryParams, setPagePageOptions, addEntityPageNumberToEntity, getGeometriesFromEntities } from '../../../src/middleware/issueTable.middleware.js'
 // import { pagination } from '../../../src/utils/pagination.js'
 
 import mocker from '../../utils/mocker.js'
@@ -143,4 +143,42 @@ describe('issueTable.middleware.js', () => {
       expect(req.templateParams).toEqual(expectedTemplateParams)
     })
   })
+
+  describe('getGeometriesFromEntities', () => {
+    it('should extract geometries from entities', () => {
+      const entities = [
+        {
+          geometry: { value: 'POINT(1 2)' }
+        },
+        {
+          point: { value: 'POINT(3 4)' }
+        },
+        {
+          noGeometry: true
+        }
+      ]
+
+      const req = { entities }
+      const res = {}
+      const next = vi.fn()
+
+      getGeometriesFromEntities(req, res, next)
+
+      expect(req.geometries).toEqual(['POINT(1 2)', 'POINT(3 4)'])
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+
+    it('should handle empty entities array', () => {
+      const entities = []
+
+      const req = { entities }
+      const res = {}
+      const next = vi.fn()
+
+      getGeometriesFromEntities(req, res, next)
+
+      expect(req.geometries).toEqual([])
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+  })
 })

From a3e4b0376ba27621604cb4263785d9e26515e102 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:48:31 +0100
Subject: [PATCH 082/109] add default value for issuesWithCounts

---
 src/middleware/datasetTaskList.middleware.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index 3b86fc3f..99c44a7f 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -48,7 +48,8 @@ export const prepareDatasetTaskListTemplateParams = (req, res, next) => {
   const { lpa, dataset: datasetId } = params
   console.assert(typeof entityCount === 'number', 'entityCount should be a number')
 
-  const taskList = issuesWithCounts.map((issue) => {
+  const issuesWithCountsOrDefault = issuesWithCounts || []
+  const taskList = issuesWithCountsOrDefault.map((issue) => {
     return {
       title: {
         text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field })

From b3f2ce9d9b153c2c6ae89a7377c40cd82b04949f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:55:18 +0100
Subject: [PATCH 083/109] avoid false negatives on 0 or ''

---
 src/middleware/issueTable.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 6826b6ef..36688c12 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -109,7 +109,7 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
         if (field === 'reference') {
           const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${entity.entityPageNumber}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field].value}</a>`, error: entity[field].issue }
-        } else if (entity[field]) {
+        } else if (field in entity) {
           columns[field] = { value: entity[field].value, error: entity[field].issue }
         } else {
           columns[field] = { value: '' }

From bbfd06b362968641678fc3c64ddae57ac8496c37 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 10:55:25 +0100
Subject: [PATCH 084/109] encode url params

---
 src/middleware/issueTable.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 36688c12..90eed0e7 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -107,7 +107,7 @@ export const prepareIssueTableTemplateParams = (req, res, next) => {
       const columns = {}
       columnHeaders.forEach(field => {
         if (field === 'reference') {
-          const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${entity.entityPageNumber}`
+          const entityLink = `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(issueType)}/${encodeURIComponent(issueField)}/entry/${entity.entityPageNumber}`
           columns[field] = { html: `<a href="${entityLink}">${entity[field].value}</a>`, error: entity[field].issue }
         } else if (field in entity) {
           columns[field] = { value: entity[field].value, error: entity[field].issue }

From 9f8032f6d290d123218a88630ef9002f514605d2 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:14:40 +0100
Subject: [PATCH 085/109] Avoid mutating entity object in
 prepareIssueDetailsTemplateParams middleware

---
 src/middleware/issueDetails.middleware.js | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 2c11a861..89351fe3 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -108,12 +108,8 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   const fields = datasetFields.map(datasetField => {
     let valueHtml = ''
     let classes = ''
-    if (!entity[datasetField]) {
-      entity[datasetField] = {
-        value: ''
-      }
-    }
-    if (entity[datasetField].issue) {
+    const fieldValue = entity[datasetField] || { value: '' }
+    if (fieldValue.issue) {
       valueHtml += issueErrorMessageHtml(entity[datasetField].issue.message, null)
       classes += 'dl-summary-card-list__row--error'
     }

From c5b37db8b6236b5bd21283fa96a6271c4555996d Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:21:39 +0100
Subject: [PATCH 086/109] fix tests

---
 src/middleware/issueDetails.middleware.js          | 2 +-
 test/unit/middleware/issueTable.middleware.test.js | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 89351fe3..59655833 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -113,7 +113,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
       valueHtml += issueErrorMessageHtml(entity[datasetField].issue.message, null)
       classes += 'dl-summary-card-list__row--error'
     }
-    valueHtml += entity[datasetField].value || ''
+    valueHtml += entity[datasetField]?.value || ''
     return getIssueField(datasetField, valueHtml, classes)
   })
 
diff --git a/test/unit/middleware/issueTable.middleware.test.js b/test/unit/middleware/issueTable.middleware.test.js
index 1d18dcc8..37fc8391 100644
--- a/test/unit/middleware/issueTable.middleware.test.js
+++ b/test/unit/middleware/issueTable.middleware.test.js
@@ -95,7 +95,8 @@ describe('issueTable.middleware.js', () => {
           { number: 1, href: '/pagenation-link-1' },
           { number: 2, href: '/pagenation-link-2' }
         ]
-      }
+      },
+      geometries: ['geometry1']
     }
     const res = {}
     const next = vi.fn()
@@ -117,7 +118,7 @@ describe('issueTable.middleware.js', () => {
             columns: {
               reference: {
                 error: undefined,
-                html: `<a href="/organisations/${req.params.lpa}/${req.params.dataset}/${req.params.issue_type}/${req.params.issue_field}/entry/${1}">${req.entities[0].reference.value}</a>`
+                html: `<a href="/organisations/${encodeURI(req.params.lpa)}/${encodeURI(req.params.dataset)}/${encodeURI(req.params.issue_type)}/${encodeURI(req.params.issue_field)}/entry/${encodeURI(1)}">${req.entities[0].reference.value}</a>`
               },
               'start-date': {
                 error: {
@@ -137,7 +138,8 @@ describe('issueTable.middleware.js', () => {
         errorSummary: req.errorSummary,
         issueType: req.params.issue_type,
         tableParams,
-        pagination: req.pagination
+        pagination: req.pagination,
+        geometries: req.geometries
       }
 
       expect(req.templateParams).toEqual(expectedTemplateParams)

From 24efac938544feb37b4d0f5f19c358a5f48c3fbe Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:23:18 +0100
Subject: [PATCH 087/109] throw error if there's no entities

---
 src/middleware/issueDetails.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 59655833..e2ffd064 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -94,7 +94,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   const { entities, errorSummary, specification, pagination } = req
   const { issue_type: issueType, issue_field: issueField, pageNumber } = req.params
 
-  if (pageNumber > entities.length) {
+  if (pageNumber > entities.length || entities.length === 0) {
     const error = new Error('pageNumber out of bounds')
     error.status = 400
     next(error)

From 9466042a5f5259a339f14a977c6dcedd29d7534e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:30:59 +0100
Subject: [PATCH 088/109] spy on logger instead of reassign

---
 test/unit/middleware/common.middleware.test.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 4be75e48..e68f5a66 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -6,8 +6,7 @@ vi.mock('../../../src/utils/logger')
 
 describe('logPageError', () => {
   it('logs an error with handlerName', () => {
-    const loggerMock = vi.fn()
-    logger.warn = loggerMock
+    const loggerMock = vi.spyOn(logger, 'warn')
 
     const err = new Error('Test error')
     const req = { handlerName: 'testHandler', originalUrl: '/test' }

From f3efe80ed82e6e3634530206953ca3cc50a6f37d Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:38:52 +0100
Subject: [PATCH 089/109] encode url

---
 src/middleware/issueDetails.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index e2ffd064..24a4763d 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -72,7 +72,7 @@ export const setPagePaginationOptions = (req, res, next) => {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
   req.resultsCount = entities.length
-  req.urlSubPath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/`
+  req.urlSubPath = `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(issueType)}/${encodeURIComponent(issueField)}/entry/`
   req.paginationPageLength = 1
 
   next()

From 14a63480894493b3bf900edadd96be0f418850af Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:46:59 +0100
Subject: [PATCH 090/109] added test for multipage issue table

---
 test/unit/issueDetailsPage.test.js            |  3 +-
 .../organisations/issueTablePage.test.js      | 89 ++++++++++++++++++-
 2 files changed, 89 insertions(+), 3 deletions(-)

diff --git a/test/unit/issueDetailsPage.test.js b/test/unit/issueDetailsPage.test.js
index 83567e72..cfe11acd 100644
--- a/test/unit/issueDetailsPage.test.js
+++ b/test/unit/issueDetailsPage.test.js
@@ -124,8 +124,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => {
     }
     params.issueEntitiesCount = 10
     const multiPageHtml = nunjucks.render('organisations/issueDetails.html', params)
-    // const multiPageDom = new JSDOM(multiPageHtml)
-    // const multiPageDocument = multiPageDom.window.document
+
     const paginationTitleSection = params.issueEntitiesCount > 1 ? `(Page ${params.pageNumber} of ${params.issueEntitiesCount}) ` : ''
     runGenericPageTests(multiPageHtml, {
       pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues ${paginationTitleSection}- ${config.serviceNames.submit}`,
diff --git a/test/unit/views/organisations/issueTablePage.test.js b/test/unit/views/organisations/issueTablePage.test.js
index d097cf1d..8c85282b 100644
--- a/test/unit/views/organisations/issueTablePage.test.js
+++ b/test/unit/views/organisations/issueTablePage.test.js
@@ -67,9 +67,96 @@ const runTestsWithSeed = (seed) => {
     runTableTests(params.tableParams, document)
 
     describe('multi page', () => {
-      // runGenericPageTests
+      const items = [
+        {
+          type: 'number',
+          current: false,
+          number: 1,
+          href: 'organisations/mock-org/mock-dataset/1'
+        },
+        {
+          type: 'number',
+          current: true,
+          number: 2,
+          href: 'organisations/mock-org/mock-dataset/2'
+        },
+        {
+          type: 'number',
+          current: false,
+          number: 3,
+          href: 'organisations/mock-org/mock-dataset/3'
+        },
+        {
+          type: 'ellipsis',
+          ellipsis: true,
+          href: '#'
+        },
+        {
+          type: 'number',
+          current: false,
+          number: 10,
+          href: 'organisations/mock-org/mock-dataset/10'
+        }
+      ]
+      const next = {
+        href: 'organisations/mock-org/mock-dataset/3'
+      }
+      const previous = {
+        href: 'organisations/mock-org/mock-dataset/1'
+      }
+      params.pagination = {
+        previous,
+        next,
+        items
+      }
+
+      const multiPageHtml = nunjucks.render('organisations/issueTable.html', params)
+
+      runGenericPageTests(multiPageHtml, {
+        pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues - ${config.serviceNames.submit}`,
+        breadcrumbs: [
+          { text: 'Home', href: '/' },
+          { text: 'Organisations', href: '/organisations' },
+          { text: params.organisation.name, href: `/organisations/${params.organisation.organisation}` },
+          { text: params.dataset.name, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` },
+          { text: 'mock issue' }
+        ]
+      })
+
+      const domMultiPage = new JSDOM(multiPageHtml)
+      const documentMultiPage = domMultiPage.window.document
 
       it('correctly renders the pagination component', () => {
+        const pagination = documentMultiPage.querySelector('.govuk-pagination')
+        const paginationChildren = pagination.children
+
+        expect(paginationChildren.length).toEqual(3)
+
+        const previousLink = paginationChildren[0]
+        expect(previousLink.getAttribute('class')).toContain('prev')
+        expect(previousLink.children[0].getAttribute('href')).toContain(previous.href)
+
+        const nextLink = paginationChildren[2]
+        expect(nextLink.getAttribute('class')).toContain('next')
+        expect(nextLink.children[0].getAttribute('href')).toContain(next.href)
+
+        const itemsList = paginationChildren[1]
+        expect(itemsList.getAttribute('class')).toContain('list')
+
+        const listElements = itemsList.children
+
+        expect(listElements.length).toEqual(5)
+
+        items.forEach((item, i) => {
+          if (item.type === 'number') {
+            expect(listElements[i].textContent).toContain(item.number)
+            expect(listElements[i].children[0].getAttribute('href')).toEqual(item.href)
+          } else if (item.type === 'ellipsis') {
+            expect(listElements[i].textContent).toContain('⋯')
+          } else {
+            expect.fail('pagination item type should be number or ellipsis')
+          }
+        })
       })
     })
   })

From 82471cd747190033ad3778694c77539edbd781c1 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:49:59 +0100
Subject: [PATCH 091/109] make sure mockTableParams isn't used before its
 defined

---
 test/utils/mocker.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/utils/mocker.js b/test/utils/mocker.js
index cdcec132..295c6682 100644
--- a/test/utils/mocker.js
+++ b/test/utils/mocker.js
@@ -63,14 +63,6 @@ export const resetRandomNumberGenerator = () => {
   xorShiftSeed = undefined
 }
 
-const enhanceMockedData = (data, schema) => {
-  if ('tableParams' in data) {
-    data.tableParams = mockTableParams(data.tableParams, schema.properties.tableParams)
-  }
-
-  return data
-}
-
 const mockTableParams = (tableParams, schema) => {
   const columnSchema = schema.properties.columns
   columnSchema.minItems = 2
@@ -109,6 +101,14 @@ const mockTableParams = (tableParams, schema) => {
   return tableParams
 }
 
+const enhanceMockedData = (data, schema) => {
+  if ('tableParams' in data) {
+    data.tableParams = mockTableParams(data.tableParams, schema.properties.tableParams)
+  }
+
+  return data
+}
+
 export const getSeed = () => {
   return process.env.TEST_SEED || new Date().getTime()
 }

From d64873a70e3de91991beb5a8843557b02ada0471 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 11:55:29 +0100
Subject: [PATCH 092/109] coderabbit suggestions on mocker.je

---
 test/utils/mocker.js | 92 ++++++++++++++++++++++----------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/test/utils/mocker.js b/test/utils/mocker.js
index 295c6682..e7879ac4 100644
--- a/test/utils/mocker.js
+++ b/test/utils/mocker.js
@@ -5,6 +5,52 @@ import { toJSONSchema } from '@gcornut/valibot-json-schema'
 import { JSONSchemaFaker } from 'json-schema-faker'
 import { date, number, string } from 'valibot'
 
+const mockTableParams = (tableParams, schema) => {
+  const columnSchema = { ...schema.properties.columns }
+  columnSchema.minItems = 2
+  columnSchema.maxItems = 10
+
+  const columns = JSONSchemaFaker.generate(columnSchema)
+
+  const fieldSchema = { ...schema.properties.fields }
+  fieldSchema.minItems = columns.length
+  fieldSchema.maxItems = columns.length
+  fieldSchema.uniqueItems = true
+
+  const fields = JSONSchemaFaker.generate(fieldSchema)
+
+  const rowsSchema = { ...schema.properties.rows }
+  rowsSchema.items.properties.columns.required = []
+
+  const rowSchema = { ...schema.properties.rows.items.properties.columns.additionalProperties }
+  rowSchema.oneOf = [
+    { required: ['html'] },
+    { required: ['value'] }
+  ]
+  rowSchema.additionalProperties = false
+
+  fields.forEach(field => {
+    rowsSchema.items.properties.columns.properties[field] = rowSchema
+    rowsSchema.items.properties.columns.required.push(field)
+  })
+
+  rowsSchema.items.properties.columns.additionalProperties = false
+
+  const rows = JSONSchemaFaker.generate(rowsSchema)
+
+  tableParams = { columns, fields, rows }
+
+  return tableParams
+}
+
+const enhanceMockedData = (data, schema) => {
+  if ('tableParams' in data) {
+    data.tableParams = mockTableParams(data.tableParams, schema.properties.tableParams)
+  }
+
+  return data
+}
+
 export default (schema, seed) => {
   const jsonSchema = toJSONSchema({
     schema,
@@ -63,52 +109,6 @@ export const resetRandomNumberGenerator = () => {
   xorShiftSeed = undefined
 }
 
-const mockTableParams = (tableParams, schema) => {
-  const columnSchema = schema.properties.columns
-  columnSchema.minItems = 2
-  columnSchema.maxItems = 10
-
-  const columns = JSONSchemaFaker.generate(columnSchema)
-
-  const fieldSchema = schema.properties.fields
-  fieldSchema.minItems = columns.length
-  fieldSchema.maxItems = columns.length
-  fieldSchema.uniqueItems = true
-
-  const fields = JSONSchemaFaker.generate(fieldSchema)
-
-  const rowsSchema = { ...schema.properties.rows }
-  rowsSchema.items.properties.columns.required = []
-
-  const rowSchema = { ...schema.properties.rows.items.properties.columns.additionalProperties }
-  rowSchema.oneOff = [
-    { required: ['html'] },
-    { required: ['value'] }
-  ]
-  rowSchema.additionalProperties = false
-
-  fields.forEach(field => {
-    rowsSchema.items.properties.columns.properties[field] = rowSchema
-    rowsSchema.items.properties.columns.required.push(field)
-  })
-
-  rowsSchema.items.properties.columns.additionalProperties = false
-
-  const rows = JSONSchemaFaker.generate(rowsSchema)
-
-  tableParams = { columns, fields, rows }
-
-  return tableParams
-}
-
-const enhanceMockedData = (data, schema) => {
-  if ('tableParams' in data) {
-    data.tableParams = mockTableParams(data.tableParams, schema.properties.tableParams)
-  }
-
-  return data
-}
-
 export const getSeed = () => {
   return process.env.TEST_SEED || new Date().getTime()
 }

From 9ffb0b90afde914971530498dbf110b726ecc697 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 12:40:28 +0100
Subject: [PATCH 093/109] escape html

---
 package-lock.json                         | 4 +++-
 package.json                              | 1 +
 src/middleware/issueDetails.middleware.js | 5 +++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index cd5bd938..f8430418 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -22,6 +22,7 @@
         "cookie-parser": "^1.4.6",
         "csv-parser": "^3.0.0",
         "dotenv": "^16.4.5",
+        "escape-html": "^1.0.3",
         "express": "^4.19.2",
         "express-session": "^1.18.0",
         "govuk-frontend": "^5.6.0",
@@ -6413,7 +6414,8 @@
     },
     "node_modules/escape-html": {
       "version": "1.0.3",
-      "license": "MIT"
+      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
     },
     "node_modules/escape-string-regexp": {
       "version": "1.0.5",
diff --git a/package.json b/package.json
index ce8bb3d7..c2612557 100644
--- a/package.json
+++ b/package.json
@@ -73,6 +73,7 @@
     "cookie-parser": "^1.4.6",
     "csv-parser": "^3.0.0",
     "dotenv": "^16.4.5",
+    "escape-html": "^1.0.3",
     "express": "^4.19.2",
     "express-session": "^1.18.0",
     "govuk-frontend": "^5.6.0",
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 24a4763d..2b11f3fa 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -23,6 +23,7 @@ import {
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
+import escape from 'escape-html'
 
 export const IssueDetailsQueryParams = v.strictObject({
   lpa: v.string(),
@@ -45,7 +46,7 @@ const validateIssueDetailsQueryParams = validateQueryParams({
  */
 export const issueErrorMessageHtml = (errorMessage, issue) =>
     `<p class="govuk-error-message">${errorMessage}</p>${
-      issue ? issue.value ?? '' : ''
+      escape(issue ? issue.value ?? '' : '')
     }`
 
 /**
@@ -113,7 +114,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
       valueHtml += issueErrorMessageHtml(entity[datasetField].issue.message, null)
       classes += 'dl-summary-card-list__row--error'
     }
-    valueHtml += entity[datasetField]?.value || ''
+    valueHtml += escape(entity[datasetField]?.value || '')
     return getIssueField(datasetField, valueHtml, classes)
   })
 

From 7a9237602093f5237d9e1a22d1eae69ad2e7001d Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 12:49:11 +0100
Subject: [PATCH 094/109] add test for empty error message

---
 src/middleware/issueDetails.middleware.js        | 16 +++++++++++-----
 .../middleware/issueDetails.middleware.test.js   |  6 ++++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 2b11f3fa..4bbd24b0 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -24,6 +24,7 @@ import {
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import escape from 'escape-html'
+import logger from '../utils/logger.js'
 
 export const IssueDetailsQueryParams = v.strictObject({
   lpa: v.string(),
@@ -44,10 +45,12 @@ const validateIssueDetailsQueryParams = validateQueryParams({
  * @param {{value: string}?} issue
  * @returns {string}
  */
-export const issueErrorMessageHtml = (errorMessage, issue) =>
-    `<p class="govuk-error-message">${errorMessage}</p>${
-      escape(issue ? issue.value ?? '' : '')
-    }`
+export const issueErrorMessageHtml = (errorMessage, issue) => {
+  if (!errorMessage) return ''
+  return `<p class="govuk-error-message">${errorMessage}</p>${
+    escape(issue ? issue.value ?? '' : '')
+  }`
+}
 
 /**
  *
@@ -111,7 +114,10 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
     let classes = ''
     const fieldValue = entity[datasetField] || { value: '' }
     if (fieldValue.issue) {
-      valueHtml += issueErrorMessageHtml(entity[datasetField].issue.message, null)
+      if (!fieldValue.issue.message) {
+        logger.warn('no issue message found for issue in entity', { entity })
+      }
+      valueHtml += issueErrorMessageHtml(fieldValue.issue.message, null)
       classes += 'dl-summary-card-list__row--error'
     }
     valueHtml += escape(entity[datasetField]?.value || '')
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 2adfaa3d..abf43ca4 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -27,6 +27,12 @@ describe('issueDetails.middleware.js', () => {
       const result = issueErrorMessageHtml(errorMessage, issue)
       expect(result).toBe(`<p class="govuk-error-message">${errorMessage}</p>`)
     })
+
+    it('should return an empty string if errorMessage is null', () => {
+      const issue = { value: '02-02-2022' }
+      const result = issueErrorMessageHtml(null, issue)
+      expect(result).toBe('')
+    })
   })
 
   describe('getIssueField', () => {

From 309750b3e2bbaec6f1b076d20448df0d5ab03c09 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 12:51:55 +0100
Subject: [PATCH 095/109] dont use delete for better performance

---
 src/middleware/common.middleware.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 121dcb31..64cefc70 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -234,7 +234,7 @@ export const extractJsonFieldFromEntities = (req, res, next) => {
       logger.info(`common.middleware/extractJsonField: No json field for entity ${entity.toString()}`)
       return entity
     }
-    delete entity.json
+    entity.json = undefined
     const parsedJson = JSON.parse(jsonField)
     entity = { ...entity, ...parsedJson }
     return entity
@@ -251,7 +251,7 @@ export const replaceUnderscoreWithHyphenForEntities = (req, res, next) => {
       if (key.includes('_')) {
         const newKey = key.replace(/_/g, '-')
         entity[newKey] = entity[key]
-        delete entity[key]
+        entity[key] = undefined
       }
     })
   })

From c69078643218c90e657fbbb19ba6a9ab10aca215 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 12:53:01 +0100
Subject: [PATCH 096/109] ensure entity field exists

---
 src/middleware/common.middleware.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 64cefc70..e1f87a63 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -297,6 +297,10 @@ export const addIssuesToEntities = (req, res, next) => {
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
+      if (!entity[issue.datasetField]) {
+        entity[issue.datasetField] = {}
+      }
+
       entity[issue.datasetField].value = issue.value || entity[issue.datasetField].value || ''
       entity[issue.datasetField].issue = issue
     })

From aac664184e771078f79005d4b1f339680b486daa Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:09:29 +0100
Subject: [PATCH 097/109] make sure getIssueField returns some default values

---
 src/middleware/issueDetails.middleware.js     |  8 +--
 .../issueDetails.middleware.test.js           | 66 +++++++++++++++++++
 2 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 4bbd24b0..1bd64291 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -59,15 +59,15 @@ export const issueErrorMessageHtml = (errorMessage, issue) => {
  * @param {*} classes
  * @returns {{key: {text: string}, value: { html: string}, classes: string}}
  */
-export const getIssueField = (text, html, classes = '') => {
+export const getIssueField = (text = '', html = '', classes = '') => {
   return {
     key: {
-      text
+      text: text ?? ''
     },
     value: {
-      html
+      html: html ?? ''
     },
-    classes
+    classes: classes ?? ''
   }
 }
 
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index abf43ca4..1ddb4b9f 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -58,6 +58,72 @@ describe('issueDetails.middleware.js', () => {
         classes: ''
       })
     })
+
+    it('should return an object with default values if text is null', () => {
+      const html = '<p>Mock html</p>'
+      const classes = 'mock-classes'
+      const result = getIssueField(null, html, classes)
+      expect(result).toEqual({
+        key: { text: '' },
+        value: { html },
+        classes
+      })
+    })
+
+    it('should return an object with default values if text is undefined', () => {
+      const html = '<p>Mock html</p>'
+      const classes = 'mock-classes'
+      const result = getIssueField(undefined, html, classes)
+      expect(result).toEqual({
+        key: { text: '' },
+        value: { html },
+        classes
+      })
+    })
+
+    it('should return an object with default values if html is null', () => {
+      const text = 'Mock text'
+      const classes = 'mock-classes'
+      const result = getIssueField(text, null, classes)
+      expect(result).toEqual({
+        key: { text },
+        value: { html: '' },
+        classes
+      })
+    })
+
+    it('should return an object with default values if html is undefined', () => {
+      const text = 'Mock text'
+      const classes = 'mock-classes'
+      const result = getIssueField(text, undefined, classes)
+      expect(result).toEqual({
+        key: { text },
+        value: { html: '' },
+        classes
+      })
+    })
+
+    it('should return an object with default values if classes is null', () => {
+      const text = 'Mock text'
+      const html = '<p>Mock html</p>'
+      const result = getIssueField(text, html, null)
+      expect(result).toEqual({
+        key: { text },
+        value: { html },
+        classes: ''
+      })
+    })
+
+    it('should return an object with default values if classes is undefined', () => {
+      const text = 'Mock text'
+      const html = '<p>Mock html</p>'
+      const result = getIssueField(text, html, undefined)
+      expect(result).toEqual({
+        key: { text },
+        value: { html },
+        classes: ''
+      })
+    })
   })
 
   describe('prepareIssueDetailsTemplateParams', () => {

From 0f440ea6f4f7d5e1ca3bbd369d2551b283ea085a Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:18:16 +0100
Subject: [PATCH 098/109] add more tests for prepareIssueDetailsTempalteParams

---
 src/middleware/issueDetails.middleware.js     | 12 ++++
 .../issueDetails.middleware.test.js           | 69 +++++++++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 1bd64291..b7eb47e3 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -98,6 +98,18 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
   const { entities, errorSummary, specification, pagination } = req
   const { issue_type: issueType, issue_field: issueField, pageNumber } = req.params
 
+  if (!entities) {
+    const error = new Error('entities is not defined')
+    next(error)
+    return
+  }
+
+  if (!specification) {
+    const error = new Error('specification is not defined')
+    next(error)
+    return
+  }
+
   if (pageNumber > entities.length || entities.length === 0) {
     const error = new Error('pageNumber out of bounds')
     error.status = 400
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 1ddb4b9f..a3f5ff01 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -207,6 +207,75 @@ describe('issueDetails.middleware.js', () => {
       prepareIssueDetailsTemplateParams(req, res, nextSpy)
       expect(nextSpy).toHaveBeenCalledTimes(1)
     })
+
+    it('should call next with an error if req.entities is missing', () => {
+      const req = {
+      // no entities property
+        params: {
+          lpa: 'lpa-1',
+          dataset: 'dataset-1',
+          issue_type: 'issue-type-1',
+          issue_field: 'issue-field-1',
+          pageNumber: '1'
+        },
+        orgInfo: { name: 'Org Name' },
+        dataset: { name: 'Dataset Name' },
+        pagination: 'paginationObject'
+      }
+      const res = {}
+      const next = vi.fn()
+
+      prepareIssueDetailsTemplateParams(req, res, next)
+
+      expect(next).toHaveBeenCalledWith(new Error('entities is not defined'))
+    })
+
+    it('should throw an error if req.entities is null', () => {
+      const req = {
+        entities: null,
+        params: {
+          lpa: 'lpa-1',
+          dataset: 'dataset-1',
+          issue_type: 'issue-type-1',
+          issue_field: 'issue-field-1',
+          pageNumber: '1'
+        },
+        orgInfo: { name: 'Org Name' },
+        dataset: { name: 'Dataset Name' },
+        pagination: 'paginationObject'
+      }
+      const res = {}
+      const next = vi.fn()
+
+      prepareIssueDetailsTemplateParams(req, res, next)
+
+      expect(next).toHaveBeenCalledWith(new Error('entities is not defined'))
+    })
+
+    it('should throw an error if req.specification is missing', () => {
+      const req = {
+        entities: [
+          { reference: { value: 'entry-1' }, geometry: { value: 'geom-1' }, field1: { value: 'val-1', issue: { message: 'error' } } }
+        ],
+        params: {
+          lpa: 'lpa-1',
+          dataset: 'dataset-1',
+          issue_type: 'issue-type-1',
+          issue_field: 'issue-field-1',
+          pageNumber: '1'
+        },
+        orgInfo: { name: 'Org Name' },
+        dataset: { name: 'Dataset Name' },
+        pagination: 'paginationObject'
+      // no specification property
+      }
+      const res = {}
+      const next = vi.fn()
+
+      prepareIssueDetailsTemplateParams(req, res, next)
+
+      expect(next).toHaveBeenCalledWith(new Error('specification is not defined'))
+    })
   })
 
   describe('getIssueDetails', () => {

From 5aa5d7267c44eadddfe01134feb8e57782b23814 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:22:05 +0100
Subject: [PATCH 099/109] add error handling to specification json parsing

---
 src/middleware/common.middleware.js | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index e1f87a63..b45900e1 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -102,7 +102,13 @@ export const fetchSpecification = fetchOne({
 
 export const pullOutDatasetSpecification = (req, res, next) => {
   const { specification } = req
-  const collectionSpecifications = JSON.parse(specification.json)
+  let collectionSpecifications
+  try {
+    collectionSpecifications = JSON.parse(specification.json)
+  } catch (error) {
+    logger.error('Invalid JSON in specification.json', { error })
+    return next(new Error('Invalid specification format'))
+  }
   const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
   req.specification = datasetSpecification
   next()

From 5153357673f555d98421fb8f3601418f0c39db46 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:22:16 +0100
Subject: [PATCH 100/109] update jsdoc

---
 src/middleware/issueDetails.middleware.js | 9 +++------
 src/services/performanceDbApi.js          | 2 +-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index b7eb47e3..6f81de30 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -100,21 +100,18 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
 
   if (!entities) {
     const error = new Error('entities is not defined')
-    next(error)
-    return
+    return next(error)
   }
 
   if (!specification) {
     const error = new Error('specification is not defined')
-    next(error)
-    return
+    return next(error)
   }
 
   if (pageNumber > entities.length || entities.length === 0) {
     const error = new Error('pageNumber out of bounds')
     error.status = 400
-    next(error)
-    return
+    return next(error)
   }
 
   const entity = entities[pageNumber - 1]
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 74be9c81..cfb8868e 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -345,7 +345,7 @@ export default {
      * Retrieves the count of entities with issues of a specific type and field.
      *
      * @param {Object} params - Parameters for the query
-     * @param {string} params.resource - Resource to filter by
+     * @param {string[]} params.resources - Resource to filter by
      * @param {string} params.issueType - Issue type to filter by
      * @param {string} params.issueField - Field to filter by
      * @param {string} [database="digital-land"] - Database to query (optional)

From 1f34a0124a231b47557a361472b3c358e79faacb Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:26:32 +0100
Subject: [PATCH 101/109] improve error handling around entities with invalid
 json

---
 src/middleware/common.middleware.js            |  8 ++++++--
 test/unit/middleware/common.middleware.test.js | 15 +++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index b45900e1..719c34aa 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -241,8 +241,12 @@ export const extractJsonFieldFromEntities = (req, res, next) => {
       return entity
     }
     entity.json = undefined
-    const parsedJson = JSON.parse(jsonField)
-    entity = { ...entity, ...parsedJson }
+    try {
+      const parsedJson = JSON.parse(jsonField)
+      entity = { ...entity, ...parsedJson }
+    } catch (err) {
+      logger.warn(`common.middleware/extractJsonField: Error parsing JSON for entity ${entity.toString()}: ${err.message}`)
+    }
     return entity
   })
 
diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index e68f5a66..660cd10c 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -301,6 +301,21 @@ describe('extractJsonFieldFromEntities', () => {
     expect(req.entities).toHaveLength(1)
     expect(req.entities[0]).toEqual({ id: 1 })
   })
+
+  it('handles entities with invalid json field', () => {
+    const req = {
+      entities: [
+        { id: 1, json: '{ invalid json }' }
+      ]
+    }
+    const res = {}
+    const next = vi.fn()
+
+    extractJsonFieldFromEntities(req, res, next)
+
+    expect(req.entities).toHaveLength(1)
+    expect(req.entities[0]).toEqual({ id: 1 })
+  })
 })
 
 describe('replaceUnderscoreWithHyphenForEntities', () => {

From f064a8b4895fc4c408cc255e29f1c81f5bf53bce Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:33:16 +0100
Subject: [PATCH 102/109] improved error handling and testing around nest
 entity fields

---
 src/middleware/common.middleware.js            |  5 +++++
 test/unit/middleware/common.middleware.test.js | 15 +++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 719c34aa..0f03fa7e 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -272,6 +272,11 @@ export const replaceUnderscoreWithHyphenForEntities = (req, res, next) => {
 export const nestEntityFields = (req, res, next) => {
   const { entities, specification } = req
 
+  if (!specification) {
+    const error = new Error('Specification is not defined')
+    return next(error)
+  }
+
   req.entities = entities.map(entity => {
     const columnHeaders = [...new Set(specification.fields.map(field => field.datasetField || field.field))]
     columnHeaders.forEach(field => {
diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index 660cd10c..fc3c5ab5 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -438,6 +438,21 @@ describe('nestEntityFields', () => {
     expect(req.entities).toHaveLength(1)
     expect(req.entities[0]).toEqual({ id: 1, name: 'John' })
   })
+
+  it('handles entities with undefined specification', () => {
+    const req = {
+      entities: [
+        { id: 1, name: 'John' }
+      ],
+      specification: null
+    }
+    const res = {}
+    const next = vi.fn()
+
+    nestEntityFields(req, res, next)
+
+    expect(next).toHaveBeenCalledWith(new Error('Specification is not defined'))
+  })
 })
 
 describe('addDatasetFieldsToIssues', () => {

From 4bb49cc7f90ec3fe6b9643ef6d647c2383c7b22c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:37:37 +0100
Subject: [PATCH 103/109] improve error handling and testing around
 getPaginationOptions

---
 src/middleware/common.middleware.js           |  6 ++++-
 .../unit/middleware/common.middleware.test.js | 26 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 0f03fa7e..0e8fc26c 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -168,7 +168,11 @@ export const paginateEntitiesAndPullOutCount = (req, res, next) => {
 }
 
 export const getPaginationOptions = (resultsCount) => (req, res, next) => {
-  const { pageNumber } = req.params
+  let pageNumber = parseInt(req.params.pageNumber, 10) || 1
+
+  if (pageNumber <= 0) {
+    pageNumber = 1
+  }
 
   req.pagination = { offset: (pageNumber - 1) * resultsCount, limit: resultsCount }
 
diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js
index fc3c5ab5..c26cf8ce 100644
--- a/test/unit/middleware/common.middleware.test.js
+++ b/test/unit/middleware/common.middleware.test.js
@@ -154,6 +154,32 @@ describe('pagination', () => {
       getPaginationOptionsMiddleware(req, res, next)
       expect(next).toHaveBeenCalledTimes(1)
     })
+
+    it('handles negative page numbers gracefully', () => {
+      const resultsCount = 10
+      const getPaginationOptionsMiddleware = getPaginationOptions(resultsCount)
+      const req = { params: { pageNumber: -1 } }
+      const res = {}
+      const next = vi.fn()
+
+      getPaginationOptionsMiddleware(req, res, next)
+      expect(req.pagination.offset).toBe(0)
+      expect(req.pagination.limit).toBe(10)
+      expect(next).toHaveBeenCalledTimes(1)
+    })
+
+    it('handles non-integer page numbers gracefully', () => {
+      const resultsCount = 10
+      const getPaginationOptionsMiddleware = getPaginationOptions(resultsCount)
+      const req = { params: { pageNumber: 'abc' } }
+      const res = {}
+      const next = vi.fn()
+
+      getPaginationOptionsMiddleware(req, res, next)
+      expect(req.pagination.offset).toBe(0)
+      expect(req.pagination.limit).toBe(10)
+      expect(next).toHaveBeenCalledTimes(1)
+    })
   })
 
   describe('paginateEntitiesAndPullOutCount', () => {

From 5114bbe0340349c93e9e912b2dac099c322d4c54 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:38:40 +0100
Subject: [PATCH 104/109] encode uri components

---
 src/middleware/issueTable.middleware.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js
index 90eed0e7..b04a255a 100644
--- a/src/middleware/issueTable.middleware.js
+++ b/src/middleware/issueTable.middleware.js
@@ -64,7 +64,7 @@ export const setPagePageOptions = (pageLength) => (req, res, next) => {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
 
   req.resultsCount = entitiesWithIssuesCount
-  req.urlSubPath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
+  req.urlSubPath = `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(issueType)}/${encodeURIComponent(issueField)}/`
   req.paginationPageLength = pageLength
 
   next()

From 1acd964f90a0ca1ef4fc1a9e01bdbe0ae1e383c5 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:40:42 +0100
Subject: [PATCH 105/109] remove redundant line

---
 test/unit/sharedTests/tableTests.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/test/unit/sharedTests/tableTests.js b/test/unit/sharedTests/tableTests.js
index 3f268ae6..84080e4c 100644
--- a/test/unit/sharedTests/tableTests.js
+++ b/test/unit/sharedTests/tableTests.js
@@ -43,7 +43,4 @@ function checkRowContent (columns, rowData) {
       expect(columns[j].textContent).toContain(prettifyColumnName(field.error.message))
     }
   })
-
-  // Check for unexpected additional columns
-  expect(columns.length).toEqual(Object.keys(rowData.columns).length)
 }

From e3a27eaf12752fec9a1f32027613aa681d435e89 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:45:01 +0100
Subject: [PATCH 106/109] use json.parse as ... is only a shallow copy

---
 test/utils/mocker.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/utils/mocker.js b/test/utils/mocker.js
index e7879ac4..20475191 100644
--- a/test/utils/mocker.js
+++ b/test/utils/mocker.js
@@ -6,23 +6,23 @@ import { JSONSchemaFaker } from 'json-schema-faker'
 import { date, number, string } from 'valibot'
 
 const mockTableParams = (tableParams, schema) => {
-  const columnSchema = { ...schema.properties.columns }
+  const columnSchema = JSON.parse(JSON.stringify(schema.properties.columns))
   columnSchema.minItems = 2
   columnSchema.maxItems = 10
 
   const columns = JSONSchemaFaker.generate(columnSchema)
 
-  const fieldSchema = { ...schema.properties.fields }
+  const fieldSchema = JSON.parse(JSON.stringify(schema.properties.fields))
   fieldSchema.minItems = columns.length
   fieldSchema.maxItems = columns.length
   fieldSchema.uniqueItems = true
 
   const fields = JSONSchemaFaker.generate(fieldSchema)
 
-  const rowsSchema = { ...schema.properties.rows }
+  const rowsSchema = JSON.parse(JSON.stringify(schema.properties.rows))
   rowsSchema.items.properties.columns.required = []
 
-  const rowSchema = { ...schema.properties.rows.items.properties.columns.additionalProperties }
+  const rowSchema = JSON.parse(JSON.stringify(schema.properties.rows.items.properties.columns.additionalProperties))
   rowSchema.oneOf = [
     { required: ['html'] },
     { required: ['value'] }

From d5b7a668ab77c7c2f0f028eb3dac1e91f38303e4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:53:18 +0100
Subject: [PATCH 107/109] rename escape to escapeHtml

---
 src/middleware/issueDetails.middleware.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index 6f81de30..9b7a4591 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -23,7 +23,7 @@ import {
 } from './common.middleware.js'
 import { fetchIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
-import escape from 'escape-html'
+import escapeHtml from 'escape-html'
 import logger from '../utils/logger.js'
 
 export const IssueDetailsQueryParams = v.strictObject({
@@ -48,7 +48,7 @@ const validateIssueDetailsQueryParams = validateQueryParams({
 export const issueErrorMessageHtml = (errorMessage, issue) => {
   if (!errorMessage) return ''
   return `<p class="govuk-error-message">${errorMessage}</p>${
-    escape(issue ? issue.value ?? '' : '')
+    escapeHtml(issue ? issue.value ?? '' : '')
   }`
 }
 
@@ -129,7 +129,7 @@ export function prepareIssueDetailsTemplateParams (req, res, next) {
       valueHtml += issueErrorMessageHtml(fieldValue.issue.message, null)
       classes += 'dl-summary-card-list__row--error'
     }
-    valueHtml += escape(entity[datasetField]?.value || '')
+    valueHtml += escapeHtml(entity[datasetField]?.value || '')
     return getIssueField(datasetField, valueHtml, classes)
   })
 

From 9424b4286637e886990a584e3d14a95eb89c3ee7 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:53:31 +0100
Subject: [PATCH 108/109] make copy of entity

---
 src/middleware/common.middleware.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 0e8fc26c..a86265d0 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -312,7 +312,9 @@ export const addDatasetFieldsToIssues = (req, res, next) => {
 export const addIssuesToEntities = (req, res, next) => {
   const { entities, issuesWithReferences } = req
 
-  req.entitiesWithIssues = entities.map(entity => {
+  req.entitiesWithIssues = entities.map(origionalEntity => {
+    const entity = JSON.parse(JSON.stringify(origionalEntity))
+
     const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {

From 48f5630ddec1deb5582b6cdee24eecb4ee742d20 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 21 Oct 2024 14:56:32 +0100
Subject: [PATCH 109/109] make sure to delete key

---
 src/middleware/common.middleware.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index a86265d0..bf61ad65 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -261,11 +261,12 @@ export const replaceUnderscoreWithHyphenForEntities = (req, res, next) => {
   const { entities } = req
 
   entities.forEach(entity => {
-    Object.keys(entity).forEach(key => {
+    const keys = Object.keys(entity)
+    keys.forEach(key => {
       if (key.includes('_')) {
         const newKey = key.replace(/_/g, '-')
         entity[newKey] = entity[key]
-        entity[key] = undefined
+        delete entity[key]
       }
     })
   })