From d75ea8741ef34cc2b82030f4aa77d1ce4d2e963e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 17 Sep 2024 14:21:14 +0100
Subject: [PATCH 01/65] add method to get specifications

---
 package.json                   |  2 ++
 src/services/DatasetService.js | 61 ++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/package.json b/package.json
index ed0bf9a8..e9f8d381 100644
--- a/package.json
+++ b/package.json
@@ -76,11 +76,13 @@
     "hmpo-form-wizard": "^13.0.0",
     "hmpo-i18n": "^6.0.1",
     "js-yaml": "^4.1.0",
+    "json5": "^2.2.3",
     "lodash": "^4.17.21",
     "maplibre-gl": "^4.1.0",
     "multer": "^1.4.5-lts.1",
     "notifications-node-client": "^8.2.0",
     "nunjucks": "^3.2.4",
+    "papaparse": "^5.4.1",
     "redis": "^4.6.13",
     "sass": "^1.69.4",
     "uuid": "^9.0.1",
diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 6a1f9323..856c5a03 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -1,6 +1,8 @@
 import datasette from './datasette.js'
 import logger from '../utils/logger.js'
 import performanceDbApi from './performanceDbApi.js'
+import Papa from 'papaparse'
+import JSON5 from 'json5' // need to use this as spec is written with single quotes. need to pass this onto data designers team to fix
 
 export async function getGeometryEntriesForResourceId (dataset, resourceId) {
   const sql = `
@@ -33,6 +35,20 @@ export async function getLatestDatasetGeometryEntriesForLpa (dataset, lpa) {
   }
 }
 
+/*
+  Needs to get:
+    - Number of records
+    - Number of fields supplied
+    - Number of fields matched
+    - Licence
+    - source documentation url
+    - for each data url
+      - the endpoint
+      - last accessed
+      - last updated
+      - any access errors
+
+*/
 async function getDatasetStatsForResourceId (dataset, resourceId) {
   const sql = `
     SELECT 'numberOfRecords' AS metric, COUNT(*) AS value
@@ -64,10 +80,24 @@ async function getDatasetStatsForResourceId (dataset, resourceId) {
   return formattedData
 }
 
+// async function getColumnSummary (dataset, lpa) {
+//   const sql = `select * from column_field_summary
+//     where resource != ''
+//     and pipeline = '${dataset}'
+//     AND organisation = '${lpa}'
+//     limit 1000`
+
+//   const { formattedData } = await datasette.runQuery(sql, 'performance')
+
+//   return formattedData
+// }
+
 export async function getDatasetStats (dataset, lpa) {
   try {
     const stats = {}
     const { resource: resourceId } = await performanceDbApi.getLatestResource(lpa, dataset)
+    // const columnSummary = await getColumnSummary(dataset, lpa)
+
     const metrics = await getDatasetStatsForResourceId(dataset, resourceId)
 
     metrics.forEach(({ metric, value }) => {
@@ -87,3 +117,34 @@ export async function getDatasetStats (dataset, lpa) {
     return {}
   }
 }
+
+export async function getSpecifications () {
+  try {
+    const response = await fetch('https://raw.githubusercontent.com/digital-land/specification/main/specification/specification.csv')
+
+    const csvData = await response.text()
+
+    const initalParse = Papa.parse(csvData, { header: true })
+
+    const fullyParsed = initalParse.data.filter(dataset => dataset.datasets !== '').map((dataset) => {
+      const { json, ...rest } = dataset
+      const formattedJson = JSON5.parse(json)
+      return {
+        ...rest,
+        json: formattedJson
+      }
+    })
+
+    const specifications = fullyParsed.reduce((accumulator, current) => {
+      const datasets = current.datasets.split(';')
+      datasets.forEach((dataset, index) => {
+        accumulator[dataset] = current.json[index]
+      })
+      return accumulator
+    }, {})
+
+    return specifications
+  } catch (error) {
+    console.error(error)
+  }
+}

From f4138123d08fb1e550d8706093c0e2acfbbd5df7 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 17 Sep 2024 17:34:49 +0100
Subject: [PATCH 02/65] update package lock

---
 package-lock.json | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/package-lock.json b/package-lock.json
index ba9d8aa6..cb91bb1e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -29,11 +29,13 @@
         "hmpo-form-wizard": "^13.0.0",
         "hmpo-i18n": "^6.0.1",
         "js-yaml": "^4.1.0",
+        "json5": "^2.2.3",
         "lodash": "^4.17.21",
         "maplibre-gl": "^4.1.0",
         "multer": "^1.4.5-lts.1",
         "notifications-node-client": "^8.2.0",
         "nunjucks": "^3.2.4",
+        "papaparse": "^5.4.1",
         "redis": "^4.6.13",
         "sass": "^1.69.4",
         "uuid": "^9.0.1",
@@ -8742,7 +8744,8 @@
     },
     "node_modules/json5": {
       "version": "2.2.3",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+      "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
       "bin": {
         "json5": "lib/cli.js"
       },
@@ -9947,6 +9950,11 @@
       "version": "1.0.0",
       "license": "BlueOak-1.0.0"
     },
+    "node_modules/papaparse": {
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz",
+      "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw=="
+    },
     "node_modules/parent-module": {
       "version": "1.0.1",
       "dev": true,

From d0ffb92169b5fc1c3dc443d06c20162b0b7b450c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 17 Sep 2024 17:35:55 +0100
Subject: [PATCH 03/65] now get and display endpoints, and correct number of
 fields and matched

---
 src/assets/scss/index.scss                    |   8 ++
 src/controllers/OrganisationsController.js    |  54 +++++++-
 src/routes/schemas.js                         |  14 +-
 src/services/DatasetService.js                | 108 ++++++----------
 src/services/performanceDbApi.js              |  23 +++-
 src/views/organisations/dataset-overview.html | 121 ++++++++++++++----
 test/unit/services/DatasetService.test.js     |   9 +-
 7 files changed, 231 insertions(+), 106 deletions(-)

diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss
index 3e1b6a16..07219dca 100644
--- a/src/assets/scss/index.scss
+++ b/src/assets/scss/index.scss
@@ -107,4 +107,12 @@ $govuk-global-styles: true;
 code,
 code * {
   font-family: monospace;
+}
+
+.padding-top {
+  padding-top: 20px !important;
+}
+
+.light-bottom-border {
+  border-bottom: 1px solid #e0e1e2ce !important;
 }
\ No newline at end of file
diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index acc931e2..969bd3ca 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -9,7 +9,7 @@ import { templateSchema } from '../routes/schemas.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 import config from '../../config/index.js'
-import { getDatasetStats, getLatestDatasetGeometryEntriesForLpa } from '../services/DatasetService.js'
+import { getFieldStats, getLatestDatasetGeometryEntriesForLpa, getSources } from '../services/DatasetService.js'
 
 // get a list of available datasets
 const availableDatasets = Object.values(dataSubjects)
@@ -131,7 +131,47 @@ const fetchDatasetGeometries = async (req, res, next) => {
 }
 
 const fetchDatasetStats = async (req, res, next) => {
-  req.stats = await getDatasetStats(req.params.dataset, req.params.lpa)
+  const { dataset, lpa } = req.params
+
+  const { numberOfFieldsSupplied, numberOfFieldsMatched, NumberOfExpectedFields } = await getFieldStats(lpa, dataset)
+
+  const sources = await getSources(lpa, dataset)
+
+  // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
+  const endpoints = sources.sort((a, b) => {
+    if (a.status >= 200 && a.status < 300) return -1
+    if (b.status >= 200 && b.status < 300) return 1
+    return 0
+  }).map((source, index) => {
+    let error
+
+    if (parseInt(source.status) <= 200 || parseInt(source.status) > 300) {
+      error = {
+        code: parseInt(source.status),
+        exception: source.exception
+      }
+    }
+
+    return {
+      name: `Data Url ${index}`,
+      endpoint: source.endpoint_url,
+      lastAccessed: source.latest_log_entry_date,
+      lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
+      error
+    }
+  })
+
+  const numberOfRecords = 10 // ToDo: await performanceDbApi.getEntityCount(lpa, dataset)
+
+  // ToDo: get the documentation url
+
+  req.stats = {
+    numberOfFieldsSupplied,
+    numberOfFieldsMatched,
+    NumberOfExpectedFields,
+    numberOfRecords,
+    endpoints
+  }
 
   next()
 }
@@ -139,7 +179,11 @@ const fetchDatasetStats = async (req, res, next) => {
 const getDatasetOverview = renderTemplate.bind(
   {
     templateParams (req) {
-      const { orgInfo: organisation, dataset, geometries, stats } = req
+      const {
+        orgInfo: organisation,
+        dataset, geometries,
+        stats
+      } = req
       return { organisation, dataset, geometries, stats }
     },
     template: 'organisations/dataset-overview.html',
@@ -262,7 +306,7 @@ async function fetchEntityCount (req, res, next) {
   const resourceId = passedResourceId ?? req.resourceId
   console.assert(resourceId, 'missng resourceId')
 
-  const entityCount = await performanceDbApi.getEntityCount(resourceId, datasetId)
+  const entityCount = await performanceDbApi.getEntityCountForResource(resourceId, datasetId)
   req.entityCount = entityCount
   next()
 }
@@ -659,7 +703,7 @@ const organisationsController = {
 
       const issues = await performanceDbApi.getLpaDatasetIssues(resource.resource, datasetId)
 
-      const entityCount = await performanceDbApi.getEntityCount(resource.resource, datasetId)
+      const entityCount = await performanceDbApi.getEntityCountForResource(resource.resource, datasetId)
 
       const taskList = issues.map((issue) => {
         return {
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index a2b13e85..7b1e89b9 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -73,7 +73,19 @@ export const OrgDatasetOverview = v.strictObject({
   geometries: v.array(v.string()),
   stats: v.strictObject({
     numberOfRecords: v.integer(),
-    numberOfFieldsSupplied: v.integer()
+    numberOfFieldsSupplied: v.integer(),
+    numberOfFieldsMatched: v.integer(),
+    NumberOfExpectedFields: v.integer(),
+    endpoints: v.array(v.strictObject({
+      name: v.string(),
+      endpoint: v.string(),
+      lastAccessed: v.string(), // ToDo: can we have date type on these?
+      lastUpdated: v.string(),
+      error: v.optional(v.strictObject({
+        code: v.integer(),
+        exception: v.string()
+      }))
+    }))
   })
 })
 
diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 856c5a03..4a1001e7 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -35,86 +35,41 @@ export async function getLatestDatasetGeometryEntriesForLpa (dataset, lpa) {
   }
 }
 
-/*
-  Needs to get:
-    - Number of records
-    - Number of fields supplied
-    - Number of fields matched
-    - Licence
-    - source documentation url
-    - for each data url
-      - the endpoint
-      - last accessed
-      - last updated
-      - any access errors
-
-*/
-async function getDatasetStatsForResourceId (dataset, resourceId) {
-  const sql = `
-    SELECT 'numberOfRecords' AS metric, COUNT(*) AS value
-    FROM
-      (
-        SELECT
-          *
-        FROM
-          fact_resource fr
-        WHERE
-          fr.resource = '${resourceId}'
-        GROUP BY
-          entry_number
-      )
-    UNION ALL
-    SELECT 'numberOfFieldsSupplied' AS metric, COUNT(*) AS value
-    FROM
-    (
-      SELECT
-        *
-      FROM
-        fact_resource fr
-      WHERE
-        fr.resource = '${resourceId}'
-    )`
+async function getColumnSummary (dataset, lpa) {
+  const sql = `select * from column_field_summary
+    where resource != ''
+    and pipeline = '${dataset}'
+    AND organisation = '${lpa}'
+    limit 1000`
 
-  const { formattedData } = await datasette.runQuery(sql, dataset)
+  const { formattedData } = await datasette.runQuery(sql, 'performance')
 
   return formattedData
 }
 
-// async function getColumnSummary (dataset, lpa) {
-//   const sql = `select * from column_field_summary
-//     where resource != ''
-//     and pipeline = '${dataset}'
-//     AND organisation = '${lpa}'
-//     limit 1000`
-
-//   const { formattedData } = await datasette.runQuery(sql, 'performance')
+export async function getFieldStats (lpa, dataset) {
+  const columnSummary = await getColumnSummary(dataset, lpa)
+  const specifications = await getSpecifications()
+  const datasetSpecification = specifications[dataset]
 
-//   return formattedData
-// }
-
-export async function getDatasetStats (dataset, lpa) {
-  try {
-    const stats = {}
-    const { resource: resourceId } = await performanceDbApi.getLatestResource(lpa, dataset)
-    // const columnSummary = await getColumnSummary(dataset, lpa)
+  const matchingFields = columnSummary[0].matching_field.split(',')
+  const nonMatchingFields = columnSummary[0].non_matching_field.split(',')
+  const allFields = [...matchingFields, ...nonMatchingFields]
 
-    const metrics = await getDatasetStatsForResourceId(dataset, resourceId)
+  const numberOfFieldsSupplied = datasetSpecification.fields.map(field => field.field).reduce((acc, current) => {
+    return allFields.includes(current) ? acc + 1 : acc
+  }, 0)
 
-    metrics.forEach(({ metric, value }) => {
-      stats[metric] = value
-    })
+  const numberOfFieldsMatched = datasetSpecification.fields.map(field => field.field).reduce((acc, current) => {
+    return matchingFields.includes(current) ? acc + 1 : acc
+  }, 0)
 
-    return stats
-  } catch (error) {
-    logger.warn(
-      `DatasetService.getDatasetStats(): Error getting dataset stats for ${lpa} in ${dataset}`,
-      {
-        errorMessage: error.message,
-        errorStack: error.stack
-      }
-    )
+  const NumberOfExpectedFields = datasetSpecification.fields.length
 
-    return {}
+  return {
+    numberOfFieldsSupplied,
+    numberOfFieldsMatched,
+    NumberOfExpectedFields
   }
 }
 
@@ -148,3 +103,16 @@ export async function getSpecifications () {
     console.error(error)
   }
 }
+
+export async function getSources (lpa, dataset) {
+  const sql = `
+    select endpoint, endpoint_url, status, exception, resource, latest_log_entry_date, endpoint_entry_date, endpoint_end_date, resource_start_date, resource_end_date
+    from reporting_historic_endpoints 
+    where REPLACE(organisation, '-eng', '') = '${lpa}' and pipeline = '${dataset}'
+    AND (resource_end_date >= current_timestamp OR resource_end_date is null)
+  `
+
+  const { formattedData } = await datasette.runQuery(sql, 'performance')
+
+  return formattedData
+}
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 0b914052..d830c7c2 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -394,7 +394,7 @@ ORDER BY
      * @param {string} dataset - The dataset to retrieve the entity count from.
      * @returns {number} The entity count for the given resource and dataset.
      */
-  async getEntityCount (resource, dataset) {
+  async getEntityCountForResource (resource, dataset) {
     const query =
     /* sql */
     `
@@ -405,6 +405,27 @@ ORDER BY
 
     const result = await datasette.runQuery(query, dataset)
 
+    return result.formattedData[0].entity_count
+  },
+
+  /**
+   * Retrieves the entity count for a given resource and dataset.
+   *
+   * @param {string} resource - The resource to retrieve the entity count for.
+   * @param {string} dataset - The dataset to retrieve the entity count from.
+   * @returns {number} The entity count for the given resource and dataset.
+   */
+  async getEntityCount (lpa, dataset) {
+    const query =
+    /* sql */
+    `
+    select dataset, entity_count, resource
+    from dataset_resource
+    WHERE resource = '${lpa}'
+  `
+
+    const result = await datasette.runQuery(query, dataset)
+
     return result.formattedData[0].entity_count
   }
 }
diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index ee9ebf5c..c4d45db1 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -33,6 +33,93 @@
 
 {% endblock %}
 
+{% set mainRows = [
+  {
+    key: {
+      text: "Number of records"
+    },
+    value: {
+      text: stats.numberOfRecords | default(0)
+    }
+  },
+  {
+    key: {
+      text: "Number of fields supplied"
+    },
+    value: {
+      text: stats.numberOfFieldsSupplied + '/' + stats.NumberOfExpectedFields | default(0)
+    }
+  },
+  {
+    key: {
+      text: "Number of fields matched"
+    },
+    value: {
+      text: stats.numberOfFieldsMatched + '/' + stats.NumberOfExpectedFields | default(0)
+    }
+  },
+  {
+    key: {
+      text: "License"
+    },
+    value: {
+      text: "To Do"
+    }
+  }  
+] %}
+
+{% set endpointRows = []%}
+
+{% for endpoint in stats.endpoints %}
+  {% set endpointRow = {
+    key: {
+      text: endpoint.name
+    },
+    value: {
+      text: endpoint.endpoint
+    },
+    classes: 'padding-top'
+  } %}
+  {{ endpointRows.push(endpointRow) }}
+
+
+
+  {% if endpoint.error %}
+    {% set lastAccessedRow = {
+      key: {
+        text: 'Data URL last accessed'
+      },
+      value: {
+        html: endpoint.lastAccessed + '<br><p class="govuk-error-message">There was a '+endpoint.error.code+' error accessing the data URL</p> <p class="app-inset-text__error"></p>'
+      },
+      classes: 'app-inset-text---error'
+    } %}
+  {% else %}
+    {% set lastAccessedRow = {
+      key: {
+        text: 'Data URL last accessed'
+      },
+      value: {
+        text: endpoint.lastAccessed
+      }
+    } %}
+  {% endif %}
+
+
+  {{ endpointRows.push(lastAccessedRow) }}
+
+  {% set lastUpdatedRow = {
+    key: {
+      text: 'Data URL last updated'
+    },
+    value: {
+      text: endpoint.lastUpdated
+    }
+  } %}
+  {{ endpointRows.push(lastUpdatedRow) }}
+
+{% endfor %}
+
 {% block content %}
 
 <div class="govuk-grid-row">
@@ -57,34 +144,12 @@ <h2 class="govuk-heading-m govuk-visually-hidden">Map of dataset</h2>
   <div class="govuk-grid-column-two-thirds">
     <section>
       <h2 class="govuk-heading-m">Dataset details</h2>
-      {{ govukSummaryList({
-        rows: [
-          {
-            key: {
-              text: "Number of records"
-            },
-            value: {
-              text: stats.numberOfRecords | default(0)
-            }
-          },
-          {
-            key: {
-              text: "Number of fields supplied"
-            },
-            value: {
-              text: stats.numberOfFieldsSupplied | default(0)
-            }
-          },
-          {
-            key: {
-              text: "License"
-            },
-            value: {
-              text: "Open Government Licence"
-            }
-          }
-        ]
-      }) }}
+      {{ govukSummaryList({ rows: mainRows }) }}
+
+      
+      <h2 class="govuk-heading-m">Endpoints</h2>
+
+      {{ govukSummaryList({ rows: endpointRows }) }}
     </section>
   </div>
 </div>
diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 1bfedd4e..9592ff19 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -1,5 +1,5 @@
 import { describe, it, expect, vi } from 'vitest'
-import { getGeometryEntriesForResourceId, getDatasetStats } from '../../../src/services/DatasetService.js'
+import { getGeometryEntriesForResourceId, getDatasetStats, getSpecifications } from '../../../src/services/DatasetService.js'
 import datasette from '../../../src/services/datasette'
 
 vi.mock('../../../src/services/datasette')
@@ -48,4 +48,11 @@ describe('DatasetService', () => {
       expect(result).toEqual({})
     })
   })
+
+  describe('getSpecifications', async () => {
+    it('can get specs', async () => {
+      const specifications = await getSpecifications()
+      expect(specifications).toBeDefined()
+    })
+  })
 })

From 9ef41de6ad68d15945202a280874bab6c54158b5 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 17 Sep 2024 17:50:28 +0100
Subject: [PATCH 04/65] number of records now correctly output

---
 src/controllers/OrganisationsController.js | 5 +++--
 src/routes/schemas.js                      | 2 +-
 src/services/performanceDbApi.js           | 8 ++++----
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 969bd3ca..04d90f87 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -104,7 +104,7 @@ const getGetStarted = renderTemplate.bind(
 
 const fetchOrgInfo = fetchOne.bind({
   query: ({ params }) => {
-    return `SELECT name, organisation FROM organisation WHERE organisation = '${params.lpa}'`
+    return `SELECT name, organisation, entity FROM organisation WHERE organisation = '${params.lpa}'`
   },
   result: 'orgInfo'
 })
@@ -132,6 +132,7 @@ const fetchDatasetGeometries = async (req, res, next) => {
 
 const fetchDatasetStats = async (req, res, next) => {
   const { dataset, lpa } = req.params
+  const { orgInfo: organisation } = req
 
   const { numberOfFieldsSupplied, numberOfFieldsMatched, NumberOfExpectedFields } = await getFieldStats(lpa, dataset)
 
@@ -161,7 +162,7 @@ const fetchDatasetStats = async (req, res, next) => {
     }
   })
 
-  const numberOfRecords = 10 // ToDo: await performanceDbApi.getEntityCount(lpa, dataset)
+  const numberOfRecords = await performanceDbApi.getEntityCount(organisation.entity, dataset)
 
   // ToDo: get the documentation url
 
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 7b1e89b9..73f743a4 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -38,7 +38,7 @@ const datasetStatusEnum = {
   'Not submitted': 'Not submitted'
 }
 
-const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(NonEmptyString) })
+const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(NonEmptyString), entity: v.optional(v.integer()) })
 const DatasetNameField = v.strictObject({ name: NonEmptyString })
 
 export const OrgOverviewPage = v.strictObject({
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index d830c7c2..c78e9c81 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -415,13 +415,13 @@ ORDER BY
    * @param {string} dataset - The dataset to retrieve the entity count from.
    * @returns {number} The entity count for the given resource and dataset.
    */
-  async getEntityCount (lpa, dataset) {
+  async getEntityCount (orgEntity, dataset) {
     const query =
     /* sql */
     `
-    select dataset, entity_count, resource
-    from dataset_resource
-    WHERE resource = '${lpa}'
+    select count(entity) as entity_count 
+    from entity
+    WHERE organisation_entity = '${orgEntity}'
   `
 
     const result = await datasette.runQuery(query, dataset)

From f4750f268068c270dae7451eb32a7ead313dc036 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Tue, 17 Sep 2024 17:53:05 +0100
Subject: [PATCH 05/65] for now just have static Open Government Licence

---
 src/views/organisations/dataset-overview.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index c4d45db1..54f66cc5 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -63,7 +63,7 @@
       text: "License"
     },
     value: {
-      text: "To Do"
+      text: "Open Government Licence" 
     }
   }  
 ] %}

From 5343714a1a07c8a5464a80efbbbcd68e61ca42c7 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 09:41:55 +0100
Subject: [PATCH 06/65] entityMerge remote-tracking branch 'origin/main' into
 407-dataset-details-summary-table

---
 src/assets/js/map.js                          | 237 ++++++++++++++----
 src/controllers/OrganisationsController.js    |  21 +-
 src/routes/schemas.js                         |   6 +-
 src/views/organisations/dataset-overview.html |  18 +-
 src/views/organisations/overview.html         |   2 +-
 test/unit/services/DatasetService.test.js     |  23 +-
 .../organisations/dataset-overview.test.js    |  13 +-
 7 files changed, 224 insertions(+), 96 deletions(-)

diff --git a/src/assets/js/map.js b/src/assets/js/map.js
index 09a88ce0..e3363cc0 100644
--- a/src/assets/js/map.js
+++ b/src/assets/js/map.js
@@ -1,22 +1,66 @@
 import parse from 'wellknown'
 import maplibregl from 'maplibre-gl'
 
+const fillColor = '#008'
+const lineColor = '#000000'
+const boundaryLineColor = '#000000'
+const opacity = 0.4
+
+/**
+ * Creates a Map instance.
+ * @param {MapOptions} opts - The options for creating the map.
+ * @constructor
+ */
+/**
+ * Options for creating a Map instance.
+ * @typedef {Object} MapOptions
+ * @property {string} containerId - Required - The ID of the HTML container element for the map.
+ * @property {string[]} data - Required - An array of URLs or WKT geometries to be added to the map.
+ * @property {string} [boundaryGeoJsonUrl] - Optional - The URL of the boundary GeoJSON to be added to the map.
+ * @property {boolean} [interactive] - Optional - Indicates whether the map should be interactive. Default is true.
+ * @property {boolean} [wktFormat] - Optional - Indicates whether the data is in WKT format. Default is false.
+ * @property {number[]} [boundingBox] - Optional - The bounding box coordinates [minX, minY, maxX, maxY] to set the initial view of the map.
+ */
 class Map {
-  constructor (containerId, geometries, interactive = true) {
+  constructor (opts) {
+    this.bbox = opts.boundingBox ?? null
     this.map = new maplibregl.Map({
-      container: containerId,
+      container: opts.containerId,
       style: 'https://api.maptiler.com/maps/basic-v2/style.json?key=ncAXR9XEn7JgHBLguAUw',
       zoom: 11,
       center: [-0.1298779, 51.4959698],
-      interactive
+      interactive: opts.interactive ?? true
     })
 
     this.map.on('load', () => {
-      this.addDataToMap(geometries)
+      // Store the first symbol layer id
+      this.setFirstMapLayerId()
+
+      // Add the boundary GeoJSON to the map
+      if (opts.boundaryGeoJsonUrl) this.addBoundaryGeoJsonToMap(opts.boundaryGeoJsonUrl)
+
+      // Add layer data to map
+      if (opts.wktFormat) this.addWktDataToMap(opts.data)
+      else this.addGeoJsonUrlsToMap(opts.data)
+
+      // Move the map to the bounding box
+      if (this.bbox) this.setMapViewToBoundingBox()
     })
   }
 
-  addDataToMap (geometriesWkt) {
+  setFirstMapLayerId () {
+    const layers = this.map.getStyle().layers
+
+    // Find the index of the first symbol layer in the map style
+    for (let i = 0; i < layers.length; i++) {
+      if (layers[i].type === 'symbol') {
+        this.firstMapLayerId = layers[i].id
+        break
+      }
+    }
+  }
+
+  addWktDataToMap (geometriesWkt) {
     const geometries = []
     geometriesWkt.forEach((geometryWkt, index) => {
       const name = `geometry-${index}`
@@ -31,10 +75,6 @@ class Map {
         data: geometry
       })
 
-      const color = '#008'
-      const lineColor = '#000000'
-      const opacity = 0.4
-
       // Add a layer to the map based on the geometry type
       if (geometry.type === 'Polygon' || geometry.type === 'MultiPolygon') {
         this.map.addLayer({
@@ -43,10 +83,10 @@ class Map {
           source: name,
           layout: {},
           paint: {
-            'fill-color': color,
+            'fill-color': fillColor,
             'fill-opacity': opacity
           }
-        })
+        }, this.firstMapLayerId)
 
         this.map.addLayer({
           id: name + '-border',
@@ -57,7 +97,7 @@ class Map {
             'line-color': lineColor,
             'line-width': 1
           }
-        })
+        }, this.firstMapLayerId)
       } else if (geometry.type === 'Point' || geometry.type === 'MultiPoint') {
         this.map.addLayer({
           id: name,
@@ -65,15 +105,65 @@ class Map {
           source: name,
           paint: {
             'circle-radius': 10,
-            'circle-color': color,
+            'circle-color': fillColor,
             'circle-opacity': opacity
           }
-        })
+        }, this.firstMapLayerId)
       }
     })
 
     this.bbox = this.calculateBoundingBoxFromGeometries(geometries.map(g => g.coordinates))
-    this.setMapViewToBoundingBox()
+  }
+
+  addGeoJsonUrlsToMap (geoJsonUrls) {
+    geoJsonUrls.forEach(async (url, index) => {
+      const name = `geometry-${index}`
+      this.map.addSource(name, {
+        type: 'geojson',
+        data: url
+      })
+
+      this.map.addLayer({
+        id: name,
+        type: 'fill',
+        source: name,
+        layout: {},
+        paint: {
+          'fill-color': fillColor,
+          'fill-opacity': opacity
+        }
+      }, this.firstMapLayerId)
+
+      this.map.addLayer({
+        id: `${name}-border`,
+        type: 'line',
+        source: name,
+        layout: {},
+        paint: {
+          'line-color': lineColor,
+          'line-width': 1
+        }
+      }, this.firstMapLayerId)
+    })
+  }
+
+  addBoundaryGeoJsonToMap (geoJsonUrl) {
+    this.map.addSource('boundary', {
+      type: 'geojson',
+      data: geoJsonUrl
+    })
+
+    this.map.addLayer({
+      id: 'boundary',
+      type: 'line',
+      source: 'boundary',
+      layout: {},
+      paint: {
+        'line-color': boundaryLineColor,
+        'line-width': 2,
+        'line-opacity': opacity
+      }
+    }, this.firstMapLayerId)
   }
 
   setMapViewToBoundingBox () {
@@ -86,51 +176,106 @@ class Map {
       zoom: 11
     })
   }
+}
 
-  calculateBoundingBoxFromGeometries (geometries) {
-    let minX = Infinity
-    let minY = Infinity
-    let maxX = -Infinity
-    let maxY = -Infinity
-
-    const pullOutCoordinates = (geometry) => {
-      if (Array.isArray(geometry[0])) {
-        geometry.forEach(pullOutCoordinates)
-      } else {
-        const [x, y] = geometry
-
-        // if x or y isn't a valid number log an error and continue
-        if (isNaN(x) || isNaN(y)) {
-          console.error('Invalid coordinates', x, y)
-          return
-        }
+const calculateBoundingBoxFromGeometries = (geometries) => {
+  let minX = Infinity
+  let minY = Infinity
+  let maxX = -Infinity
+  let maxY = -Infinity
 
-        minX = Math.min(minX, x)
-        minY = Math.min(minY, y)
-        maxX = Math.max(maxX, x)
-        maxY = Math.max(maxY, y)
+  const pullOutCoordinates = (geometry) => {
+    if (Array.isArray(geometry[0])) {
+      geometry.forEach(pullOutCoordinates)
+    } else {
+      const [x, y] = geometry
+
+      // if x or y isn't a valid number log an error and continue
+      if (isNaN(x) || isNaN(y)) {
+        console.error('Invalid coordinates', x, y)
+        return
       }
+
+      minX = Math.min(minX, x)
+      minY = Math.min(minY, y)
+      maxX = Math.max(maxX, x)
+      maxY = Math.max(maxY, y)
     }
+  }
+
+  pullOutCoordinates(geometries)
 
-    pullOutCoordinates(geometries)
+  // Return the bounding box
+  return [[minX, minY], [maxX, maxY]]
+}
 
-    // Return the bounding box
-    return [[minX, minY], [maxX, maxY]]
+const generatePaginatedGeoJsonLinks = async (geoJsonUrl) => {
+  const geoJsonLinks = [geoJsonUrl]
+  const initialResponse = await fetch(geoJsonUrl)
+  const initialData = await initialResponse.json()
+
+  // return if no pagination is needed
+  if (!initialData.links || !initialData.links.last) {
+    return geoJsonLinks
+  }
+
+  const lastLink = new URL(initialData.links.last)
+  const limit = parseInt(lastLink.searchParams.get('limit'))
+  const lastOffset = parseInt(lastLink.searchParams.get('offset'))
+
+  if (!limit || !lastOffset) {
+    console.error('Invalid pagination links', lastLink)
+    return geoJsonLinks
   }
+
+  // create a loop to generate the links
+  for (let offset = limit; offset < lastOffset; offset += limit) {
+    const newLink = new URL(geoJsonUrl)
+    newLink.searchParams.set('offset', offset)
+
+    geoJsonLinks.push(newLink.toString())
+  }
+
+  return geoJsonLinks
+}
+
+const generateBoundingBox = async (boundaryGeoJsonUrl) => {
+  const res = await fetch(boundaryGeoJsonUrl)
+  const boundaryGeoJson = await res.json()
+
+  return calculateBoundingBoxFromGeometries(boundaryGeoJson.features[0].geometry.coordinates)
 }
 
-const createMapFromServerContext = () => {
-  const { containerId, geometries, mapType } = window.serverContext
+const createMapFromServerContext = async () => {
+  const { containerId, geometries, mapType, geoJsonUrl, boundaryGeoJsonUrl } = window.serverContext
+  const options = {
+    containerId,
+    data: geometries,
+    boundaryGeoJsonUrl,
+    interactive: mapType !== 'static',
+    wktFormat: geoJsonUrl === undefined
+  }
+
+  // if the geoJsonUrl is provided, generate the paginated GeoJSON links
+  if (geoJsonUrl) {
+    options.data = await generatePaginatedGeoJsonLinks(geoJsonUrl)
+  }
+
+  if (options.boundaryGeoJsonUrl) {
+    options.boundingBox = await generateBoundingBox(boundaryGeoJsonUrl)
+  }
 
   // if any of the required properties are missing, return null
-  if (!containerId || !geometries) {
+  if (!options.containerId || !options.data) {
     console.log('Missing required properties (containerId, geometries) on window.serverContext', window.serverContext)
     return null
   }
 
-  return new Map(containerId, geometries, mapType !== 'static')
+  return new Map(options)
 }
 
-const newMap = createMapFromServerContext()
-
-window.map = newMap
+try {
+  window.map = await createMapFromServerContext()
+} catch (error) {
+  console.error('Error creating map', error)
+}
diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 046691b6..3809d370 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -7,7 +7,7 @@ import { statusToTagClass } from '../filters/filters.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 import config from '../../config/index.js'
-import { getFieldStats, getLatestDatasetGeometryEntriesForLpa, getSources } from '../services/DatasetService.js'
+import { getFieldStats, getSources } from '../services/DatasetService.js'
 import {
   fetchOne,
   fetchMany,
@@ -35,7 +35,7 @@ const getGetStarted = renderTemplate.bind({
 
 const fetchOrgInfo = fetchOne.bind({
   query: ({ params }) => {
-    return `SELECT name, organisation, entity FROM organisation WHERE organisation = '${params.lpa}'`
+    return `SELECT name, organisation, entity, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
   },
   result: 'orgInfo'
 })
@@ -54,13 +54,6 @@ const fetchDatasetInfo = fetchOne.bind({
   result: 'dataset'
 })
 
-const fetchDatasetGeometries = async (req, res, next) => {
-  const datasetEntries = await getLatestDatasetGeometryEntriesForLpa(req.params.dataset, req.params.lpa)
-  req.geometries = datasetEntries.map(entry => entry.value)
-
-  next()
-}
-
 const fetchDatasetStats = async (req, res, next) => {
   const { dataset, lpa } = req.params
   const { orgInfo: organisation } = req
@@ -111,12 +104,8 @@ const fetchDatasetStats = async (req, res, next) => {
 const getDatasetOverview = renderTemplate.bind(
   {
     templateParams (req) {
-      const {
-        orgInfo: organisation,
-        dataset, geometries,
-        stats
-      } = req
-      return { organisation, dataset, geometries, stats }
+      const { orgInfo: organisation, dataset, stats } = req
+      return { organisation, dataset, stats }
     },
     template: 'organisations/dataset-overview.html',
     handlerName: 'datasetOverview'
@@ -549,7 +538,7 @@ const getGetStartedMiddleware = [
   logPageError
 ]
 
-const getDatasetOverviewMiddleware = [fetchOrgInfo, fetchDatasetName, fetchDatasetGeometries, fetchDatasetStats, getDatasetOverview, logPageError]
+const getDatasetOverviewMiddleware = [fetchOrgInfo, fetchDatasetInfo, fetchDatasetStats, getDatasetOverview, logPageError]
 
 const getOverviewMiddleware = [
   fetchOrgInfo,
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 73f743a4..6be66582 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -69,8 +69,10 @@ export const OrgGetStarted = v.strictObject({
 
 export const OrgDatasetOverview = v.strictObject({
   organisation: OrgField,
-  dataset: DatasetNameField,
-  geometries: v.array(v.string()),
+  dataset: v.strictObject({
+    name: NonEmptyString,
+    dataset: NonEmptyString
+  }),
   stats: v.strictObject({
     numberOfRecords: v.integer(),
     numberOfFieldsSupplied: v.integer(),
diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index 54f66cc5..2f0b9ec0 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -2,6 +2,15 @@
 {% from "govuk/components/summary-list/macro.njk" import govukSummaryList %}
 {% extends "layouts/main.html" %}
 
+{% set showMap = [
+  "article-4-direction-area",
+  "conservation-area",
+  "listed-building",
+  "listed-building-outline",
+  "tree",
+  "tree-preservation-zone"
+].includes(dataset.dataset) %}
+
 {% set serviceType = 'Submit' %}
 {% set pageName %}
     {{organisation.name}} - {{dataset.name}} - Dataset overview
@@ -129,7 +138,7 @@ <h1 class="govuk-heading-xl">{{ dataset.name }}</h1>
     </div>
 </div>
 
-{% if geometries and geometries.length %}
+{% if showMap %}
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-full">
       <section>
@@ -158,14 +167,15 @@ <h2 class="govuk-heading-m">Endpoints</h2>
 
 {% block scripts %}
   {{ super() }}
-  {% if geometries and geometries.length %}
+  {% if showMap %}
     <link href='https://unpkg.com/maplibre-gl@latest/dist/maplibre-gl.css' rel='stylesheet' />
     <script>
       window.serverContext = {
         containerId: 'map',
-        geometries: {{ geometries | dump | safe }},
+        geoJsonUrl: "https://www.planning.data.gov.uk/entity.geojson?dataset={{ dataset.dataset }}&geometry_curie=statistical-geography:{{ organisation.statistical_geography }}&limit=100",
+        boundaryGeoJsonUrl: "https://www.planning.data.gov.uk/entity.geojson?dataset=local-planning-authority&geometry_curie=statistical-geography:{{ organisation.statistical_geography }}"
       }
     </script>
     <script src="/public/js/map.bundle.js"></script>
   {% endif %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html
index 93a5323a..bd200bcd 100644
--- a/src/views/organisations/overview.html
+++ b/src/views/organisations/overview.html
@@ -69,7 +69,7 @@ <h2 class="govuk-heading-m">Datasets</h2>
         <li class="govuk-task-list__item govuk-task-list__item--with-link">
           <div class="govuk-task-list__name-and-hint">
             <h2 class="govuk-heading-m">
-                {% if dataset.status === "Live" and not ("tree-preservation-order" in dataset.slug or "tree-preservation-zone" in dataset.slug or "tree" in dataset.slug) %}
+                {% if dataset.status === "Live" %}
                 <a class="govuk-link govuk-task-list__link"
                   href="/organisations/{{ organisation.organisation }}/{{dataset.slug}}/overview">
                   {{dataset.slug | datasetSlugToReadableName}}
diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 9592ff19..1ab49626 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -1,31 +1,10 @@
 import { describe, it, expect, vi } from 'vitest'
-import { getGeometryEntriesForResourceId, getDatasetStats, getSpecifications } from '../../../src/services/DatasetService.js'
+import { getDatasetStats, getSpecifications } from '../../../src/services/DatasetService.js'
 import datasette from '../../../src/services/datasette'
 
 vi.mock('../../../src/services/datasette')
 
 describe('DatasetService', () => {
-  describe('getGeometryEntriesForResourceId', () => {
-    it('should return geometry entries for a given resource ID', async () => {
-      const mockData = {
-        formattedData: [
-          { rowid: 1, end_date: '2023-01-01', fact: 'fact1', entry_date: '2023-01-01', entry_number: 1, resource: 'resource1', start_date: '2023-01-01', entity: 'entity1', field: 'geometry', value: 'value1' }
-        ]
-      }
-      datasette.runQuery.mockResolvedValue(mockData)
-
-      const result = await getGeometryEntriesForResourceId('dataset1', 'resource1')
-      expect(result).toEqual(mockData.formattedData)
-    })
-
-    it('should return an empty array if no data is found', async () => {
-      datasette.runQuery.mockResolvedValue({ formattedData: [] })
-
-      const result = await getGeometryEntriesForResourceId('dataset1', 'resource1')
-      expect(result).toEqual([])
-    })
-  })
-
   describe('getDatasetStats', () => {
     it('should return dataset stats for a given LPA', async () => {
       const mockStats = [
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index db77200c..867d88dd 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -11,13 +11,13 @@ describe('Dataset Overview Page', () => {
       organisation: 'mock-org'
     },
     dataset: {
+      dataset: 'world-heritage-site-buffer-zone',
       name: 'World heritage site buffer zone'
     },
     stats: {
       numberOfRecords: 10,
       numberOfFieldsSupplied: 5
-    },
-    geometries: []
+    }
   }
 
   const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
@@ -52,14 +52,17 @@ describe('Dataset Overview Page', () => {
     expect(breadcrumbs[3].textContent).toContain('World heritage site buffer zone')
   })
 
-  it('Does not render the map section when geometries are absent', () => {
+  it('Does not render the map section when non mappable dataset is viewed', () => {
     expect(document.querySelector('#map')).toBeNull()
   })
 
-  it('Renders the map section when geometries are present', () => {
+  it('Renders the map section when a mappable dataset is viewed', () => {
     const paramsWithGeometries = {
       ...params,
-      geometries: ['MULTIPOLYGON (((-0.1275 51.5072, -0.1275 51.5072, -0.1275 51.5072, -0.1275 51.5072)))']
+      dataset: {
+        dataset: 'article-4-direction-area',
+        name: 'Article 4 direction area'
+      }
     }
     const htmlWithGeometries = stripWhitespace(nunjucks.render('organisations/dataset-overview.html', paramsWithGeometries))
     const domWithGeometries = new jsdom.JSDOM(htmlWithGeometries)

From a1b35e66968c931d7b69bbe996e93e0b0d163b95 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 09:42:16 +0100
Subject: [PATCH 07/65] change schema to accept empty string statistical
 geometries

---
 src/routes/schemas.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 6be66582..1d308fe6 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -38,7 +38,7 @@ const datasetStatusEnum = {
   'Not submitted': 'Not submitted'
 }
 
-const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(NonEmptyString), entity: v.optional(v.integer()) })
+const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
 const DatasetNameField = v.strictObject({ name: NonEmptyString })
 
 export const OrgOverviewPage = v.strictObject({

From 2faa985bcf0a2c846f84089c13d76066f4c10acd Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 09:51:47 +0100
Subject: [PATCH 08/65] add margin to top of each data url row

---
 src/assets/scss/index.scss                    |  2 +-
 src/views/organisations/dataset-overview.html | 26 +++++++------------
 2 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss
index 07219dca..05927de1 100644
--- a/src/assets/scss/index.scss
+++ b/src/assets/scss/index.scss
@@ -110,7 +110,7 @@ code * {
 }
 
 .padding-top {
-  padding-top: 20px !important;
+  padding-top: 40px;
 }
 
 .light-bottom-border {
diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index 2f0b9ec0..3e96b036 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -42,7 +42,7 @@
 
 {% endblock %}
 
-{% set mainRows = [
+{% set rows = [
   {
     key: {
       text: "Number of records"
@@ -77,31 +77,30 @@
   }  
 ] %}
 
-{% set endpointRows = []%}
-
 {% for endpoint in stats.endpoints %}
   {% set endpointRow = {
     key: {
-      text: endpoint.name
+      text: 'Data URL',
+      classes: 'padding-top'
     },
     value: {
       text: endpoint.endpoint
     },
     classes: 'padding-top'
   } %}
-  {{ endpointRows.push(endpointRow) }}
+  {{ rows.push(endpointRow) }}
 
 
 
   {% if endpoint.error %}
     {% set lastAccessedRow = {
       key: {
-        text: 'Data URL last accessed'
+        text: 'Data URL last accessed',
+        classes: 'app-inset-text---error'
       },
       value: {
         html: endpoint.lastAccessed + '<br><p class="govuk-error-message">There was a '+endpoint.error.code+' error accessing the data URL</p> <p class="app-inset-text__error"></p>'
-      },
-      classes: 'app-inset-text---error'
+      }
     } %}
   {% else %}
     {% set lastAccessedRow = {
@@ -115,7 +114,7 @@
   {% endif %}
 
 
-  {{ endpointRows.push(lastAccessedRow) }}
+  {{ rows.push(lastAccessedRow) }}
 
   {% set lastUpdatedRow = {
     key: {
@@ -125,7 +124,7 @@
       text: endpoint.lastUpdated
     }
   } %}
-  {{ endpointRows.push(lastUpdatedRow) }}
+  {{ rows.push(lastUpdatedRow) }}
 
 {% endfor %}
 
@@ -153,12 +152,7 @@ <h2 class="govuk-heading-m govuk-visually-hidden">Map of dataset</h2>
   <div class="govuk-grid-column-two-thirds">
     <section>
       <h2 class="govuk-heading-m">Dataset details</h2>
-      {{ govukSummaryList({ rows: mainRows }) }}
-
-      
-      <h2 class="govuk-heading-m">Endpoints</h2>
-
-      {{ govukSummaryList({ rows: endpointRows }) }}
+      {{ govukSummaryList({ rows: rows }) }}
     </section>
   </div>
 </div>

From 480a53727b1bb8c26ea62fe7f28ea5b21e8bd506 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 11:15:09 +0100
Subject: [PATCH 09/65] get spec from datasette

---
 src/services/DatasetService.js | 47 +++++-----------------------------
 1 file changed, 6 insertions(+), 41 deletions(-)

diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 4a1001e7..88c1d093 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -1,40 +1,7 @@
 import datasette from './datasette.js'
 import logger from '../utils/logger.js'
-import performanceDbApi from './performanceDbApi.js'
-import Papa from 'papaparse'
 import JSON5 from 'json5' // need to use this as spec is written with single quotes. need to pass this onto data designers team to fix
 
-export async function getGeometryEntriesForResourceId (dataset, resourceId) {
-  const sql = `
-      SELECT ft.field, ft.value
-      FROM fact_resource fr
-      LEFT JOIN fact ft ON fr.fact = ft.fact
-      WHERE fr.resource = '${resourceId}'
-      AND ft.field = 'geometry'`
-
-  const { formattedData } = await datasette.runQuery(sql, dataset)
-
-  return formattedData
-}
-
-export async function getLatestDatasetGeometryEntriesForLpa (dataset, lpa) {
-  try {
-    const { resource: resourceId } = await performanceDbApi.getLatestResource(lpa, dataset)
-
-    return getGeometryEntriesForResourceId(dataset, resourceId)
-  } catch (error) {
-    logger.warn(
-      `DatasetService.getLatestDatasetGeometryEntriesForLpa(): Error getting geometry entries for ${lpa} in ${dataset}`,
-      {
-        errorMessage: error.message,
-        errorStack: error.stack
-      }
-    )
-
-    return []
-  }
-}
-
 async function getColumnSummary (dataset, lpa) {
   const sql = `select * from column_field_summary
     where resource != ''
@@ -75,13 +42,10 @@ export async function getFieldStats (lpa, dataset) {
 
 export async function getSpecifications () {
   try {
-    const response = await fetch('https://raw.githubusercontent.com/digital-land/specification/main/specification/specification.csv')
-
-    const csvData = await response.text()
-
-    const initalParse = Papa.parse(csvData, { header: true })
+    const sql = 'select * from specification order by specification'
+    const result = await datasette.runQuery(sql)
 
-    const fullyParsed = initalParse.data.filter(dataset => dataset.datasets !== '').map((dataset) => {
+    const resultWithParsedJson = result.formattedData.filter(dataset => dataset.datasets !== '').map((dataset) => {
       const { json, ...rest } = dataset
       const formattedJson = JSON5.parse(json)
       return {
@@ -90,7 +54,7 @@ export async function getSpecifications () {
       }
     })
 
-    const specifications = fullyParsed.reduce((accumulator, current) => {
+    const specifications = resultWithParsedJson.reduce((accumulator, current) => {
       const datasets = current.datasets.split(';')
       datasets.forEach((dataset, index) => {
         accumulator[dataset] = current.json[index]
@@ -100,7 +64,8 @@ export async function getSpecifications () {
 
     return specifications
   } catch (error) {
-    console.error(error)
+    // ToDo: handle this error
+    logger.error(error) // ToDo: make this error appropriate
   }
 }
 

From 197f4f2b7d66b08587e9fa97da90affa527c7a62 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 12:00:42 +0100
Subject: [PATCH 10/65] add case for no dataset in specification

---
 src/services/DatasetService.js | 49 +++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 88c1d093..110ab5d0 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -1,5 +1,6 @@
 import datasette from './datasette.js'
 import logger from '../utils/logger.js'
+import { types } from '../utils/logging.js'
 import JSON5 from 'json5' // need to use this as spec is written with single quotes. need to pass this onto data designers team to fix
 
 async function getColumnSummary (dataset, lpa) {
@@ -17,6 +18,11 @@ async function getColumnSummary (dataset, lpa) {
 export async function getFieldStats (lpa, dataset) {
   const columnSummary = await getColumnSummary(dataset, lpa)
   const specifications = await getSpecifications()
+  if (!(dataset in specifications)) {
+    logger.warn(`services/datasetService.getFieldStats(): cannot find specification for dataset: ${dataset}`, { type: types.app, dataset })
+    return null
+  }
+
   const datasetSpecification = specifications[dataset]
 
   const matchingFields = columnSummary[0].matching_field.split(',')
@@ -41,32 +47,27 @@ export async function getFieldStats (lpa, dataset) {
 }
 
 export async function getSpecifications () {
-  try {
-    const sql = 'select * from specification order by specification'
-    const result = await datasette.runQuery(sql)
-
-    const resultWithParsedJson = result.formattedData.filter(dataset => dataset.datasets !== '').map((dataset) => {
-      const { json, ...rest } = dataset
-      const formattedJson = JSON5.parse(json)
-      return {
-        ...rest,
-        json: formattedJson
-      }
+  const sql = 'select * from specification order by specification'
+  const result = await datasette.runQuery(sql)
+
+  const resultWithParsedJson = result.formattedData.filter(dataset => dataset.datasets !== '').map((dataset) => {
+    const { json, ...rest } = dataset
+    const formattedJson = JSON5.parse(json)
+    return {
+      ...rest,
+      json: formattedJson
+    }
+  })
+
+  const specifications = resultWithParsedJson.reduce((accumulator, current) => {
+    const datasets = current.datasets.split(';')
+    datasets.forEach((dataset, index) => {
+      accumulator[dataset] = current.json[index]
     })
+    return accumulator
+  }, {})
 
-    const specifications = resultWithParsedJson.reduce((accumulator, current) => {
-      const datasets = current.datasets.split(';')
-      datasets.forEach((dataset, index) => {
-        accumulator[dataset] = current.json[index]
-      })
-      return accumulator
-    }, {})
-
-    return specifications
-  } catch (error) {
-    // ToDo: handle this error
-    logger.error(error) // ToDo: make this error appropriate
-  }
+  return specifications
 }
 
 export async function getSources (lpa, dataset) {

From 43e4e90636b94b46b96e748b5e5794d9b2c23dcf Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 12:00:53 +0100
Subject: [PATCH 11/65] add unit tests for datasetService

---
 test/unit/services/DatasetService.test.js | 84 ++++++++++++++++-------
 1 file changed, 61 insertions(+), 23 deletions(-)

diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 1ab49626..673a64e9 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -1,37 +1,75 @@
-import { describe, it, expect, vi } from 'vitest'
-import { getDatasetStats, getSpecifications } from '../../../src/services/DatasetService.js'
-import datasette from '../../../src/services/datasette'
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { getFieldStats, getSpecifications } from '../../../src/services/DatasetService.js'
+import datasette from '../../../src/services/datasette.js'
 
 vi.mock('../../../src/services/datasette')
 
 describe('DatasetService', () => {
-  describe('getDatasetStats', () => {
-    it('should return dataset stats for a given LPA', async () => {
-      const mockStats = [
-        { metric: 'numberOfRecords', value: 10 },
-        { metric: 'numberOfFieldsSupplied', value: 5 }
-      ]
-      datasette.runQuery.mockResolvedValue({ formattedData: mockStats })
-
-      const result = await getDatasetStats('dataset1', 'lpa1')
-      expect(result).toEqual({
-        numberOfRecords: 10,
-        numberOfFieldsSupplied: 5
+  describe('getSpecifications', () => {
+    beforeEach(() => {
+      datasette.runQuery.mockReturnValue({
+        formattedData: [
+          {
+            datasets: 'dataset1',
+            json: "[{'key': 'value'}]"
+          },
+          {
+            datasets: 'dataset2;dataset3',
+            json: "[{'key2': 'value2'},{'key3': 'value3'}]"
+          },
+          {
+            datasets: 'dataset4',
+            json: "[{'key4': 'value4'}]"
+          }
+        ]
       })
     })
 
-    it('should return an empty object if an error occurs', async () => {
-      datasette.runQuery.mockRejectedValue(new Error('Test error'))
+    const expectedSpecifications = {
+      dataset1: { key: 'value' },
+      dataset2: { key2: 'value2' },
+      dataset3: { key3: 'value3' },
+      dataset4: { key4: 'value4' }
+    }
 
-      const result = await getDatasetStats('dataset1', 'lpa1')
-      expect(result).toEqual({})
+    it('returns specifications in the correct format', async () => {
+      const specifications = await getSpecifications()
+      expect(specifications).toEqual(expectedSpecifications)
     })
   })
 
-  describe('getSpecifications', async () => {
-    it('can get specs', async () => {
-      const specifications = await getSpecifications()
-      expect(specifications).toBeDefined()
+  describe('getFieldStats', () => {
+    const columnSummary = [
+      {
+        matching_field: 'field1,field2',
+        non_matching_field: 'field3,field4'
+      }
+    ]
+
+    const specifications = [
+      {
+        datasets: 'dataset1',
+        json: '[{fields: [{field: "field1"},{field: "field3"}]}]'
+      }
+    ]
+
+    beforeEach(() => {
+      vi.mocked(datasette.runQuery).mockResolvedValueOnce({ formattedData: columnSummary })
+      vi.mocked(datasette.runQuery).mockResolvedValueOnce({ formattedData: specifications })
+    })
+
+    it('returns field stats', async () => {
+      const fieldStats = await getFieldStats('lpa', 'dataset1')
+      expect(fieldStats).toEqual({
+        numberOfFieldsSupplied: 2,
+        numberOfFieldsMatched: 1,
+        NumberOfExpectedFields: 2
+      })
+    })
+
+    it('returns null if dataset specification is missing', async () => {
+      const fieldStats = await getFieldStats('lpa', 'nonExistentDataset')
+      expect(fieldStats).toBeNull()
     })
   })
 })

From f05a963bb7c5b17f8aaa162852e06154e819c34c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 13:58:56 +0100
Subject: [PATCH 12/65] pre-merge

---
 src/controllers/OrganisationsController.js | 435 ++++++++++-----------
 src/controllers/middleware.js              | 167 +++++++-
 src/routes/organisations.js                |   2 +-
 src/routes/schemas.js                      |  12 +-
 src/services/performanceDbApi.js           |  63 ++-
 src/utils/custom-renderer.js               |   8 +-
 src/utils/logger.js                        |   8 +
 test/unit/http-errorPage.test.js           |   3 +-
 test/unit/organisationsController.test.js  | 291 +++-----------
 9 files changed, 471 insertions(+), 518 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 3809d370..7d8f42b8 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -1,4 +1,3 @@
-import datasette from '../services/datasette.js'
 import performanceDbApi from '../services/performanceDbApi.js' // Assume you have an API service module
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
@@ -9,12 +8,14 @@ import { pagination } from '../utils/pagination.js'
 import config from '../../config/index.js'
 import { getFieldStats, getSources } from '../services/DatasetService.js'
 import {
-  fetchOne,
+  fetchIf,
   fetchMany,
+  fetchOne,
   logPageError,
-  maybeFetch,
+  parallel,
   renderTemplate,
-  validateAndRender
+  FetchOptions,
+  FetchOneFallbackPolicy
 } from './middleware.js'
 
 // get a list of available datasets
@@ -24,7 +25,7 @@ const availableDatasets = Object.values(dataSubjects).flatMap((dataSubject) =>
     .map((dataset) => dataset.value)
 )
 
-const getGetStarted = renderTemplate.bind({
+const getGetStarted = renderTemplate({
   templateParams (req) {
     const { orgInfo: organisation, dataset } = req
     return { organisation, dataset }
@@ -33,21 +34,14 @@ const getGetStarted = renderTemplate.bind({
   handlerName: 'getStarted'
 })
 
-const fetchOrgInfo = fetchOne.bind({
+const fetchOrgInfo = fetchOne({
   query: ({ params }) => {
     return `SELECT name, organisation, entity, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
   },
   result: 'orgInfo'
 })
 
-const fetchDatasetName = fetchOne.bind({
-  query: ({ params }) => {
-    return `SELECT name FROM dataset WHERE dataset = '${params.dataset}'`
-  },
-  result: 'dataset'
-})
-
-const fetchDatasetInfo = fetchOne.bind({
+const fetchDatasetInfo = fetchOne({
   query: ({ params }) => {
     return `SELECT name, dataset FROM dataset WHERE dataset = '${params.dataset}'`
   },
@@ -101,7 +95,7 @@ const fetchDatasetStats = async (req, res, next) => {
   next()
 }
 
-const getDatasetOverview = renderTemplate.bind(
+const getDatasetOverview = renderTemplate(
   {
     templateParams (req) {
       const { orgInfo: organisation, dataset, stats } = req
@@ -112,6 +106,21 @@ const getDatasetOverview = renderTemplate.bind(
   }
 )
 
+const fetchOrgInfoWithStatGeo = fetchOne({
+  query: ({ params }) => {
+    return /* sql */ `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
+  },
+  result: 'orgInfo'
+})
+
+/**
+ * Middleware.
+ */
+const fetchResourceStatus = fetchOne({
+  query: ({ params }) => performanceDbApi.resourceStatusQuery(params.lpa, params.dataset),
+  result: 'resourceStatus'
+})
+
 /**
  * Middleware.
  *
@@ -134,22 +143,12 @@ async function fetchLpaOverview (req, res, next) {
 }
 
 /**
- * Middleware.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
+ * Middleware. Updates req with `resource`.
  */
-async function fetchLatestResource (req, res, next) {
-  const { lpa, dataset } = req.params
-  try {
-    const resource = await performanceDbApi.getLatestResource(lpa, dataset)
-    req.resourceId = resource.resource
-    next()
-  } catch (error) {
-    next(error)
-  }
-}
+const fetchLatestResource = fetchOne({
+  query: ({ params }) => performanceDbApi.latestResourceQuery(params.lpa, params.dataset),
+  result: 'resource'
+})
 
 /**
  *
@@ -162,11 +161,15 @@ async function fetchLatestResource (req, res, next) {
  * @param {*} next
  */
 async function fetchIssues (req, res, next) {
-  const { dataset: datasetId, resourceId: passedResourceId, issue_type: issueType, issue_field: issueField } = req.params
-  const resourceId = passedResourceId ?? req.resourceId
-  console.assert(resourceId, 'missng resourceId')
+  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { resource: resourceId } = req.resource
+  if (!resourceId) {
+    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
+    throw Error('fetchIssues: missing resourceId')
+  }
+
   try {
-    const issues = await performanceDbApi.getIssues({ resource: req.resourceId, issueType, issueField }, datasetId)
+    const issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
     req.issues = issues
     next()
   } catch (error) {
@@ -199,40 +202,27 @@ async function reformatIssuesToBeByEntryNumber (req, res, next) {
  *
  * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
  *
- * Requires `resourceId` in request params or request (in that order).
+ * Requires `req.resource.resource`
  *
  * @param {*} req
  * @param {*} res
  * @param {*} next
  */
 async function fetchIssueEntitiesCount (req, res, next) {
-  const { dataset: datasetId, resourceId: passedResourceId, issue_type: issueType, issue_field: issueField } = req.params
-  const resourceId = passedResourceId ?? req.resourceId
-  console.assert(resourceId, 'missng resourceId')
+  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { resource: resourceId } = req.resource
+  console.assert(resourceId, 'missng resource id')
   const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId)
   req.issueEntitiesCount = parseInt(issueEntitiesCount)
   next()
 }
 
-/**
- *
- * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
- *
- * Requires `resourceId` in request params or request (in that order).
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchEntityCount (req, res, next) {
-  const { dataset: datasetId, resourceId: passedResourceId } = req.params
-  const resourceId = passedResourceId ?? req.resourceId
-  console.assert(resourceId, 'missng resourceId')
-
-  const entityCount = await performanceDbApi.getEntityCountForResource(resourceId, datasetId)
-  req.entityCount = entityCount
-  next()
-}
+const fetchEntityCount = fetchOne({
+  query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource),
+  result: 'entityCount',
+  dataset: FetchOptions.fromParams,
+  fallbackPolicy: FetchOneFallbackPolicy.continue
+})
 
 /**
  *
@@ -253,11 +243,10 @@ async function fetchEntry (req, res, next) {
 
   // look at issue Entries and get the index of that entry - 1
 
-  const entityNum =
-    Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
+  const entityNum = Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
 
   req.entryData = await performanceDbApi.getEntry(
-    req.resourceId,
+    req.resource.resource,
     entityNum,
     datasetId
   )
@@ -265,14 +254,6 @@ async function fetchEntry (req, res, next) {
   next()
 }
 
-const maybeFetchLatestResource = maybeFetch.bind({
-  fetchFn: fetchLatestResource,
-  else: (req) => {
-    req.resourceId = req.params.resourceId
-  },
-  condition: ({ params }) => !('resourceId' in params)
-})
-
 function prepareOverviewTemplateParams (req, res, next) {
   const { lpaOverview, orgInfo: organisation } = req
 
@@ -323,7 +304,7 @@ function prepareOverviewTemplateParams (req, res, next) {
   next()
 }
 
-const getOverview = renderTemplate.bind({
+const getOverview = renderTemplate({
   templateParams (req) {
     if (!req.templateParams) throw new Error('missing templateParams')
     return req.templateParams
@@ -430,8 +411,9 @@ const processEntryRow = (issueType, issuesByEntryNumber, row) => {
  * Middleware. Updates req with `templateParams`
  */
 function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount } = req
+  const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount: entityCountRow } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
   let errorHeading
   let issueItems
@@ -525,15 +507,22 @@ function prepareIssueDetailsTemplateParams (req, res, next) {
  * Middleware. Renders the issue details page with the list of issues, entry data,
  * and organisation and dataset details.
  */
-const getIssueDetails = renderTemplate.bind({
+const getIssueDetails = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/issueDetails.html',
   handlerName: 'getIssueDetails'
 })
 
+const isResourceIdInParams = ({ params }) => !('resourceId' in params)
+
+const takeResourceIdFromParams = (req) => {
+  logger.debug('skipping resource fetch', { type: types.App, params: req.params })
+  req.resource = { resource: req.params.resourceId }
+}
+
 const getGetStartedMiddleware = [
   fetchOrgInfo,
-  fetchDatasetName,
+  fetchDatasetInfo,
   getGetStarted,
   logPageError
 ]
@@ -550,20 +539,24 @@ const getOverviewMiddleware = [
 
 const getIssueDetailsMiddleware = [
   validateIssueDetailsQueryParams,
-  fetchOrgInfo,
-  fetchDatasetInfo,
-  maybeFetchLatestResource,
+  parallel([
+    fetchOrgInfo,
+    fetchDatasetInfo
+  ]),
+  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchIssues,
   reformatIssuesToBeByEntryNumber,
-  fetchEntry,
-  fetchEntityCount,
-  fetchIssueEntitiesCount,
+  parallel([
+    fetchEntry,
+    fetchEntityCount,
+    fetchIssueEntitiesCount
+  ]),
   prepareIssueDetailsTemplateParams,
   getIssueDetails,
   logPageError
 ]
 
-const fetchOrganisations = fetchMany.bind({
+const fetchOrganisations = fetchMany({
   query: ({ req, params }) => 'select name, organisation from organisation',
   result: 'organisations'
 })
@@ -576,7 +569,7 @@ const fetchOrganisations = fetchMany.bind({
  * @param {*} next
  */
 const prepareGetOrganisationsTemplateParams = (req, res, next) => {
-  const sortedResults = req.organisations.formattedData.sort((a, b) => {
+  const sortedResults = req.organisations.sort((a, b) => {
     return a.name.localeCompare(b.name)
   })
 
@@ -592,12 +585,117 @@ const prepareGetOrganisationsTemplateParams = (req, res, next) => {
   next()
 }
 
-const getOrganisations = renderTemplate.bind({
+const getOrganisations = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/find.html',
   handlerName: 'getOrganisations'
 })
 
+/**
+ * Was the resource accessed successfully via HTTP?
+ *
+ * @param {*} req
+ * @returns {boolean}
+ */
+const isResourceAccessible = (req) => req.resourceStatus.status === '200'
+const isResourceNotAccessible = (req) => !isResourceAccessible(req)
+
+const fetchLpaDatasetIssues = fetchMany({
+  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
+  result: 'issues'
+})
+
+const onlyIf = (condition, middlewareFn) => {
+  return async (req, res, next) => {
+    if (condition(req)) {
+      const result = middlewareFn(req, res, next)
+      if (result instanceof Promise) {
+        await result
+      }
+    } else {
+      next()
+    }
+  }
+}
+
+/**
+ * Middleware. Updates req with `templateParams`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ * @returns { { templateParams: object }}
+ */
+const prepareDatasetTaskListTemplateParams = (req, res, next) => {
+  const { issues, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req
+  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
+  const { lpa, dataset: datasetId } = params
+  console.assert(req.resourceStatus.resource === req.resource.resource, 'mismatch between resourceStatus and resource data')
+  console.assert(typeof entityCount === 'number', 'entityCount should be a number')
+
+  const taskList = issues.map((issue) => {
+    return {
+      title: {
+        text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field })
+      },
+      href: `/organisations/${lpa}/${datasetId}/${issue.issue_type}/${issue.field}`,
+      status: getStatusTag(issue.status)
+    }
+  })
+
+  req.templateParams = {
+    taskList,
+    organisation,
+    dataset
+  }
+
+  next()
+}
+
+/**
+ * Middleware. Updates req with `templateParams`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {} next
+ * @returns {{ templateParams: object }}
+ */
+const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
+  const { orgInfo: organisation, dataset, resourceStatus: resource } = req
+
+  const daysSince200 = resource.days_since_200
+  const today = new Date()
+  const last200Date = new Date(
+    today.getTime() - daysSince200 * 24 * 60 * 60 * 1000
+  )
+  const last200Datetime = last200Date.toISOString().slice(0, 19) + 'Z'
+
+  req.templateParams = {
+    organisation,
+    dataset,
+    errorData: {
+      endpoint_url: resource.endpoint_url,
+      http_status: resource.status,
+      latest_log_entry_date: resource.latest_log_entry_date,
+      latest_200_date: last200Datetime
+    }
+  }
+
+  next()
+}
+
+const getDatasetTaskList = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/datasetTaskList.html',
+  handlerName: 'getDatasetTaskList'
+})
+
+const getDatasetTaskListError = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/http-error.html',
+  handlerName: 'getDatasetTaskListError'
+})
+
 const getOrganisationsMiddleware = [
   fetchOrganisations,
   prepareGetOrganisationsTemplateParams,
@@ -605,6 +703,24 @@ const getOrganisationsMiddleware = [
   logPageError
 ]
 
+const getDatasetTaskListMiddleware = [
+  fetchResourceStatus,
+  parallel([
+    fetchOrgInfoWithStatGeo,
+    fetchDatasetInfo]),
+  fetchIf(isResourceAccessible, fetchLatestResource),
+  parallel([
+    fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
+    fetchIf(isResourceAccessible, fetchEntityCount)
+  ]),
+  onlyIf(isResourceAccessible, prepareDatasetTaskListTemplateParams),
+  onlyIf(isResourceAccessible, getDatasetTaskList),
+
+  onlyIf(isResourceNotAccessible, prepareDatasetTaskListErrorTemplateParams),
+  onlyIf(isResourceNotAccessible, getDatasetTaskListError),
+  logPageError
+]
+
 /**
  * Returns a status tag object with a text label and a CSS class based on the status.
  *
@@ -621,158 +737,11 @@ function getStatusTag (status) {
 }
 
 const organisationsController = {
-  /**
-   * Handles GET requests for the dataset task list page.
-   *
-   * @param {Express.Request} req - The incoming request object.
-   * @param {Express.Response} res - The response object to send back to the client.
-   * @param {Express.NextFunction} next - The next function in the middleware chain.
-   *
-   * Retrieves the organisation and dataset names from the database, fetches the issues for the given LPA and dataset,
-   * and renders the dataset task list page with the list of tasks and organisation and dataset details.
-   */
-  async getDatasetTaskList (req, res, next) {
-    const lpa = req.params.lpa
-    const datasetId = req.params.dataset
-
-    try {
-      const organisationResult = await datasette.runQuery(
-        /* sql */ `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${lpa}'`
-      )
-      const organisation = organisationResult.formattedData[0]
-
-      const datasetResult = await datasette.runQuery(
-        `SELECT dataset, name FROM dataset WHERE dataset = '${datasetId}'`
-      )
-      const dataset = datasetResult.formattedData[0]
-
-      const resource = await performanceDbApi.getLatestResource(lpa, datasetId)
-
-      const issues = await performanceDbApi.getLpaDatasetIssues(resource.resource, datasetId)
-
-      const entityCount = await performanceDbApi.getEntityCountForResource(resource.resource, datasetId)
-
-      const taskList = issues.map((issue) => {
-        return {
-          title: {
-            text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field })
-          },
-          href: `/organisations/${lpa}/${datasetId}/${issue.issue_type}/${issue.field}`,
-          status: getStatusTag(issue.status)
-        }
-      })
-
-      const params = {
-        taskList,
-        organisation,
-        dataset
-      }
-
-      validateAndRender(res, 'organisations/datasetTaskList.html', params)
-    } catch (e) {
-      logger.warn(
-        `getDatasetTaskList() failed for lpa='${lpa}', datasetId='${datasetId}'`,
-        {
-          type: types.App,
-          errorMessage: e.message,
-          errorStack: e.stack
-        }
-      )
-      next(e)
-    }
-  },
-
-  /**
-   * Handles endpoint error responses for organizations.
-   *
-   * @param {Object} req - The incoming request object.
-   * @param {Object} res - The outgoing response object.
-   * @param {Function} next - The next middleware function in the chain.
-   * @param {Object} resourceStatus - An object containing information about the resource status.
-   *
-   * @returns {Promise<void>} A promise that resolves when the error response has been rendered.
-   *
-   * @throws {Error} If an error occurs while processing the request.
-   */
-  async getEndpointError (req, res, next, { resourceStatus }) {
-    const { lpa, dataset: datasetId } = req.params
-
-    try {
-      const organisationResult = await datasette.runQuery(
-        `SELECT name, organisation FROM organisation WHERE organisation = '${lpa}'`
-      )
-      const organisation = organisationResult.formattedData[0]
-
-      const datasetResult = await datasette.runQuery(
-        `SELECT name FROM dataset WHERE dataset = '${datasetId}'`
-      )
-      const dataset = datasetResult.formattedData[0]
-
-      const daysSince200 = resourceStatus.days_since_200
-      const today = new Date()
-      const last200Date = new Date(
-        today.getTime() - daysSince200 * 24 * 60 * 60 * 1000
-      )
-      const last200Datetime = last200Date.toISOString().slice(0, 19) + 'Z'
-
-      const params = {
-        organisation,
-        dataset,
-        errorData: {
-          endpoint_url: resourceStatus.endpoint_url,
-          http_status: resourceStatus.status,
-          latest_log_entry_date: resourceStatus.latest_log_entry_date,
-          latest_200_date: last200Datetime
-        }
-      }
-      validateAndRender(res, 'organisations/http-error.html', params)
-    } catch (e) {
-      logger.warn(
-        `conditionalTaskListHandler() failed for lpa='${lpa}', datasetId='${datasetId}'`,
-        { type: types.App }
-      )
-      next(e)
-    }
-  },
-
-  /**
-   * Handles conditional task list request.
-   *
-   * @param {object} req - The HTTP request object.
-   * @param {object} res - The HTTP response object.
-   * @param {function} next - The next middleware function in the chain.
-   *
-   * @throws {Error} - If an error occurs while processing the request.
-   *
-   * @description
-   * This function checks the resource status for a given LPA and dataset ID.
-   * If the resource status is 200, it calls the `getDatasetTaskList` function to retrieve the task list.
-   * Otherwise, it calls the `getEndpointError` function to handle the error.
-   */
-  async conditionalTaskListHandler (req, res, next) {
-    const { lpa, dataset: datasetId } = req.params
-
-    try {
-      const resourceStatus = await performanceDbApi.getResourceStatus(
-        lpa,
-        datasetId
-      )
-
-      if (resourceStatus.status !== '200') {
-        return await organisationsController.getEndpointError(req, res, next, {
-          resourceStatus
-        })
-      } else {
-        return await organisationsController.getDatasetTaskList(req, res, next)
-      }
-    } catch (e) {
-      logger.warn(
-        `conditionalTaskListHandler() failed for lpa='${lpa}', datasetId='${datasetId}'`,
-        { type: types.App }
-      )
-      next(e)
-    }
-  },
+  prepareDatasetTaskListErrorTemplateParams,
+  prepareDatasetTaskListTemplateParams,
+  getDatasetTaskListError,
+  getDatasetTaskList,
+  getDatasetTaskListMiddleware,
 
   /**
    * Middleware chain for GET requests for the "Get Started" page.
diff --git a/src/controllers/middleware.js b/src/controllers/middleware.js
index 703c0818..c9c04029 100644
--- a/src/controllers/middleware.js
+++ b/src/controllers/middleware.js
@@ -5,11 +5,50 @@ import { render } from '../utils/custom-renderer.js'
 import datasette from '../services/datasette.js'
 import * as v from 'valibot'
 
+export const FetchOptions = {
+  /**
+   * Use 'dataset' from requets params.
+   */
+  fromParams: Symbol('from-params')
+}
+
+const datasetOverride = (val, req) => {
+  if (!val) {
+    return 'digital-land'
+  }
+  if (val === FetchOptions.fromParams) {
+    console.assert(req.params.dataset, 'no "dataset" in request params')
+    return req.params.dataset
+  } else {
+    return val(req)
+  }
+}
+
+const fetchOneFallbackPolicy = (req, res, next) => {
+  res.status(404).render('errorPages/404', {})
+}
+
+/**
+ * Collection of fallback policies for the {@link fetchOneFn} middleware.
+ * The policy is enacted when zero records is returned from the data source.
+ */
+export const FetchOneFallbackPolicy = {
+  /**
+   * Renders a 404 response.
+   */
+  'not-found-error': fetchOneFallbackPolicy,
+
+  /**
+   * Proceeds by calling `next()`.
+   */
+  continue: (_req, _res, next) => next()
+}
+
 /**
  * Middleware. Attempts to fetch data from datasette and short-circuits with 404 when
  * data for given query does not exist. Meant to be used to fetch singular records.
  *
- * `this` needs `{ query({ req, params }) => any, result: string }`
+ * `this` needs `{ query({ req, params }) => any, result: string, dataset?: FetchParams | (req) => string, fallbackPolicy: (req, res, next) => void }`
  *
  * where the `result` is the key under which result of the query will be stored in `req`
  *
@@ -17,20 +56,21 @@ import * as v from 'valibot'
  * @param {*} res
  * @param {*} next
  */
-export async function fetchOne (req, res, next) {
+async function fetchOneFn (req, res, next) {
   logger.debug({ type: types.DataFetch, message: 'fetchOne', resultKey: this.result })
   try {
     const query = this.query({ req, params: req.params })
-    const result = await datasette.runQuery(query)
+    const result = await datasette.runQuery(query, datasetOverride(this.dataset, req))
+    const fallbackPolicy = this.fallbackPolicy ?? FetchOneFallbackPolicy['not-found-error']
     if (result.formattedData.length === 0) {
       // we can make the 404 more informative by informing the use what exactly was "not found"
-      res.status(404).render('errorPages/404', {})
+      fallbackPolicy(req, res, next)
     } else {
       req[this.result] = result.formattedData[0]
       next()
     }
   } catch (error) {
-    logger.debug('fetchOne: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl })
+    logger.debug('fetchOne: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
     req.handlerName = `fetching '${this.result}'`
     next(error)
   }
@@ -39,20 +79,21 @@ export async function fetchOne (req, res, next) {
 /**
  * Middleware. Attempts to fetch a collection of data from datasette.
  *
- * `this` needs `{ query( {req, params } ) => any, result: string }`
+ * `this` needs `{ query( {req, params } ) => any, result: string, dataset?: FetchParams | (req) => string }`
  *
  * @param {*} req
  * @param {*} res
  * @param {*} next
  */
-export async function fetchMany (req, res, next) {
-  logger.debug({ type: types.DataFetch, message: 'fetchMany', resultKey: this.result })
+export async function fetchManyFn (req, res, next) {
   try {
     const query = this.query({ req, params: req.params })
-    req[this.result] = await datasette.runQuery(query)
+    const result = await datasette.runQuery(query, datasetOverride(this.dataset, req))
+    req[this.result] = result.formattedData
+    logger.debug({ type: types.DataFetch, message: 'fetchMany', resultKey: this.result, resultCount: result.formattedData.length })
     next()
   } catch (error) {
-    logger.debug('fetchMany: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl })
+    logger.debug('fetchMany: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
     req.handlerName = `fetching '${this.result}'`
     next(error)
   }
@@ -63,11 +104,13 @@ export async function fetchMany (req, res, next) {
  *
  * `this` needs: `{ fetchFn, condition: (req) => boolean, else?: (req) => void }`
  *
+ * `fetchFn` should be a middleware fn. Can be async.
+ *
  * @param {*} req
  * @param {*} res
  * @param {*} next
  */
-export async function maybeFetch (req, res, next) {
+async function fetchIfFn (req, res, next) {
   if (this.condition(req)) {
     // `next` will be called in our fetchFn middleware
     const result = this.fetchFn(req, res, next)
@@ -82,6 +125,22 @@ export async function maybeFetch (req, res, next) {
   }
 }
 
+/**
+ *
+ * Returns a middleware that will fetch data if condition is satisfied,
+ * invoke `elseFn` otherwise.
+ *
+ * @param {(req) => boolean} condition predicate fn
+ * @param {*} fetchFn fetch middleware
+ * @param {(req) => void} elseFn
+ * @returns
+ */
+export const fetchIf = (condition, fetchFn, elseFn) => {
+  return fetchIfFn.bind({
+    condition, fetchFn, else: elseFn
+  })
+}
+
 /**
  * Middleware. Set `req.handlerName` to a string that will identify
  * the function that threw the error.
@@ -130,12 +189,96 @@ export function validateAndRender (res, name, params) {
  * @param {*} next
  * @returns
  */
-export function renderTemplate (req, res, next) {
+export function renderTemplateFn (req, res, next) {
   const templateParams = this.templateParams(req)
   try {
     validateAndRender(res, this.template, templateParams)
+    logger.info(`rendered ${this.template}`, { type: types.App })
   } catch (err) {
     req.handlerName = this.handlerName
     next(err)
   }
 }
+
+/**
+ * Returns a middleware that executes the given sub-middlewares in parallel
+ * and waits for all of them to complete.
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ * @returns {(req, res, next) => void} middleware
+ */
+async function parallelFn (req, res, next) {
+  const fns = this.middlewares
+  const nextParams = []
+  const nextFn = (val) => {
+    if (val) nextParams.push(val)
+  }
+  // We need to take care of explicit `next(value)`, so we hijack the 'next' callback.
+  // We also need to hadle any rejected promises in results
+  const results = await Promise.allSettled(fns.map(fn => fn(req, res, nextFn)))
+  /* eslint-disable no-unreachable-loop */
+  for (const param of nextParams) {
+    if (param instanceof Error) {
+      logger.debug('parallel: captured a "next" error', { type: types.App, errorMessage: param.message })
+    }
+    next(param)
+    return
+  }
+  for (const result of results) {
+    if (result.status === 'rejected') {
+      logger.debug('parallel: got a rejected promise', { type: types.App })
+      next(result.reason)
+      return
+    }
+  }
+
+  next()
+}
+
+/**
+ * Returns a middleware that invokes the passed middlewares in parallel.
+ *
+ * Usage: when all sub-middlewre can be fetched independently, but we can't accept a partial success
+ * (e.g. we require all middlewares to succeed).
+ *
+ * @param {((req,res,next) => void)[]} middlewares array of middleware functions
+ * @returns {(req, res, next) => Promise<void>}
+ */
+export function parallel (middlewares) {
+  return parallelFn.bind({ middlewares })
+}
+
+/**
+ * Fetches a single entity and stores it in `req` under key specified by `result` entry.
+ *
+ * Use `fallbackPolicy` to handle zero record responses differently than the default 404 response.
+ * See {@link FetchOneFallbackPolicy}
+ *
+ * @param {{ query: ({ req, params}) => object, result: string, dataset?: FetchParams | (req) => string, fallbackPolicy?: (req, res, next) => void}} context
+ * @returns
+ */
+export function fetchOne (context) {
+  return fetchOneFn.bind(context)
+}
+
+/**
+ * Fetches a collection of records and stores them in `req` under key specified by `result` entry.
+ *
+ * @param {{query: ({req, params}) => object, result: string, dataset?: FetchParams | (req) => string}} context
+ */
+export function fetchMany (context) {
+  return fetchManyFn.bind(context)
+}
+
+/**
+ * Validates and renders the template.
+ *
+ *
+ * @param {{templateParams: (req) => object, template: string,  handlerName: string}} context
+ * @returns {(req, res, next) => void} middleware
+ */
+export function renderTemplate (context) {
+  return renderTemplateFn.bind(context)
+}
diff --git a/src/routes/organisations.js b/src/routes/organisations.js
index b5e3fbd6..69c68332 100644
--- a/src/routes/organisations.js
+++ b/src/routes/organisations.js
@@ -7,7 +7,7 @@ router.get('/:lpa/:dataset/get-started', OrganisationsController.getGetStartedMi
 router.get('/:lpa/:dataset/overview', OrganisationsController.getDatasetOverviewMiddleware)
 router.get('/:lpa/:dataset/:issue_type/:issue_field/:pageNumber', OrganisationsController.getIssueDetailsMiddleware)
 router.get('/:lpa/:dataset/:issue_type/:issue_field', OrganisationsController.getIssueDetailsMiddleware)
-router.get('/:lpa/:dataset', OrganisationsController.conditionalTaskListHandler)
+router.get('/:lpa/:dataset', OrganisationsController.getDatasetTaskListMiddleware)
 router.get('/:lpa', OrganisationsController.getOverviewMiddleware)
 router.get('/', OrganisationsController.getOrganisationsMiddleware)
 
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 1d308fe6..d6687eab 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -39,7 +39,7 @@ const datasetStatusEnum = {
 }
 
 const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
-const DatasetNameField = v.strictObject({ name: NonEmptyString })
+const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString })
 
 export const OrgOverviewPage = v.strictObject({
   organisation: OrgField,
@@ -111,7 +111,10 @@ export const OrgDatasetTaskList = v.strictObject({
 
 export const OrgEndpointError = v.strictObject({
   organisation: OrgField,
-  dataset: DatasetNameField,
+  dataset: v.object({
+    name: NonEmptyString,
+    dataset: NonEmptyString
+  }),
   errorData: v.strictObject({
     endpoint_url: v.url(),
     http_status: v.integer(),
@@ -122,10 +125,7 @@ export const OrgEndpointError = v.strictObject({
 
 export const OrgIssueDetails = v.strictObject({
   organisation: OrgField,
-  dataset: v.object({
-    name: NonEmptyString,
-    dataset: v.string()
-  }),
+  dataset: DatasetNameField,
   errorHeading: v.optional(NonEmptyString),
   issueItems: v.array(v.strictObject({
     html: v.string(),
diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 4a7551b1..836585da 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -187,33 +187,23 @@ ORDER BY
     return { datasets }
   },
 
-  /**
-     * Retrieves the resource status from the performance database.
-     *
-     * @param {string} lpa - The Local Planning Authority (LPA) code.
-     * @param {string} datasetId - The ID of the dataset to retrieve the status for.
-     * @returns {object} The resource status object containing the endpoint URL, status, latest log entry date, and days since 200.
-     */
-  getResourceStatus: async (lpa, datasetId) => {
-    const sql =
-    /* sql */
-    `
-    select endpoint_url, status, latest_log_entry_date, days_since_200 from reporting_latest_endpoints
+  resourceStatusQuery (lpa, datasetId) {
+    return /* sql */ `
+    select resource, endpoint_url, status, latest_log_entry_date, days_since_200 
+    from reporting_latest_endpoints
     WHERE REPLACE(organisation, '-eng', '') = '${lpa}'
     AND pipeline = '${datasetId}'`
-
-    const result = await datasette.runQuery(sql)
-
-    return result.formattedData[0]
   },
 
+  datasetIssuesQuery,
+
   /**
      * Retrieves LPA dataset issues for a given resource and dataset ID.
      *
      * @param {string} resource - The resource to retrieve issues for.
      * @param {string} datasetId - The ID of the dataset to retrieve issues for.
      *
-     * @returns {object[]} An array of issue objects, each containing:
+     * @returns {Promise<object[]>} An array of issue objects, each containing:
      *   - field: {string} The field associated with the issue.
      *   - issue_type: {string} The type of issue.
      *   - line_number: {number} The line number of the issue.
@@ -276,6 +266,16 @@ ORDER BY
     return message.replace('{num_issues}', numIssues).replace('{num_entries}', numIssues).replace('{column_name}', field)
   },
 
+  latestResourceQuery: (lpa, dataset) => {
+    return /* sql */ `
+    SELECT rle.resource, rle.status, rle.endpoint, rle.endpoint_url, rle.status, rle.days_since_200, rle.exception
+    FROM reporting_latest_endpoints rle
+    LEFT JOIN resource_organisation ro ON rle.resource = ro.resource
+    LEFT JOIN organisation o ON REPLACE(ro.organisation, '-eng', '') = o.organisation
+    WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}'
+    AND rle.pipeline = '${dataset}'`
+  },
+
   /**
      * Retrieves the latest resource information for a given LPA and dataset.
      *
@@ -284,14 +284,7 @@ ORDER BY
      * @returns {object} The latest resource information, including the resource, status, endpoint, endpoint URL, days since 200, and exception.
      */
   async getLatestResource (lpa, dataset) {
-    const sql = `
-    SELECT rle.resource, rle.status, rle.endpoint, rle.endpoint_url, rle.status, rle.days_since_200, rle.exception
-    FROM reporting_latest_endpoints rle
-    LEFT JOIN resource_organisation ro ON rle.resource = ro.resource
-    LEFT JOIN organisation o ON REPLACE(ro.organisation, '-eng', '') = o.organisation
-    WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}'
-    AND rle.pipeline = '${dataset}'`
-
+    const sql = this.latestResourceQuery(lpa, dataset)
     const result = await datasette.runQuery(sql)
 
     return result.formattedData[0]
@@ -390,24 +383,24 @@ ORDER BY
     return result.formattedData
   },
 
+  entityCountQuery (resource) {
+    return /* sql */ `
+    select dataset, entity_count, resource
+    from dataset_resource
+    WHERE resource = '${resource}'
+  `
+  },
+
   /**
      * Retrieves the entity count for a given resource and dataset.
      *
      * @param {string} resource - The resource to retrieve the entity count for.
      * @param {string} dataset - The dataset to retrieve the entity count from.
-     * @returns {number} The entity count for the given resource and dataset.
+     * @returns {Promise<number>} The entity count for the given resource and dataset.
      */
   async getEntityCountForResource (resource, dataset) {
-    const query =
-    /* sql */
-    `
-    select dataset, entity_count, resource
-    from dataset_resource
-    WHERE resource = '${resource}'
-  `
-
+    const query = this.entityCountQuery(resource)
     const result = await datasette.runQuery(query, dataset)
-
     return result.formattedData[0].entity_count
   },
 
diff --git a/src/utils/custom-renderer.js b/src/utils/custom-renderer.js
index 55425dd7..5d670ddc 100644
--- a/src/utils/custom-renderer.js
+++ b/src/utils/custom-renderer.js
@@ -9,7 +9,13 @@ import { types } from './logging.js'
  */
 export const invalidSchemaPaths = (error) => {
   if (error instanceof v.ValiError) {
-    return error.issues.map((issue) => issue.path.flatMap((p) => p.key))
+    const paths = []
+    for (const issue of error.issues) {
+      if (issue.path) {
+        paths.push(issue.path.flatMap((p) => p.key))
+      }
+    }
+    return paths
   }
   throw new TypeError(`error is not a validation error: ${error.name}`)
 }
diff --git a/src/utils/logger.js b/src/utils/logger.js
index 61bb4fca..8a5777d8 100644
--- a/src/utils/logger.js
+++ b/src/utils/logger.js
@@ -1,6 +1,14 @@
 import { createLogger, format, transports } from 'winston'
 import config from '../../config/index.js'
 
+/* eslint-disable no-unused-vars */
+const ignoreAssetRequests = format((info, opts) => {
+  const { endpoint } = info
+  if (!endpoint) { return info }
+  if (endpoint.match(/^\/(assets|public)\/.*/)) { return false }
+  return info
+})
+
 const appTransports = () => [
   new transports.Console(),
   new transports.File({ filename: 'combined.log' })
diff --git a/test/unit/http-errorPage.test.js b/test/unit/http-errorPage.test.js
index efa87847..d908019c 100644
--- a/test/unit/http-errorPage.test.js
+++ b/test/unit/http-errorPage.test.js
@@ -12,7 +12,8 @@ describe('http-error.html', () => {
       organisation: 'mock-org'
     },
     dataset: {
-      name: 'mock-dataset'
+      name: 'mock-dataset',
+      dataset: 'mock-dataset'
     },
     errorData: {
       endpoint_url: 'https://example.com/data-url',
diff --git a/test/unit/organisationsController.test.js b/test/unit/organisationsController.test.js
index 08e0d71a..b610904d 100644
--- a/test/unit/organisationsController.test.js
+++ b/test/unit/organisationsController.test.js
@@ -2,7 +2,6 @@ import { describe, it, vi, expect, beforeEach } from 'vitest'
 import * as v from 'valibot'
 import organisationsController from '../../src/controllers/OrganisationsController.js'
 import performanceDbApi from '../../src/services/performanceDbApi.js'
-import datasette from '../../src/services/datasette.js'
 
 vi.mock('../../src/services/performanceDbApi.js')
 vi.mock('../../src/utils/utils.js', () => {
@@ -49,7 +48,7 @@ describe('OrganisationsController.js', () => {
       const next = vi.fn()
 
       organisationsController.prepareOverviewTemplateParams(req, res, () => {})
-      await organisationsController.getOverview(req, res, next)
+      organisationsController.getOverview(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/overview.html', expect.objectContaining({
@@ -73,9 +72,9 @@ describe('OrganisationsController.js', () => {
       const res = { render: vi.fn() }
       const next = vi.fn()
 
-      req.organisations = { formattedData: [] }
+      req.organisations = []
       organisationsController.prepareGetOrganisationsTemplateParams(req, res, next)
-      await organisationsController.getOrganisations(req, res, next)
+      organisationsController.getOrganisations(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/find.html', expect.objectContaining({
@@ -97,9 +96,9 @@ describe('OrganisationsController.js', () => {
         { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' }
       ]
 
-      req.organisations = { formattedData: datasetteResponse }
+      req.organisations = datasetteResponse
       organisationsController.prepareGetOrganisationsTemplateParams(req, res, next)
-      await organisationsController.getOrganisations(req, res, next)
+      organisationsController.getOrganisations(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/find.html', expect.objectContaining({
@@ -123,20 +122,6 @@ describe('OrganisationsController.js', () => {
         }
       }))
     })
-
-    it('should catch errors and pass them onto the next function', async () => {
-      const req = {}
-      const res = {}
-      const next = vi.fn()
-
-      const error = new Error('Test error')
-
-      vi.mocked(datasette.runQuery).mockRejectedValue(error)
-      await organisationsController.fetchOrganisations(req, res, next)
-
-      expect(next).toHaveBeenCalledTimes(1)
-      expect(next).toHaveBeenCalledWith(error)
-    })
   })
 
   describe('get-started', () => {
@@ -145,35 +130,23 @@ describe('OrganisationsController.js', () => {
         { name: 'Example LPA', organisation: 'LPA' }
       ]
     }
-    const exampleDataset = {
-      formattedData: [
-        { name: 'Example Dataset' }
-      ]
-    }
+    const exampleDataset = { name: 'Example Dataset', dataset: 'example-dataset' }
 
     it('should render the get-started template with the correct params', async () => {
       const req = {
         params: { lpa: 'example-lpa', dataset: 'example-dataset' },
         orgInfo: exampleLpa.formattedData[0],
-        dataset: exampleDataset.formattedData[0]
+        dataset: exampleDataset
       }
       const res = { render: vi.fn() }
       const next = vi.fn()
 
-      datasette.runQuery.mockImplementation((query) => {
-        if (query.includes('example-lpa')) {
-          return exampleLpa
-        } else if (query.includes('example-dataset')) {
-          return exampleDataset
-        }
-      })
-
-      await organisationsController.getGetStarted(req, res, next)
+      organisationsController.getGetStarted(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/get-started.html', {
         organisation: { name: 'Example LPA', organisation: 'LPA' },
-        dataset: { name: 'Example Dataset' }
+        dataset: exampleDataset
       })
     })
 
@@ -181,101 +154,56 @@ describe('OrganisationsController.js', () => {
       const req = {
         params: { lpa: 'example-lpa', dataset: 'example-dataset' },
         orgInfo: undefined, // this should fail validation
-        dataset: exampleDataset.formattedData[0]
+        dataset: exampleDataset
       }
       const res = { render: vi.fn() }
       const next = vi.fn()
 
-      await organisationsController.getGetStarted(req, res, next)
+      organisationsController.getGetStarted(req, res, next)
 
       expect(next).toHaveBeenCalledTimes(1)
+      expect(res.render).toHaveBeenCalledTimes(0)
     })
   })
 
   describe('dataset task list', () => {
-    it('should call render with the datasetTaskList page', async () => {
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      vi.mocked(datasette.runQuery).mockResolvedValueOnce({
-        formattedData: [{ name: 'Example Organisation', organisation: 'ORG' }]
-      }).mockResolvedValueOnce({
-        formattedData: [{ name: 'Example Dataset' }]
-      })
-
-      vi.mocked(performanceDbApi.getLatestResource).mockResolvedValue({
-        resource: 'mock-resource'
-      })
-
-      vi.mocked(performanceDbApi.getLpaDatasetIssues).mockResolvedValue([
-        {
-          issue: 'Example issue 1',
-          issue_type: 'Example issue type 1',
-          field: 'Example field 1',
-          num_issues: 1,
-          status: 'Error'
-        }
-      ])
-
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('task message 1')
-
-      await organisationsController.getDatasetTaskList(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/datasetTaskList.html', {
-        taskList: [{
-          title: {
-            text: 'task message 1'
+    it('should fetch the dataset tasks and correctly pass them on to the dataset task list page', async () => {
+      const req = {
+        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
+        resourceStatus: {
+          resource: 'mock-resource',
+          endpoint_url: 'http://example.com/resource',
+          status: '200',
+          latest_log_entry_date: '',
+          days_since_200: 0
+        },
+        orgInfo: { name: 'Example Organisation', organisation: 'ORG' },
+        dataset: { name: 'Example Dataset' },
+        resource: { resource: 'mock-resource' },
+        issues: [
+          {
+            issue: 'Example issue 1',
+            issue_type: 'Example issue type 1',
+            field: 'Example issue field 1',
+            num_issues: 1,
+            status: 'Error'
           },
-          href: '/organisations/example-lpa/example-dataset/Example issue type 1/Example field 1',
-          status: {
-            tag: {
-              classes: 'govuk-tag--red',
-              text: 'Error'
-            }
+          {
+            issue: 'Example issue 2',
+            issue_type: 'Example issue type 2',
+            field: 'Example issue field 2',
+            num_issues: 1,
+            status: 'Needs fixing'
           }
-        }],
-        organisation: { name: 'Example Organisation', organisation: 'ORG' },
-        dataset: { name: 'Example Dataset' }
-      })
-    })
-
-    it('should fetch the dataset tasks and correctly pass them on to the dataset task list page', async () => {
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
+        ]
+      }
       const res = { render: vi.fn() }
       const next = vi.fn()
 
-      vi.mocked(datasette.runQuery).mockResolvedValueOnce({
-        formattedData: [{ name: 'Example Organisation', organisation: 'ORG' }]
-      }).mockResolvedValueOnce({
-        formattedData: [{ name: 'Example Dataset' }]
-      })
-
-      vi.mocked(performanceDbApi.getLatestResource).mockResolvedValue({
-        resource: 'mock-resource'
-      })
-
-      vi.mocked(performanceDbApi.getLpaDatasetIssues).mockResolvedValue([
-        {
-          issue: 'Example issue 1',
-          issue_type: 'Example issue type 1',
-          field: 'Example issue field 1',
-          num_issues: 1,
-          status: 'Error'
-        },
-        {
-          issue: 'Example issue 2',
-          issue_type: 'Example issue type 2',
-          field: 'Example issue field 2',
-          num_issues: 1,
-          status: 'Needs fixing'
-        }
-      ])
-
       vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('task message 1').mockReturnValueOnce('task message 2')
 
-      await organisationsController.getDatasetTaskList(req, res, next)
+      organisationsController.prepareDatasetTaskListTemplateParams(req, res, next)
+      organisationsController.getDatasetTaskList(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/datasetTaskList.html', {
@@ -309,22 +237,6 @@ describe('OrganisationsController.js', () => {
         dataset: { name: 'Example Dataset' }
       })
     })
-
-    it('should catch errors and pass them on to the next function', async () => {
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
-      const res = {}
-      const next = vi.fn()
-
-      // Mock the datasette.runQuery method to throw an error
-      datasette.runQuery.mockImplementation(() => {
-        throw new Error('example error')
-      })
-
-      await organisationsController.getDatasetTaskList(req, res, next)
-
-      expect(next).toHaveBeenCalledTimes(1)
-      expect(next).toHaveBeenCalledWith(expect.any(Error))
-    })
   })
 
   describe('issue details', () => {
@@ -358,7 +270,7 @@ describe('OrganisationsController.js', () => {
         params: requestParams,
         // middleware supplies the below
         entryNumber: 1,
-        entityCount: 3,
+        entityCount: { entity_count: 3 },
         issueEntitiesCount: 1,
         pageNumber: 1,
         orgInfo,
@@ -391,9 +303,9 @@ describe('OrganisationsController.js', () => {
         vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
       })
       vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
-      organisationsController.prepareIssueDetailsTemplateParams(req, {}, () => {})
 
-      await organisationsController.getIssueDetails(req, res, next)
+      organisationsController.prepareIssueDetailsTemplateParams(req, {}, () => {})
+      organisationsController.getIssueDetails(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/issueDetails.html', {
@@ -462,7 +374,7 @@ describe('OrganisationsController.js', () => {
         params: requestParams,
         // middleware supplies the below
         entryNumber: 1,
-        entityCount: 3,
+        entityCount: { entity_count: 3 },
         issueEntitiesCount: 1,
         pageNumber: 1,
         orgInfo,
@@ -495,9 +407,9 @@ describe('OrganisationsController.js', () => {
         vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
       })
       vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
-      organisationsController.prepareIssueDetailsTemplateParams(req, {}, () => {})
 
-      await organisationsController.getIssueDetails(req, res, next)
+      organisationsController.prepareIssueDetailsTemplateParams(req, {}, () => {})
+      organisationsController.getIssueDetails(req, res, next)
 
       expect(res.render).toHaveBeenCalledTimes(1)
       expect(res.render).toHaveBeenCalledWith('organisations/issueDetails.html', {
@@ -549,115 +461,36 @@ describe('OrganisationsController.js', () => {
         pageNumber: 1
       })
     })
-
-    it('should catch errors and pass them onto the next function', async () => {
-      const req = {
-        params: {
-          lpa: 'test-lpa',
-          dataset: 'test-dataset',
-          issue_type: 'test-issue-type',
-          resourceId: 'test-resource-id',
-          entityNumber: '1'
-        }
-      }
-      const res = {
-        render: vi.fn()
-      }
-      const next = vi.fn()
-
-      vi.mocked(performanceDbApi.getLatestResource).mockRejectedValue(new Error('Test error'))
-
-      await organisationsController.getIssueDetails(req, res, next)
-
-      expect(next).toHaveBeenCalledTimes(1)
-      expect(next).toHaveBeenCalledWith(expect.any(Error))
-    })
   })
 
-  describe('getEndpointError', () => {
+  describe('getDatasetTaskListError', () => {
     it('should render http-error.html template with correct params', async () => {
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
+      const resourceStatus = { status: '404', days_since_200: 3, endpoint_url: 'https://example.com', latest_log_entry_date: '2022-01-01T12:00:00.000Z' }
+      const organisation = { name: 'Example Organisation', organisation: 'ORG' }
+      const dataset = { name: 'Example Dataset', dataset: 'example-dataset' }
+      const req = {
+        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
+        resourceStatus,
+        orgInfo: organisation,
+        dataset
+      }
       const res = { render: vi.fn() }
       const next = vi.fn()
-      const resourceStatus = { status: '404', days_since_200: 3, endpoint_url: 'https://example.com', latest_log_entry_date: '2022-01-01T12:00:00.000Z' }
-
-      vi.mocked(datasette.runQuery).mockResolvedValueOnce({ formattedData: [{ name: 'Example Organisation', organisation: 'ORG' }] })
-      vi.mocked(datasette.runQuery).mockResolvedValueOnce({ formattedData: [{ name: 'Example Dataset' }] })
 
-      await organisationsController.getEndpointError(req, res, next, { resourceStatus })
+      organisationsController.prepareDatasetTaskListErrorTemplateParams(req, res, next)
+      organisationsController.getDatasetTaskListError(req, res, next)
 
       const dataTimeRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z/
 
       expect(res.render).toHaveBeenCalledTimes(1)
       const renderArgs = res.render.mock.calls[0]
       expect(renderArgs[0]).toEqual('organisations/http-error.html')
-      expect(renderArgs[1].organisation).toEqual({ name: 'Example Organisation', organisation: 'ORG' })
-      expect(renderArgs[1].dataset).toEqual({ name: 'Example Dataset' })
+      expect(renderArgs[1].organisation).toEqual(organisation)
+      expect(renderArgs[1].dataset).toEqual(dataset)
       expect(renderArgs[1].errorData.endpoint_url).toEqual('https://example.com')
       expect(renderArgs[1].errorData.http_status).toEqual('404')
       expect(renderArgs[1].errorData.latest_log_entry_date).toMatch(dataTimeRegex)
       expect(renderArgs[1].errorData.latest_200_date).toMatch(dataTimeRegex)
     })
-
-    it('should catch errors and pass them to the next function', async () => {
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
-      const res = {}
-      const next = vi.fn()
-      const resourceStatus = { status: '404' }
-
-      vi.mocked(datasette.runQuery).mockRejectedValueOnce(new Error('example error'))
-
-      await organisationsController.getEndpointError(req, res, next, { resourceStatus })
-
-      expect(next).toHaveBeenCalledTimes(1)
-      expect(next).toHaveBeenCalledWith(expect.any(Error))
-    })
-  })
-
-  describe('conditionalTaskListHandler', () => {
-    it('should call getEndpointError if resource status is not 200', async () => {
-      const getResourceStatusSpy = vi.spyOn(organisationsController, 'getEndpointError')
-      getResourceStatusSpy.mockResolvedValue({})
-
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      vi.mocked(performanceDbApi.getResourceStatus).mockResolvedValueOnce({ status: '404' })
-
-      await organisationsController.conditionalTaskListHandler(req, res, next)
-
-      expect(organisationsController.getEndpointError).toHaveBeenCalledTimes(1)
-      expect(organisationsController.getEndpointError).toHaveBeenCalledWith(req, res, next, { resourceStatus: { status: '404' } })
-    })
-
-    it('should call getDatasetTaskList if resource status is 200', async () => {
-      const getDatasetTaskListSpy = vi.spyOn(organisationsController, 'getDatasetTaskList')
-      getDatasetTaskListSpy.mockResolvedValue({})
-
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      vi.mocked(performanceDbApi.getResourceStatus).mockResolvedValueOnce({ status: '200' })
-
-      await organisationsController.conditionalTaskListHandler(req, res, next)
-
-      expect(organisationsController.getDatasetTaskList).toHaveBeenCalledTimes(1)
-      expect(organisationsController.getDatasetTaskList).toHaveBeenCalledWith(req, res, next)
-    })
-
-    it('should catch errors and pass them to the next function', async () => {
-      const req = { params: { lpa: 'example-lpa', dataset: 'example-dataset' } }
-      const res = {}
-      const next = vi.fn()
-
-      vi.mocked(performanceDbApi.getResourceStatus).mockRejectedValueOnce(new Error('example error'))
-
-      await organisationsController.conditionalTaskListHandler(req, res, next)
-
-      expect(next).toHaveBeenCalledTimes(1)
-      expect(next).toHaveBeenCalledWith(expect.any(Error))
-    })
   })
 })

From 92c0e52e5397ea6eac940c88a0f493621fd1760a Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 15:40:26 +0100
Subject: [PATCH 13/65] fix tests

---
 test/unit/services/DatasetService.test.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 673a64e9..48ba353b 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -11,25 +11,25 @@ describe('DatasetService', () => {
         formattedData: [
           {
             datasets: 'dataset1',
-            json: "[{'key': 'value'}]"
+            json: "[{'dataset': 'dataset1', 'key': 'value'}]"
           },
           {
             datasets: 'dataset2;dataset3',
-            json: "[{'key2': 'value2'},{'key3': 'value3'}]"
+            json: "[{'dataset': 'dataset2', 'key2': 'value2'},{'dataset': 'dataset3', 'key3': 'value3'}]"
           },
           {
             datasets: 'dataset4',
-            json: "[{'key4': 'value4'}]"
+            json: "[{'dataset': 'dataset4', 'key4': 'value4'}]"
           }
         ]
       })
     })
 
     const expectedSpecifications = {
-      dataset1: { key: 'value' },
-      dataset2: { key2: 'value2' },
-      dataset3: { key3: 'value3' },
-      dataset4: { key4: 'value4' }
+      dataset1: { dataset: 'dataset1', key: 'value' },
+      dataset2: { dataset: 'dataset2', key2: 'value2' },
+      dataset3: { dataset: 'dataset3', key3: 'value3' },
+      dataset4: { dataset: 'dataset4', key4: 'value4' }
     }
 
     it('returns specifications in the correct format', async () => {
@@ -49,7 +49,7 @@ describe('DatasetService', () => {
     const specifications = [
       {
         datasets: 'dataset1',
-        json: '[{fields: [{field: "field1"},{field: "field3"}]}]'
+        json: '[{dataset: "dataset1", fields: [{field: "field1"},{field: "field3"}]}]'
       }
     ]
 

From 16d60941a655a3f25a8ba160740cc585c8b31f3c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:06:27 +0100
Subject: [PATCH 14/65] added test to nunjucks template

---
 src/controllers/OrganisationsController.js    |  4 +--
 src/routes/schemas.js                         |  2 +-
 src/services/DatasetService.js                |  4 +--
 src/views/organisations/dataset-overview.html |  4 +--
 test/unit/services/DatasetService.test.js     |  2 +-
 .../organisations/dataset-overview.test.js    | 36 ++++++++++++++++---
 6 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 7d8f42b8..3fc5d40b 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -52,7 +52,7 @@ const fetchDatasetStats = async (req, res, next) => {
   const { dataset, lpa } = req.params
   const { orgInfo: organisation } = req
 
-  const { numberOfFieldsSupplied, numberOfFieldsMatched, NumberOfExpectedFields } = await getFieldStats(lpa, dataset)
+  const { numberOfFieldsSupplied, numberOfFieldsMatched, numberOfExpectedFields } = await getFieldStats(lpa, dataset)
 
   const sources = await getSources(lpa, dataset)
 
@@ -87,7 +87,7 @@ const fetchDatasetStats = async (req, res, next) => {
   req.stats = {
     numberOfFieldsSupplied,
     numberOfFieldsMatched,
-    NumberOfExpectedFields,
+    numberOfExpectedFields,
     numberOfRecords,
     endpoints
   }
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index d6687eab..b52c296b 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -77,7 +77,7 @@ export const OrgDatasetOverview = v.strictObject({
     numberOfRecords: v.integer(),
     numberOfFieldsSupplied: v.integer(),
     numberOfFieldsMatched: v.integer(),
-    NumberOfExpectedFields: v.integer(),
+    numberOfExpectedFields: v.integer(),
     endpoints: v.array(v.strictObject({
       name: v.string(),
       endpoint: v.string(),
diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 9acd1700..7b499e9f 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -37,12 +37,12 @@ export async function getFieldStats (lpa, dataset) {
     return matchingFields.includes(current) ? acc + 1 : acc
   }, 0)
 
-  const NumberOfExpectedFields = datasetSpecification.fields.length
+  const numberOfExpectedFields = datasetSpecification.fields.length
 
   return {
     numberOfFieldsSupplied,
     numberOfFieldsMatched,
-    NumberOfExpectedFields
+    numberOfExpectedFields
   }
 }
 
diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index 3e96b036..374aa4b3 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -56,7 +56,7 @@
       text: "Number of fields supplied"
     },
     value: {
-      text: stats.numberOfFieldsSupplied + '/' + stats.NumberOfExpectedFields | default(0)
+      text: stats.numberOfFieldsSupplied + '/' + stats.numberOfExpectedFields | default(0)
     }
   },
   {
@@ -64,7 +64,7 @@
       text: "Number of fields matched"
     },
     value: {
-      text: stats.numberOfFieldsMatched + '/' + stats.NumberOfExpectedFields | default(0)
+      text: stats.numberOfFieldsMatched + '/' + stats.numberOfExpectedFields | default(0)
     }
   },
   {
diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 48ba353b..28544863 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -63,7 +63,7 @@ describe('DatasetService', () => {
       expect(fieldStats).toEqual({
         numberOfFieldsSupplied: 2,
         numberOfFieldsMatched: 1,
-        NumberOfExpectedFields: 2
+        numberOfExpectedFields: 2
       })
     })
 
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index 867d88dd..2085c8d0 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -16,7 +16,27 @@ describe('Dataset Overview Page', () => {
     },
     stats: {
       numberOfRecords: 10,
-      numberOfFieldsSupplied: 5
+      numberOfFieldsSupplied: 5,
+      numberOfFieldsMatched: 6,
+      numberOfExpectedFields: 10,
+      endpoints: [
+        {
+          name: 'endpoint 1',
+          endpoint: 'http://endpoint1.co.uk',
+          lastAccessed: '2024-09-09',
+          lastUpdated: '2024-09-09'
+        },
+        {
+          name: 'endpoint 2',
+          endpoint: 'http://endpoint2.co.uk',
+          lastAccessed: '2024-19-19',
+          lastUpdated: '2024-19-19',
+          error: {
+            code: 404,
+            exception: ''
+          }
+        }
+      ]
     }
   }
 
@@ -38,9 +58,17 @@ describe('Dataset Overview Page', () => {
   it('Renders dataset details correctly', () => {
     expect(document.querySelector('h2.govuk-heading-m').textContent).toContain('Dataset details')
     const summaryListValues = document.querySelectorAll('dd.govuk-summary-list__value')
-    expect(summaryListValues[0].textContent.trim()).toEqual('10')
-    expect(summaryListValues[1].textContent.trim()).toEqual('5')
-    expect(summaryListValues[2].textContent.trim()).toEqual('Open Government Licence')
+    expect(summaryListValues[0].textContent.trim()).toEqual(params.stats.numberOfRecords.toString())
+    expect(summaryListValues[1].textContent.trim()).toEqual(`${params.stats.numberOfFieldsSupplied}/${params.stats.numberOfExpectedFields}`)
+    expect(summaryListValues[2].textContent.trim()).toEqual(`${params.stats.numberOfFieldsMatched}/${params.stats.numberOfExpectedFields}`)
+    expect(summaryListValues[3].textContent.trim()).toEqual('Open Government Licence')
+    expect(summaryListValues[4].textContent).toContain(params.stats.endpoints[0].endpoint)
+    expect(summaryListValues[5].textContent).toContain(params.stats.endpoints[0].lastAccessed)
+    expect(summaryListValues[6].textContent).toContain(params.stats.endpoints[0].lastUpdated)
+    expect(summaryListValues[7].textContent).toContain(params.stats.endpoints[1].endpoint)
+    expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].lastAccessed)
+    expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].error.code)
+    expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].lastUpdated)
   })
 
   it('Renders breadcrumbs correctly', () => {

From 3a749b81a5dbfa7b0c5ea7aefaf2eaf24f012a11 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:07:09 +0100
Subject: [PATCH 15/65] remove papaparse

---
 package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/package.json b/package.json
index e9f8d381..c368f1f2 100644
--- a/package.json
+++ b/package.json
@@ -82,7 +82,6 @@
     "multer": "^1.4.5-lts.1",
     "notifications-node-client": "^8.2.0",
     "nunjucks": "^3.2.4",
-    "papaparse": "^5.4.1",
     "redis": "^4.6.13",
     "sass": "^1.69.4",
     "uuid": "^9.0.1",

From 64c6d24bf390f338af9cd16a7fb892f03fbc2783 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:07:36 +0100
Subject: [PATCH 16/65] update packagelock

---
 package-lock.json | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index cb91bb1e..054116bc 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -35,7 +35,6 @@
         "multer": "^1.4.5-lts.1",
         "notifications-node-client": "^8.2.0",
         "nunjucks": "^3.2.4",
-        "papaparse": "^5.4.1",
         "redis": "^4.6.13",
         "sass": "^1.69.4",
         "uuid": "^9.0.1",
@@ -9950,11 +9949,6 @@
       "version": "1.0.0",
       "license": "BlueOak-1.0.0"
     },
-    "node_modules/papaparse": {
-      "version": "5.4.1",
-      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz",
-      "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw=="
-    },
     "node_modules/parent-module": {
       "version": "1.0.1",
       "dev": true,

From 5b66978518a44dc537625f56e20ad82aa1d2cb1b Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:08:06 +0100
Subject: [PATCH 17/65] remove unused class

---
 src/assets/scss/index.scss | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss
index 05927de1..c7eefbed 100644
--- a/src/assets/scss/index.scss
+++ b/src/assets/scss/index.scss
@@ -112,7 +112,3 @@ code * {
 .padding-top {
   padding-top: 40px;
 }
-
-.light-bottom-border {
-  border-bottom: 1px solid #e0e1e2ce !important;
-}
\ No newline at end of file

From 1cc72e440a0c3bd32852e3612bca564ea231b5ae Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:23:06 +0100
Subject: [PATCH 18/65] move getDatasetsStats into dataset service

---
 src/controllers/OrganisationsController.js | 48 +++-------------------
 src/services/DatasetService.js             | 43 +++++++++++++++++++
 test/unit/services/DatasetService.test.js  |  4 ++
 3 files changed, 52 insertions(+), 43 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 3fc5d40b..0d998529 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -6,7 +6,7 @@ import { statusToTagClass } from '../filters/filters.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 import config from '../../config/index.js'
-import { getFieldStats, getSources } from '../services/DatasetService.js'
+import { getDatasetStats } from '../services/DatasetService.js'
 import {
   fetchIf,
   fetchMany,
@@ -49,49 +49,11 @@ const fetchDatasetInfo = fetchOne({
 })
 
 const fetchDatasetStats = async (req, res, next) => {
-  const { dataset, lpa } = req.params
-  const { orgInfo: organisation } = req
-
-  const { numberOfFieldsSupplied, numberOfFieldsMatched, numberOfExpectedFields } = await getFieldStats(lpa, dataset)
-
-  const sources = await getSources(lpa, dataset)
-
-  // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
-  const endpoints = sources.sort((a, b) => {
-    if (a.status >= 200 && a.status < 300) return -1
-    if (b.status >= 200 && b.status < 300) return 1
-    return 0
-  }).map((source, index) => {
-    let error
-
-    if (parseInt(source.status) <= 200 || parseInt(source.status) > 300) {
-      error = {
-        code: parseInt(source.status),
-        exception: source.exception
-      }
-    }
-
-    return {
-      name: `Data Url ${index}`,
-      endpoint: source.endpoint_url,
-      lastAccessed: source.latest_log_entry_date,
-      lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
-      error
-    }
+  req.stats = await getDatasetStats({
+    lpa: req.params.lpa,
+    dataset: req.params.dataset,
+    organisation: req.orgInfo
   })
-
-  const numberOfRecords = await performanceDbApi.getEntityCount(organisation.entity, dataset)
-
-  // ToDo: get the documentation url
-
-  req.stats = {
-    numberOfFieldsSupplied,
-    numberOfFieldsMatched,
-    numberOfExpectedFields,
-    numberOfRecords,
-    endpoints
-  }
-
   next()
 }
 
diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 7b499e9f..2af432b2 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -2,6 +2,7 @@ import datasette from './datasette.js'
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
 import JSON5 from 'json5' // need to use this as spec is written with single quotes. need to pass this onto data designers team to fix
+import performanceDbApi from './performanceDbApi.js'
 
 async function getColumnSummary (dataset, lpa) {
   const sql = `select * from column_field_summary
@@ -81,3 +82,45 @@ export async function getSources (lpa, dataset) {
 
   return formattedData
 }
+
+export async function getDatasetStats ({ dataset, lpa, organisation }) {
+  const { numberOfFieldsSupplied, numberOfFieldsMatched, numberOfExpectedFields } = await getFieldStats(lpa, dataset)
+
+  const sources = await getSources(lpa, dataset)
+
+  // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
+  const endpoints = sources.sort((a, b) => {
+    if (a.status >= 200 && a.status < 300) return -1
+    if (b.status >= 200 && b.status < 300) return 1
+    return 0
+  }).map((source, index) => {
+    let error
+
+    if (parseInt(source.status) <= 200 || parseInt(source.status) > 300) {
+      error = {
+        code: parseInt(source.status),
+        exception: source.exception
+      }
+    }
+
+    return {
+      name: `Data Url ${index}`,
+      endpoint: source.endpoint_url,
+      lastAccessed: source.latest_log_entry_date,
+      lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
+      error
+    }
+  })
+
+  const numberOfRecords = await performanceDbApi.getEntityCount(organisation.entity, dataset)
+
+  // ToDo: get the documentation url
+
+  return {
+    numberOfFieldsSupplied,
+    numberOfFieldsMatched,
+    numberOfExpectedFields,
+    numberOfRecords,
+    endpoints
+  }
+}
diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 28544863..938f2b7b 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -72,4 +72,8 @@ describe('DatasetService', () => {
       expect(fieldStats).toBeNull()
     })
   })
+
+  describe('getDatasetStats', () => {
+    // todo
+  })
 })

From 51763dcface39ed66404dfb68d6852f7bc0ba0dd Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:26:22 +0100
Subject: [PATCH 19/65] remove getentityCountForResource as its not used

---
 src/services/performanceDbApi.js | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 9b022c16..410e7d52 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -392,22 +392,9 @@ ORDER BY
   },
 
   /**
-     * Retrieves the entity count for a given resource and dataset.
-     *
-     * @param {string} resource - The resource to retrieve the entity count for.
-     * @param {string} dataset - The dataset to retrieve the entity count from.
-     * @returns {Promise<number>} The entity count for the given resource and dataset.
-     */
-  async getEntityCountForResource (resource, dataset) {
-    const query = this.entityCountQuery(resource)
-    const result = await datasette.runQuery(query, dataset)
-    return result.formattedData[0].entity_count
-  },
-
-  /**
-   * Retrieves the entity count for a given resource and dataset.
+   * Retrieves the entity count for a given organisation and dataset.
    *
-   * @param {string} resource - The resource to retrieve the entity count for.
+   * @param {string} orgEntity - The organisation entity to retrieve the entity count for.
    * @param {string} dataset - The dataset to retrieve the entity count from.
    * @returns {number} The entity count for the given resource and dataset.
    */

From c259d8f3aa22176efa819304f3ffe9403b2ef79e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:28:08 +0100
Subject: [PATCH 20/65] make use of entityCountQuery function

---
 src/services/performanceDbApi.js | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js
index 410e7d52..6b6de303 100644
--- a/src/services/performanceDbApi.js
+++ b/src/services/performanceDbApi.js
@@ -383,12 +383,12 @@ ORDER BY
     return result.formattedData
   },
 
-  entityCountQuery (resource) {
+  entityCountQuery (orgEntity) {
     return /* sql */ `
-    select dataset, entity_count, resource
-    from dataset_resource
-    WHERE resource = '${resource}'
-  `
+      select count(entity) as entity_count 
+      from entity
+      WHERE organisation_entity = '${orgEntity}'
+    `
   },
 
   /**
@@ -399,14 +399,7 @@ ORDER BY
    * @returns {number} The entity count for the given resource and dataset.
    */
   async getEntityCount (orgEntity, dataset) {
-    const query =
-    /* sql */
-    `
-    select count(entity) as entity_count 
-    from entity
-    WHERE organisation_entity = '${orgEntity}'
-  `
-
+    const query = this.entityCountQuery(orgEntity)
     const result = await datasette.runQuery(query, dataset)
     return result.formattedData[0].entity_count
   }

From 608d4e0d65280fcc18c2ee50e8a0b11da5978602 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 19 Sep 2024 16:58:26 +0100
Subject: [PATCH 21/65] get documentation-url

---
 src/routes/schemas.js                         |  3 ++-
 src/services/DatasetService.js                | 14 +++++++-------
 src/views/organisations/dataset-overview.html | 12 ++++++++++--
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index b52c296b..38e6f443 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -80,8 +80,9 @@ export const OrgDatasetOverview = v.strictObject({
     numberOfExpectedFields: v.integer(),
     endpoints: v.array(v.strictObject({
       name: v.string(),
+      documentation_url: v.optional(v.string()),
       endpoint: v.string(),
-      lastAccessed: v.string(), // ToDo: can we have date type on these?
+      lastAccessed: v.string(),
       lastUpdated: v.string(),
       error: v.optional(v.strictObject({
         code: v.integer(),
diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 2af432b2..8e013c09 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -72,13 +72,14 @@ export async function getSpecifications () {
 
 export async function getSources (lpa, dataset) {
   const sql = `
-    select endpoint, endpoint_url, status, exception, resource, latest_log_entry_date, endpoint_entry_date, endpoint_end_date, resource_start_date, resource_end_date
-    from reporting_historic_endpoints 
-    where REPLACE(organisation, '-eng', '') = '${lpa}' and pipeline = '${dataset}'
-    AND (resource_end_date >= current_timestamp OR resource_end_date is null)
+    select rhe.endpoint, rhe.endpoint_url, rhe.status, rhe.exception, rhe.resource, rhe.latest_log_entry_date, rhe.endpoint_entry_date, rhe.endpoint_end_date, rhe.resource_start_date, rhe.resource_end_date, s.documentation_url
+    from reporting_historic_endpoints rhe
+    LEFT JOIN source s ON rhe.endpoint = s.endpoint
+    where REPLACE(rhe.organisation, '-eng', '') = '${lpa}' and rhe.pipeline = '${dataset}'
+    AND (rhe.resource_end_date >= current_timestamp OR rhe.resource_end_date is null)
   `
 
-  const { formattedData } = await datasette.runQuery(sql, 'performance')
+  const { formattedData } = await datasette.runQuery(sql)
 
   return formattedData
 }
@@ -106,6 +107,7 @@ export async function getDatasetStats ({ dataset, lpa, organisation }) {
     return {
       name: `Data Url ${index}`,
       endpoint: source.endpoint_url,
+      documentation_url: source.documentation_url,
       lastAccessed: source.latest_log_entry_date,
       lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
       error
@@ -114,8 +116,6 @@ export async function getDatasetStats ({ dataset, lpa, organisation }) {
 
   const numberOfRecords = await performanceDbApi.getEntityCount(organisation.entity, dataset)
 
-  // ToDo: get the documentation url
-
   return {
     numberOfFieldsSupplied,
     numberOfFieldsMatched,
diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index 374aa4b3..fdddecff 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -84,13 +84,21 @@
       classes: 'padding-top'
     },
     value: {
-      text: endpoint.endpoint
+      html: '<a href="'+endpoint.endpoint+'">'+endpoint.endpoint+'</a>'
     },
     classes: 'padding-top'
   } %}
   {{ rows.push(endpointRow) }}
 
-
+  {% set documentationRow = {
+    key: {
+      text: 'Documentation URL'
+    },
+    value: {
+      html: '<a href="'+endpoint.documentation_url+'">'+endpoint.documentation_url+'</a>'
+    }
+  } %}
+  {{ rows.push(documentationRow) }}
 
   {% if endpoint.error %}
     {% set lastAccessedRow = {

From edcfacdf12edd2361343f69bcb532ce895a52004 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 20 Sep 2024 09:49:07 +0100
Subject: [PATCH 22/65] added testing for documentation_url in view

---
 src/views/organisations/dataset-overview.html | 20 ++++++++++---------
 .../organisations/dataset-overview.test.js    | 14 +++++++------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index fdddecff..608b868f 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -90,15 +90,17 @@
   } %}
   {{ rows.push(endpointRow) }}
 
-  {% set documentationRow = {
-    key: {
-      text: 'Documentation URL'
-    },
-    value: {
-      html: '<a href="'+endpoint.documentation_url+'">'+endpoint.documentation_url+'</a>'
-    }
-  } %}
-  {{ rows.push(documentationRow) }}
+  {% if endpoint.documentation_url %}
+    {% set documentationRow = {
+      key: {
+        text: 'Documentation URL'
+      },
+      value: {
+        html: '<a href="'+endpoint.documentation_url+'">'+endpoint.documentation_url+'</a>'
+      }
+    } %}p
+    {{ rows.push(documentationRow) }}
+  {% endif %}
 
   {% if endpoint.error %}
     {% set lastAccessedRow = {
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index 2085c8d0..187629ff 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -23,6 +23,7 @@ describe('Dataset Overview Page', () => {
         {
           name: 'endpoint 1',
           endpoint: 'http://endpoint1.co.uk',
+          documentation_url: 'http://endpoint1-docs.co.uk',
           lastAccessed: '2024-09-09',
           lastUpdated: '2024-09-09'
         },
@@ -63,12 +64,13 @@ describe('Dataset Overview Page', () => {
     expect(summaryListValues[2].textContent.trim()).toEqual(`${params.stats.numberOfFieldsMatched}/${params.stats.numberOfExpectedFields}`)
     expect(summaryListValues[3].textContent.trim()).toEqual('Open Government Licence')
     expect(summaryListValues[4].textContent).toContain(params.stats.endpoints[0].endpoint)
-    expect(summaryListValues[5].textContent).toContain(params.stats.endpoints[0].lastAccessed)
-    expect(summaryListValues[6].textContent).toContain(params.stats.endpoints[0].lastUpdated)
-    expect(summaryListValues[7].textContent).toContain(params.stats.endpoints[1].endpoint)
-    expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].lastAccessed)
-    expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].error.code)
-    expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].lastUpdated)
+    expect(summaryListValues[5].textContent).toContain(params.stats.endpoints[0].documentation_url)
+    expect(summaryListValues[6].textContent).toContain(params.stats.endpoints[0].lastAccessed)
+    expect(summaryListValues[7].textContent).toContain(params.stats.endpoints[0].lastUpdated)
+    expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].endpoint)
+    expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].lastAccessed)
+    expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].error.code)
+    expect(summaryListValues[10].textContent).toContain(params.stats.endpoints[1].lastUpdated)
   })
 
   it('Renders breadcrumbs correctly', () => {

From 012bbb901dffb3518abe344f4dec5109e25d1dd4 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 20 Sep 2024 11:47:08 +0100
Subject: [PATCH 23/65] added tests for datasetService

---
 src/services/DatasetService.js            |  28 +++--
 test/unit/services/DatasetService.test.js | 119 ++++++++++++++++++++--
 2 files changed, 131 insertions(+), 16 deletions(-)

diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 8e013c09..5b20c02b 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -16,9 +16,9 @@ async function getColumnSummary (dataset, lpa) {
   return formattedData
 }
 
-export async function getFieldStats (lpa, dataset) {
-  const columnSummary = await getColumnSummary(dataset, lpa)
-  const specifications = await getSpecifications()
+async function getFieldStats (lpa, dataset) {
+  const columnSummary = await datasetService.getColumnSummary(dataset, lpa)
+  const specifications = await datasetService.getSpecifications()
   if (!(dataset in specifications)) {
     logger.warn(`services/datasetService.getFieldStats(): cannot find specification for dataset: ${dataset}`, { type: types.app, dataset })
     return null
@@ -47,7 +47,7 @@ export async function getFieldStats (lpa, dataset) {
   }
 }
 
-export async function getSpecifications () {
+async function getSpecifications () {
   const sql = 'select * from specification order by specification'
   const result = await datasette.runQuery(sql)
 
@@ -85,9 +85,9 @@ export async function getSources (lpa, dataset) {
 }
 
 export async function getDatasetStats ({ dataset, lpa, organisation }) {
-  const { numberOfFieldsSupplied, numberOfFieldsMatched, numberOfExpectedFields } = await getFieldStats(lpa, dataset)
+  const stats = await datasetService.getFieldStats(lpa, dataset)
 
-  const sources = await getSources(lpa, dataset)
+  const sources = await datasetService.getSources(lpa, dataset)
 
   // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
   const endpoints = sources.sort((a, b) => {
@@ -97,7 +97,7 @@ export async function getDatasetStats ({ dataset, lpa, organisation }) {
   }).map((source, index) => {
     let error
 
-    if (parseInt(source.status) <= 200 || parseInt(source.status) > 300) {
+    if (parseInt(source.status) < 200 || parseInt(source.status) >= 300) {
       error = {
         code: parseInt(source.status),
         exception: source.exception
@@ -117,10 +117,18 @@ export async function getDatasetStats ({ dataset, lpa, organisation }) {
   const numberOfRecords = await performanceDbApi.getEntityCount(organisation.entity, dataset)
 
   return {
-    numberOfFieldsSupplied,
-    numberOfFieldsMatched,
-    numberOfExpectedFields,
+    numberOfFieldsSupplied: stats?.numberOfFieldsSupplied ?? 0,
+    numberOfFieldsMatched: stats?.numberOfFieldsMatched ?? 0,
+    numberOfExpectedFields: stats?.numberOfExpectedFields ?? 0,
     numberOfRecords,
     endpoints
   }
 }
+
+export const datasetService = {
+  getColumnSummary,
+  getFieldStats,
+  getSpecifications,
+  getSources,
+  getDatasetStats
+}
diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
index 938f2b7b..05d5fcf2 100644
--- a/test/unit/services/DatasetService.test.js
+++ b/test/unit/services/DatasetService.test.js
@@ -1,8 +1,10 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest'
-import { getFieldStats, getSpecifications } from '../../../src/services/DatasetService.js'
+import { datasetService } from '../../../src/services/DatasetService.js'
 import datasette from '../../../src/services/datasette.js'
+import performanceDbApi from '../../../src/services/performanceDbApi.js'
 
 vi.mock('../../../src/services/datasette')
+vi.mock('../../../src/services/performanceDbApi.js')
 
 describe('DatasetService', () => {
   describe('getSpecifications', () => {
@@ -33,7 +35,7 @@ describe('DatasetService', () => {
     }
 
     it('returns specifications in the correct format', async () => {
-      const specifications = await getSpecifications()
+      const specifications = await datasetService.getSpecifications()
       expect(specifications).toEqual(expectedSpecifications)
     })
   })
@@ -59,7 +61,7 @@ describe('DatasetService', () => {
     })
 
     it('returns field stats', async () => {
-      const fieldStats = await getFieldStats('lpa', 'dataset1')
+      const fieldStats = await datasetService.getFieldStats('lpa', 'dataset1')
       expect(fieldStats).toEqual({
         numberOfFieldsSupplied: 2,
         numberOfFieldsMatched: 1,
@@ -68,12 +70,117 @@ describe('DatasetService', () => {
     })
 
     it('returns null if dataset specification is missing', async () => {
-      const fieldStats = await getFieldStats('lpa', 'nonExistentDataset')
+      const fieldStats = await datasetService.getFieldStats('lpa', 'nonExistentDataset')
       expect(fieldStats).toBeNull()
     })
   })
 
-  describe('getDatasetStats', () => {
-    // todo
+  describe('getDatasetStats', async () => {
+    const lpa = 'lpa-example'
+    const dataset = 'dataset-example'
+    const organisation = 'organisation-example'
+
+    const getFieldStatsSpy = vi.spyOn(datasetService, 'getFieldStats')
+    const getSourcesSpy = vi.spyOn(datasetService, 'getSources')
+
+    beforeEach(() => {
+      getFieldStatsSpy.mockImplementation(() => {
+        return {
+          numberOfFieldsSupplied: 2,
+          numberOfFieldsMatched: 1,
+          numberOfExpectedFields: 2
+        }
+      })
+      getSourcesSpy.mockImplementation(() => [
+        {
+          status: 200,
+          endpoint_url: 'https://example.com/endpoint1',
+          documentation_url: 'https://example.com/doc1',
+          latest_log_entry_date: '2023-02-20T14:30:00Z',
+          endpoint_entry_date: '2023-02-19T12:00:00Z'
+        },
+        {
+          status: 404,
+          endpoint_url: 'https://example.com/endpoint2',
+          documentation_url: 'https://example.com/doc2',
+          latest_log_entry_date: '2023-02-20T14:30:00Z',
+          endpoint_entry_date: '2023-02-19T12:00:00Z',
+          exception: 'Not Found'
+        }
+      ])
+      performanceDbApi.getEntityCount = vi.fn().mockResolvedValueOnce(100)
+    })
+
+    it('returns dataset stats with field stats and sources', async () => {
+      const datasetStats = await datasetService.getDatasetStats({ dataset, lpa, organisation })
+
+      expect(datasetStats).toEqual({
+        numberOfFieldsSupplied: 2,
+        numberOfFieldsMatched: 1,
+        numberOfExpectedFields: 2,
+        numberOfRecords: 100,
+        endpoints: [
+          {
+            name: 'Data Url 0',
+            endpoint: 'https://example.com/endpoint1',
+            documentation_url: 'https://example.com/doc1',
+            lastAccessed: '2023-02-20T14:30:00Z',
+            lastUpdated: '2023-02-19T12:00:00Z'
+          },
+          {
+            name: 'Data Url 1',
+            endpoint: 'https://example.com/endpoint2',
+            documentation_url: 'https://example.com/doc2',
+            lastAccessed: '2023-02-20T14:30:00Z',
+            lastUpdated: '2023-02-19T12:00:00Z',
+            error: {
+              code: 404,
+              exception: 'Not Found'
+            }
+          }
+        ]
+      })
+    })
+
+    it('returns 0s if getFieldStats returns null', async () => {
+      getFieldStatsSpy.mockResolvedValueOnce(null)
+      const datasetStats = await datasetService.getDatasetStats({ dataset, lpa, organisation })
+      expect(datasetStats).toEqual({
+        numberOfFieldsSupplied: 0,
+        numberOfFieldsMatched: 0,
+        numberOfExpectedFields: 0,
+        numberOfRecords: 100,
+        endpoints: [
+          {
+            name: 'Data Url 0',
+            endpoint: 'https://example.com/endpoint1',
+            documentation_url: 'https://example.com/doc1',
+            lastAccessed: '2023-02-20T14:30:00Z',
+            lastUpdated: '2023-02-19T12:00:00Z'
+          },
+          {
+            name: 'Data Url 1',
+            endpoint: 'https://example.com/endpoint2',
+            documentation_url: 'https://example.com/doc2',
+            lastAccessed: '2023-02-20T14:30:00Z',
+            lastUpdated: '2023-02-19T12:00:00Z',
+            error: {
+              code: 404,
+              exception: 'Not Found'
+            }
+          }
+        ]
+      })
+    })
+
+    it('throws an error if getFieldStats throws an error', async () => {
+      getFieldStatsSpy.mockRejectedValueOnce(new Error('Error fetching sources'))
+      await expect(datasetService.getDatasetStats({ dataset, lpa, organisation })).rejects.toThrowError('Error fetching sources')
+    })
+
+    it('throws an error if getSources throws an error', async () => {
+      getSourcesSpy.mockRejectedValueOnce(new Error('Error fetching sources'))
+      await expect(datasetService.getDatasetStats({ dataset, lpa, organisation })).rejects.toThrowError('Error fetching sources')
+    })
   })
 })

From 67ea2648c69f6222b56ba61c9ac3b6192aa0cdba Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 11:39:46 +0100
Subject: [PATCH 24/65] started pulling out middlewares into own files

---
 src/middleware/common.middleware.js          |  58 ++++
 src/middleware/datasetTaskList.middleware.js | 135 +++++++++
 src/middleware/middleware.builders.js        | 276 +++++++++++++++++++
 3 files changed, 469 insertions(+)
 create mode 100644 src/middleware/common.middleware.js
 create mode 100644 src/middleware/datasetTaskList.middleware.js
 create mode 100644 src/middleware/middleware.builders.js

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
new file mode 100644
index 00000000..7a772a12
--- /dev/null
+++ b/src/middleware/common.middleware.js
@@ -0,0 +1,58 @@
+import logger from '../utils/logger.js'
+import { types } from '../utils/logging.js'
+import performanceDbApi from '../services/performanceDbApi.js'
+import { fetchOne } from './middleware.builders.js'
+
+/**
+ * Middleware. Set `req.handlerName` to a string that will identify
+ * the function that threw the error.
+ *
+ * @param {Error} err
+ * @param {{handlerName: string}} req
+ * @param {*} res
+ * @param {*} next
+ */
+export const logPageError = (err, req, res, next) => {
+  console.assert(req.handlerName, 'handlerName missing ')
+  logger.warn({
+    message: `OrganisationsController.${req.handlerName}(): ${err.message}`,
+    endpoint: req.originalUrl,
+    errorStack: err.stack,
+    errorMessage: err.message,
+    type: types.App
+  })
+  next(err)
+}
+
+export const fetchDatasetInfo = fetchOne({
+  query: ({ params }) => {
+    return `SELECT name, dataset FROM dataset WHERE dataset = '${params.dataset}'`
+  },
+  result: 'dataset'
+})
+
+export const isResourceIdInParams = ({ params }) => !('resourceId' in params)
+
+/**
+ * Was the resource accessed successfully via HTTP?
+ *
+ * @param {*} req
+ * @returns {boolean}
+ */
+export const isResourceAccessible = (req) => req.resourceStatus.status === '200'
+export const isResourceNotAccessible = (req) => !isResourceAccessible(req)
+
+/**
+ * Middleware. Updates req with `resource`.
+ */
+export const fetchLatestResource = fetchOne({
+  query: ({ params }) => performanceDbApi.latestResourceQuery(params.lpa, params.dataset),
+  result: 'resource'
+})
+
+const fetchEntityCount = fetchOne({
+  query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource),
+  result: 'entityCount',
+  dataset: FetchOptions.fromParams,
+  fallbackPolicy: FetchOneFallbackPolicy.continue
+})
diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
new file mode 100644
index 00000000..ed32979b
--- /dev/null
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -0,0 +1,135 @@
+import { fetchDatasetInfo, isResourceAccessible, isResourceNotAccessible, fetchLatestResource, fetchEntityCount, logPageError } from './common.middleware'
+import { parallel, fetchOne, fetchIf, fetchMany, onlyIf, renderTemplate } from './middleware.builders'
+import performanceDbApi from '../services/performanceDbApi.js'
+import { statusToTagClass } from '../filters/filters.js'
+
+/**
+ * Fetches the resource status
+ */
+export const fetchResourceStatus = fetchOne({
+  query: ({ params }) => performanceDbApi.resourceStatusQuery(params.lpa, params.dataset),
+  result: 'resourceStatus'
+})
+
+const fetchOrgInfoWithStatGeo = fetchOne({
+  query: ({ params }) => {
+    return /* sql */ `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
+  },
+  result: 'orgInfo'
+})
+
+const fetchLpaDatasetIssues = fetchMany({
+  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
+  result: 'issues'
+})
+
+/**
+ * Returns a status tag object with a text label and a CSS class based on the status.
+ *
+ * @param {string} status - The status to generate a tag for (e.g. "Error", "Needs fixing", etc.)
+ * @returns {object} - An object with a `tag` property containing the text label and CSS class.
+ */
+function getStatusTag (status) {
+  return {
+    tag: {
+      text: status,
+      classes: statusToTagClass(status)
+    }
+  }
+}
+
+/**
+ * Middleware. Updates req with `templateParams`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ * @returns { { templateParams: object }}
+ */
+const prepareDatasetTaskListTemplateParams = (req, res, next) => {
+  const { issues, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req
+  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
+  const { lpa, dataset: datasetId } = params
+  console.assert(req.resourceStatus.resource === req.resource.resource, 'mismatch between resourceStatus and resource data')
+  console.assert(typeof entityCount === 'number', 'entityCount should be a number')
+
+  const taskList = issues.map((issue) => {
+    return {
+      title: {
+        text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field })
+      },
+      href: `/organisations/${lpa}/${datasetId}/${issue.issue_type}/${issue.field}`,
+      status: getStatusTag(issue.status)
+    }
+  })
+
+  req.templateParams = {
+    taskList,
+    organisation,
+    dataset
+  }
+
+  next()
+}
+
+const getDatasetTaskList = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/datasetTaskList.html',
+  handlerName: 'getDatasetTaskList'
+})
+
+/**
+ * Middleware. Updates req with `templateParams`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {} next
+ * @returns {{ templateParams: object }}
+ */
+const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
+  const { orgInfo: organisation, dataset, resourceStatus: resource } = req
+
+  const daysSince200 = resource.days_since_200
+  const today = new Date()
+  const last200Date = new Date(
+    today.getTime() - daysSince200 * 24 * 60 * 60 * 1000
+  )
+  const last200Datetime = last200Date.toISOString().slice(0, 19) + 'Z'
+
+  req.templateParams = {
+    organisation,
+    dataset,
+    errorData: {
+      endpoint_url: resource.endpoint_url,
+      http_status: resource.status,
+      latest_log_entry_date: resource.latest_log_entry_date,
+      latest_200_date: last200Datetime
+    }
+  }
+
+  next()
+}
+
+const getDatasetTaskListError = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/http-error.html',
+  handlerName: 'getDatasetTaskListError'
+})
+
+export default [
+  fetchResourceStatus,
+  parallel([
+    fetchOrgInfoWithStatGeo,
+    fetchDatasetInfo]),
+  fetchIf(isResourceAccessible, fetchLatestResource),
+  parallel([
+    fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
+    fetchIf(isResourceAccessible, fetchEntityCount)
+  ]),
+  onlyIf(isResourceAccessible, prepareDatasetTaskListTemplateParams),
+  onlyIf(isResourceAccessible, getDatasetTaskList),
+
+  onlyIf(isResourceNotAccessible, prepareDatasetTaskListErrorTemplateParams),
+  onlyIf(isResourceNotAccessible, getDatasetTaskListError),
+  logPageError
+]
diff --git a/src/middleware/middleware.builders.js b/src/middleware/middleware.builders.js
new file mode 100644
index 00000000..45dbd70d
--- /dev/null
+++ b/src/middleware/middleware.builders.js
@@ -0,0 +1,276 @@
+import logger from '../utils/logger.js'
+import { types } from '../utils/logging.js'
+import { templateSchema } from '../routes/schemas.js'
+import { render } from '../utils/custom-renderer.js'
+import datasette from '../services/datasette.js'
+import * as v from 'valibot'
+
+export const FetchOptions = {
+  /**
+     * Use 'dataset' from requets params.
+     */
+  fromParams: Symbol('from-params')
+}
+
+const datasetOverride = (val, req) => {
+  if (!val) {
+    return 'digital-land'
+  }
+  if (val === FetchOptions.fromParams) {
+    console.assert(req.params.dataset, 'no "dataset" in request params')
+    return req.params.dataset
+  } else {
+    return val(req)
+  }
+}
+
+const fetchOneFallbackPolicy = (req, res, next) => {
+  res.status(404).render('errorPages/404', {})
+}
+
+/**
+     * Collection of fallback policies for the {@link fetchOneFn} middleware.
+     * The policy is enacted when zero records is returned from the data source.
+     */
+export const FetchOneFallbackPolicy = {
+  /**
+       * Renders a 404 response.
+       */
+  'not-found-error': fetchOneFallbackPolicy,
+
+  /**
+       * Proceeds by calling `next()`.
+       */
+  continue: (_req, _res, next) => next()
+}
+
+/**
+ * Middleware. Attempts to fetch data from datasette and short-circuits with 404 when
+ * data for given query does not exist. Meant to be used to fetch singular records.
+ *
+ * `this` needs `{ query({ req, params }) => any, result: string, dataset?: FetchParams | (req) => string, fallbackPolicy: (req, res, next) => void }`
+ *
+ * where the `result` is the key under which result of the query will be stored in `req`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+async function fetchOneFn (req, res, next) {
+  logger.debug({ type: types.DataFetch, message: 'fetchOne', resultKey: this.result })
+  try {
+    const query = this.query({ req, params: req.params })
+    const result = await datasette.runQuery(query, datasetOverride(this.dataset, req))
+    const fallbackPolicy = this.fallbackPolicy ?? FetchOneFallbackPolicy['not-found-error']
+    if (result.formattedData.length === 0) {
+      // we can make the 404 more informative by informing the use what exactly was "not found"
+      fallbackPolicy(req, res, next)
+    } else {
+      req[this.result] = result.formattedData[0]
+      next()
+    }
+  } catch (error) {
+    logger.debug('fetchOne: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
+    req.handlerName = `fetching '${this.result}'`
+    next(error)
+  }
+}
+
+/**
+   * Middleware. Attempts to fetch a collection of data from datasette.
+   *
+   * `this` needs `{ query( {req, params } ) => any, result: string, dataset?: FetchParams | (req) => string }`
+   *
+   * @param {*} req
+   * @param {*} res
+   * @param {*} next
+   */
+export async function fetchManyFn (req, res, next) {
+  try {
+    const query = this.query({ req, params: req.params })
+    const result = await datasette.runQuery(query, datasetOverride(this.dataset, req))
+    req[this.result] = result.formattedData
+    logger.debug({ type: types.DataFetch, message: 'fetchMany', resultKey: this.result, resultCount: result.formattedData.length })
+    next()
+  } catch (error) {
+    logger.debug('fetchMany: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
+    req.handlerName = `fetching '${this.result}'`
+    next(error)
+  }
+}
+
+/**
+   * Middleware. Does a conditional fetch. Optionally invokes `else` if condition is false.
+   *
+   * `this` needs: `{ fetchFn, condition: (req) => boolean, else?: (req) => void }`
+   *
+   * `fetchFn` should be a middleware fn. Can be async.
+   *
+   * @param {*} req
+   * @param {*} res
+   * @param {*} next
+   */
+async function fetchIfFn (req, res, next) {
+  if (this.condition(req)) {
+    // `next` will be called in our fetchFn middleware
+    const result = this.fetchFn(req, res, next)
+    if (result instanceof Promise) {
+      await result
+    }
+  } else {
+    if (this.else) {
+      this.else(req)
+    }
+    next()
+  }
+}
+
+/**
+ *
+ * Returns a middleware that will fetch data if condition is satisfied,
+ * invoke `elseFn` otherwise.
+ *
+ * @param {(req) => boolean} condition predicate fn
+ * @param {*} fetchFn fetch middleware
+ * @param {(req) => void} elseFn
+ * @returns
+ */
+export const fetchIf = (condition, fetchFn, elseFn) => {
+  return fetchIfFn.bind({
+    condition, fetchFn, else: elseFn
+  })
+}
+
+/**
+ * Fetches a single entity and stores it in `req` under key specified by `result` entry.
+ *
+ * Use `fallbackPolicy` to handle zero record responses differently than the default 404 response.
+ * See {@link FetchOneFallbackPolicy}
+ *
+ * @param {{ query: ({ req, params}) => object, result: string, dataset?: FetchParams | (req) => string, fallbackPolicy?: (req, res, next) => void}} context
+ * @returns
+ */
+export function fetchOne (context) {
+  return fetchOneFn.bind(context)
+}
+
+/**
+   * Fetches a collection of records and stores them in `req` under key specified by `result` entry.
+   *
+   * @param {{query: ({req, params}) => object, result: string, dataset?: FetchParams | (req) => string}} context
+   */
+export function fetchMany (context) {
+  return fetchManyFn.bind(context)
+}
+
+/**
+ * Looks up schema for name in @{link templateSchema} (defaults to any()), validates and renders the template.
+ *
+ * @param {*} res
+ * @param {string} name
+ * @param {*} params
+ * @returns {string}
+ */
+export function validateAndRender (res, name, params) {
+  const schema = templateSchema.get(name) ?? v.any()
+  logger.info(
+        `rendering '${name}' with schema=<${schema ? 'defined' : 'any'}>`,
+        { type: types.App }
+  )
+  return render(res, name, schema, params)
+}
+
+/**
+   * Middleware. Validates and renders the template.
+   *
+   * `this` needs: `{ templateParams(req), template,  handlerName }`
+   *
+   * @param {*} req
+   * @param {*} res
+   * @param {*} next
+   * @returns
+   */
+export function renderTemplateFn (req, res, next) {
+  const templateParams = this.templateParams(req)
+  try {
+    validateAndRender(res, this.template, templateParams)
+    logger.info(`rendered ${this.template}`, { type: types.App })
+  } catch (err) {
+    req.handlerName = this.handlerName
+    next(err)
+  }
+}
+
+/**
+   * Validates and renders the template.
+   *
+   *
+   * @param {{templateParams: (req) => object, template: string,  handlerName: string}} context
+   * @returns {(req, res, next) => void} middleware
+   */
+export function renderTemplate (context) {
+  return renderTemplateFn.bind(context)
+}
+
+/**
+ * Returns a middleware that executes the given sub-middlewares in parallel
+ * and waits for all of them to complete.
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ * @returns {(req, res, next) => void} middleware
+ */
+async function parallelFn (req, res, next) {
+  const fns = this.middlewares
+  const nextParams = []
+  const nextFn = (val) => {
+    if (val) nextParams.push(val)
+  }
+  // We need to take care of explicit `next(value)`, so we hijack the 'next' callback.
+  // We also need to hadle any rejected promises in results
+  const results = await Promise.allSettled(fns.map(fn => fn(req, res, nextFn)))
+  /* eslint-disable no-unreachable-loop */
+  for (const param of nextParams) {
+    if (param instanceof Error) {
+      logger.debug('parallel: captured a "next" error', { type: types.App, errorMessage: param.message })
+    }
+    next(param)
+    return
+  }
+  for (const result of results) {
+    if (result.status === 'rejected') {
+      logger.debug('parallel: got a rejected promise', { type: types.App })
+      next(result.reason)
+      return
+    }
+  }
+
+  next()
+}
+
+/**
+   * Returns a middleware that invokes the passed middlewares in parallel.
+   *
+   * Usage: when all sub-middlewre can be fetched independently, but we can't accept a partial success
+   * (e.g. we require all middlewares to succeed).
+   *
+   * @param {((req,res,next) => void)[]} middlewares array of middleware functions
+   * @returns {(req, res, next) => Promise<void>}
+   */
+export function parallel (middlewares) {
+  return parallelFn.bind({ middlewares })
+}
+
+export const onlyIf = (condition, middlewareFn) => {
+  return async (req, res, next) => {
+    if (condition(req)) {
+      const result = middlewareFn(req, res, next)
+      if (result instanceof Promise) {
+        await result
+      }
+    } else {
+      next()
+    }
+  }
+}

From f356cb4819097173fb713f11a2074a8af5611945 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 11:45:06 +0100
Subject: [PATCH 25/65] added get started middleware to own file

---
 src/middleware/common.middleware.js     | 11 +++++++++--
 src/middleware/getStarted.middleware.js | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 src/middleware/getStarted.middleware.js

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 7a772a12..85b786c1 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -1,7 +1,7 @@
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchOne } from './middleware.builders.js'
+import { fetchOne, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js'
 
 /**
  * Middleware. Set `req.handlerName` to a string that will identify
@@ -50,9 +50,16 @@ export const fetchLatestResource = fetchOne({
   result: 'resource'
 })
 
-const fetchEntityCount = fetchOne({
+export const fetchEntityCount = fetchOne({
   query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource),
   result: 'entityCount',
   dataset: FetchOptions.fromParams,
   fallbackPolicy: FetchOneFallbackPolicy.continue
 })
+
+export const fetchOrgInfo = fetchOne({
+  query: ({ params }) => {
+    return `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
+  },
+  result: 'orgInfo'
+})
diff --git a/src/middleware/getStarted.middleware.js b/src/middleware/getStarted.middleware.js
new file mode 100644
index 00000000..55bbeeb7
--- /dev/null
+++ b/src/middleware/getStarted.middleware.js
@@ -0,0 +1,18 @@
+import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware'
+import { renderTemplate } from './middleware.builders'
+
+const getGetStarted = renderTemplate({
+  templateParams (req) {
+    const { orgInfo: organisation, dataset } = req
+    return { organisation, dataset }
+  },
+  template: 'organisations/get-started.html',
+  handlerName: 'getStarted'
+})
+
+export default [
+  fetchOrgInfo,
+  fetchDatasetInfo,
+  getGetStarted,
+  logPageError
+]

From 24edaeebb9cb9c1fff177797a05c53a0eadc7ff2 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 11:48:18 +0100
Subject: [PATCH 26/65] added dataset stats middleware to own file

---
 src/middleware/datasetOverview.middleware.js | 28 ++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 src/middleware/datasetOverview.middleware.js

diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
new file mode 100644
index 00000000..29b6e864
--- /dev/null
+++ b/src/middleware/datasetOverview.middleware.js
@@ -0,0 +1,28 @@
+import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware'
+import { renderTemplate } from './middleware.builders'
+import { getDatasetStats } from '../services/DatasetService.js'
+
+const fetchDatasetStats = async (req, res, next) => {
+  req.stats = await getDatasetStats(req.params.dataset, req.params.lpa)
+
+  next()
+}
+
+const getDatasetOverview = renderTemplate(
+  {
+    templateParams (req) {
+      const { orgInfo: organisation, dataset, stats } = req
+      return { organisation, dataset, stats }
+    },
+    template: 'organisations/dataset-overview.html',
+    handlerName: 'datasetOverview'
+  }
+)
+
+export default [
+  fetchOrgInfo,
+  fetchDatasetInfo,
+  fetchDatasetStats,
+  getDatasetOverview,
+  logPageError
+]

From 059f39d46c1634e76fce5dd664b9484bbeb28c6f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 11:51:57 +0100
Subject: [PATCH 27/65] added overview middleware to own file

---
 src/middleware/overview.middleware.js | 100 ++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 src/middleware/overview.middleware.js

diff --git a/src/middleware/overview.middleware.js b/src/middleware/overview.middleware.js
new file mode 100644
index 00000000..84c786ef
--- /dev/null
+++ b/src/middleware/overview.middleware.js
@@ -0,0 +1,100 @@
+import performanceDbApi from '../services/performanceDbApi'
+import { fetchOrgInfo, logPageError } from './common.middleware'
+import { renderTemplate } from './middleware.builders'
+import { dataSubjects } from '../utils/utils.js'
+import config from '../../config/index.js'
+
+// get a list of available datasets
+const availableDatasets = Object.values(dataSubjects).flatMap((dataSubject) =>
+  dataSubject.dataSets
+    .filter((dataset) => dataset.available)
+    .map((dataset) => dataset.value)
+)
+
+/**
+ * Middleware.
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ * @returns {Promise<*>}
+ */
+async function fetchLpaOverview (req, res, next) {
+  const { datasetsFilter } = config
+  try {
+    const overview = await performanceDbApi.getLpaOverview(req.params.lpa, {
+      datasetsFilter
+    })
+    req.lpaOverview = overview
+    next()
+  } catch (error) {
+    next(error)
+  }
+}
+
+function prepareOverviewTemplateParams (req, res, next) {
+  const { lpaOverview, orgInfo: organisation } = req
+
+  const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => {
+    return {
+      slug: key,
+      ...value
+    }
+  })
+
+  // add in any of the missing key 8 datasets
+  const keys = Object.keys(lpaOverview.datasets)
+  availableDatasets.forEach((dataset) => {
+    if (!keys.includes(dataset)) {
+      datasets.push({
+        slug: dataset,
+        endpoint: null,
+        issue_count: 0,
+        status: 'Not submitted'
+      })
+    }
+  })
+
+  // re-sort the datasets to be in alphabetical order
+  datasets.sort((a, b) => a.slug.localeCompare(b.slug))
+
+  const totalDatasets = datasets.length
+  const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] =
+      datasets.reduce(
+        (accumulator, dataset) => {
+          if (dataset.endpoint) accumulator[0]++
+          if (dataset.status === 'Needs fixing') accumulator[1]++
+          if (dataset.status === 'Error') accumulator[2]++
+          return accumulator
+        },
+        [0, 0, 0]
+      )
+
+  req.templateParams = {
+    organisation,
+    datasets,
+    totalDatasets,
+    datasetsWithEndpoints,
+    datasetsWithIssues,
+    datasetsWithErrors
+  }
+
+  next()
+}
+
+const getOverview = renderTemplate({
+  templateParams (req) {
+    if (!req.templateParams) throw new Error('missing templateParams')
+    return req.templateParams
+  },
+  template: 'organisations/overview.html',
+  handlerName: 'getOverview'
+})
+
+export default [
+  fetchOrgInfo,
+  fetchLpaOverview,
+  prepareOverviewTemplateParams,
+  getOverview,
+  logPageError
+]

From af71409fa5f883273c6836717f9ccaac01f959bc Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 12:03:57 +0100
Subject: [PATCH 28/65] adding issueDetails middleware to own file

---
 src/middleware/issueDetails.middleware.js | 305 ++++++++++++++++++++++
 1 file changed, 305 insertions(+)
 create mode 100644 src/middleware/issueDetails.middleware.js

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
new file mode 100644
index 00000000..ca2af3b3
--- /dev/null
+++ b/src/middleware/issueDetails.middleware.js
@@ -0,0 +1,305 @@
+import performanceDbApi from '../services/performanceDbApi'
+import logger from '../utils/logger'
+import { types } from '../utils/logging.js'
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware'
+import { fetchIf, parallel, renderTemplate } from './middleware.builders'
+import * as v from 'vali'
+import { pagination } from '../utils/pagination.js'
+
+const IssueDetailsQueryParams = v.object({
+  lpa: v.string(),
+  dataset: v.string(),
+  issue_type: v.string(),
+  issue_field: v.string(),
+  pageNumber: v.optional(v.string()),
+  resourceId: v.optional(v.string())
+})
+
+const validateIssueDetailsQueryParams = validateQueryParams.bind({
+  schema: IssueDetailsQueryParams
+})
+
+/**
+ *
+ * Middleware. Updates `req` with `issues`.
+ *
+ * Requires `resourceId` in request params or request (in that order).
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+async function fetchIssues (req, res, next) {
+  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { resource: resourceId } = req.resource
+  if (!resourceId) {
+    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
+    throw Error('fetchIssues: missing resourceId')
+  }
+
+  try {
+    const issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
+    req.issues = issues
+    next()
+  } catch (error) {
+    next(error)
+  }
+}
+
+/**
+ *
+ * Middleware. Updates `req` with `issues`.
+ *
+ * Requires `issues` in request.
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+async function reformatIssuesToBeByEntryNumber (req, res, next) {
+  const { issues } = req
+  const issuesByEntryNumber = issues.reduce((acc, current) => {
+    acc[current.entry_number] = acc[current.entry_number] || []
+    acc[current.entry_number].push(current)
+    return acc
+  }, {})
+  req.issuesByEntryNumber = issuesByEntryNumber
+  next()
+}
+
+/**
+ *
+ * Middleware. Updates `req` with `entryData`
+ *
+ * Requires `pageNumber`, `dataset` and
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ *
+ */
+async function fetchEntry (req, res, next) {
+  const { dataset: datasetId, pageNumber } = req.params
+  const { issuesByEntryNumber } = req
+  const pageNum = pageNumber ? parseInt(pageNumber) : 1
+  req.pageNumber = pageNum
+
+  // look at issue Entries and get the index of that entry - 1
+
+  const entityNum =
+      Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
+
+  req.entryData = await performanceDbApi.getEntry(
+    req.resourceId,
+    entityNum,
+    datasetId
+  )
+  req.entryNumber = entityNum
+  next()
+}
+
+/**
+ *
+ * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
+ *
+ * Requires `req.resource.resource`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+async function fetchIssueEntitiesCount (req, res, next) {
+  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { resource: resourceId } = req.resource
+  console.assert(resourceId, 'missng resource id')
+  const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId)
+  req.issueEntitiesCount = parseInt(issueEntitiesCount)
+  next()
+}
+
+/**
+ *
+ * @param {string} errorMessage
+ * @param {{value: string}?} issue
+ * @returns {string}
+ */
+const issueErrorMessageHtml = (errorMessage, issue) =>
+    `<p class="govuk-error-message">${errorMessage}</p>${
+      issue ? issue.value ?? '' : ''
+    }`
+
+/**
+ *
+ * @param {*} text
+ * @param {*} html
+ * @param {*} classes
+ * @returns {{key: {text: string}, value: { html: string}, classes: string}}
+ */
+const getIssueField = (text, html, classes) => {
+  return {
+    key: {
+      text
+    },
+    value: {
+      html
+    },
+    classes
+  }
+}
+
+/**
+ *
+ * @param {*} issueType
+ * @param {*} issuesByEntryNumber
+ * @param {*} row
+ * @returns {{key: {text: string}, value: { html: string}, classes: string}}
+ */
+const processEntryRow = (issueType, issuesByEntryNumber, row) => {
+  const { entry_number: entryNumber } = row
+  console.assert(entryNumber, 'precessEntryRow(): entry_number not in row')
+  let hasError = false
+  let issueIndex
+  if (issuesByEntryNumber[entryNumber]) {
+    issueIndex = issuesByEntryNumber[entryNumber].findIndex(
+      (issue) => issue.field === row.field
+    )
+    hasError = issueIndex >= 0
+  }
+
+  let valueHtml = ''
+  let classes = ''
+  if (hasError) {
+    const message =
+        issuesByEntryNumber[entryNumber][issueIndex].message || issueType
+    valueHtml += issueErrorMessageHtml(message, null)
+    classes += 'dl-summary-card-list__row--error'
+  }
+  valueHtml += row.value
+
+  return getIssueField(row.field, valueHtml, classes)
+}
+
+/***
+ * Middleware. Updates req with `templateParams`
+ */
+function prepareIssueDetailsTemplateParams (req, res, next) {
+  const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount: entityCountRow } = req
+  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
+  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
+
+  let errorHeading
+  let issueItems
+
+  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
+
+  if (Object.keys(issuesByEntryNumber).length < entityCount) {
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
+    issueItems = Object.entries(issuesByEntryNumber).map(([entryNumber, issues], i) => {
+      const pageNum = i + 1
+      return {
+        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
+        href: `${BaseSubpath}${pageNum}`
+      }
+    })
+  } else {
+    issueItems = [{
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
+    }]
+  }
+
+  const fields = entryData.map((row) => processEntryRow(issueType, issuesByEntryNumber, row))
+  const entityIssues = Object.values(issuesByEntryNumber)[pageNumber - 1] || []
+  for (const issue of entityIssues) {
+    if (!fields.find((field) => field.key.text === issue.field)) {
+      const errorMessage = issue.message || issueType
+      // TODO: pull the html out of here and into the template
+      const valueHtml = issueErrorMessageHtml(errorMessage, issue.value)
+      const classes = 'dl-summary-card-list__row--error'
+
+      fields.push(getIssueField(issue.field, valueHtml, classes))
+    }
+  }
+
+  const geometries = entryData
+    .filter((row) => row.field === 'geometry')
+    .map((row) => row.value)
+  const entry = {
+    title: `entry: ${entryNumber}`,
+    fields,
+    geometries
+  }
+
+  const paginationObj = {}
+  if (pageNumber > 1) {
+    paginationObj.previous = {
+      href: `${BaseSubpath}${pageNumber - 1}`
+    }
+  }
+
+  if (pageNumber < issueEntitiesCount) {
+    paginationObj.next = {
+      href: `${BaseSubpath}${pageNumber + 1}`
+    }
+  }
+
+  paginationObj.items = pagination(issueEntitiesCount, pageNumber).map(item => {
+    if (item === '...') {
+      return {
+        type: 'ellipsis',
+        ellipsis: true,
+        href: '#'
+      }
+    } else {
+      return {
+        type: 'item',
+        number: item,
+        href: `${BaseSubpath}${item}`,
+        current: pageNumber === parseInt(item)
+      }
+    }
+  })
+
+  // schema: OrgIssueDetails
+  req.templateParams = {
+    organisation: req.orgInfo,
+    dataset: req.dataset,
+    errorHeading,
+    issueItems,
+    entry,
+    issueType,
+    pagination: paginationObj,
+    issueEntitiesCount,
+    pageNumber
+  }
+
+  next()
+}
+
+/**
+ * Middleware. Renders the issue details page with the list of issues, entry data,
+ * and organisation and dataset details.
+ */
+const getIssueDetails = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/issueDetails.html',
+  handlerName: 'getIssueDetails'
+})
+
+export default [
+  validateIssueDetailsQueryParams,
+  parallel([
+    fetchOrgInfo,
+    fetchDatasetInfo
+  ]),
+  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  fetchIssues,
+  reformatIssuesToBeByEntryNumber,
+  parallel([
+    fetchEntry,
+    fetchEntityCount,
+    fetchIssueEntitiesCount
+  ]),
+  prepareIssueDetailsTemplateParams,
+  getIssueDetails,
+  logPageError
+]

From f8b41a4002ea5ffadb9ef4c8e8586c977d421f82 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 12:06:24 +0100
Subject: [PATCH 29/65] add organisations middleware

---
 src/middleware/organisations.middleware.js | 44 ++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 src/middleware/organisations.middleware.js

diff --git a/src/middleware/organisations.middleware.js b/src/middleware/organisations.middleware.js
new file mode 100644
index 00000000..9a98df70
--- /dev/null
+++ b/src/middleware/organisations.middleware.js
@@ -0,0 +1,44 @@
+import { logPageError } from './common.middleware'
+import { fetchMany, renderTemplate } from './middleware.builders'
+
+const fetchOrganisations = fetchMany({
+  query: ({ req, params }) => 'select name, organisation from organisation',
+  result: 'organisations'
+})
+
+/**
+ * Middleware. Updates req with `templateParams`.
+ *
+ * @param {{ organisations: {}[] }} req
+ * @param {*} res
+ * @param {*} next
+ */
+const prepareGetOrganisationsTemplateParams = (req, res, next) => {
+  const sortedResults = req.organisations.sort((a, b) => {
+    return a.name.localeCompare(b.name)
+  })
+
+  const alphabetisedOrgs = sortedResults.reduce((acc, current) => {
+    const firstLetter = current.name.charAt(0).toUpperCase()
+    acc[firstLetter] = acc[firstLetter] || []
+    acc[firstLetter].push(current)
+    return acc
+  }, {})
+
+  req.templateParams = { alphabetisedOrgs }
+
+  next()
+}
+
+const getOrganisations = renderTemplate({
+  templateParams: (req) => req.templateParams,
+  template: 'organisations/find.html',
+  handlerName: 'getOrganisations'
+})
+
+export default [
+  fetchOrganisations,
+  prepareGetOrganisationsTemplateParams,
+  getOrganisations,
+  logPageError
+]

From 11b91b7022775b2a46691b9c0b84b8c41818e91f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 12:11:07 +0100
Subject: [PATCH 30/65] removed old code from organisationsController

---
 src/controllers/OrganisationsController.js | 731 +--------------------
 1 file changed, 9 insertions(+), 722 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 57670538..130debf2 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -1,730 +1,17 @@
-import performanceDbApi from '../services/performanceDbApi.js' // Assume you have an API service module
-import logger from '../utils/logger.js'
-import { types } from '../utils/logging.js'
-import { dataSubjects } from '../utils/utils.js'
-import { statusToTagClass } from '../filters/filters.js'
-import * as v from 'valibot'
-import { pagination } from '../utils/pagination.js'
-import config from '../../config/index.js'
-import {
-  fetchIf,
-  fetchMany,
-  fetchOne,
-  logPageError,
-  parallel,
-  renderTemplate,
-  FetchOptions,
-  FetchOneFallbackPolicy
-} from './middleware.js'
-import { getDatasetStats } from '../services/DatasetService.js'
-
-// get a list of available datasets
-const availableDatasets = Object.values(dataSubjects).flatMap((dataSubject) =>
-  dataSubject.dataSets
-    .filter((dataset) => dataset.available)
-    .map((dataset) => dataset.value)
-)
-
-const getGetStarted = renderTemplate({
-  templateParams (req) {
-    const { orgInfo: organisation, dataset } = req
-    return { organisation, dataset }
-  },
-  template: 'organisations/get-started.html',
-  handlerName: 'getStarted'
-})
-
-const fetchOrgInfo = fetchOne({
-  query: ({ params }) => {
-    return `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
-  },
-  result: 'orgInfo'
-})
-
-const fetchDatasetInfo = fetchOne({
-  query: ({ params }) => {
-    return `SELECT name, dataset FROM dataset WHERE dataset = '${params.dataset}'`
-  },
-  result: 'dataset'
-})
-
-const fetchDatasetStats = async (req, res, next) => {
-  req.stats = await getDatasetStats(req.params.dataset, req.params.lpa)
-
-  next()
-}
-
-const getDatasetOverview = renderTemplate(
-  {
-    templateParams (req) {
-      const { orgInfo: organisation, dataset, stats } = req
-      return { organisation, dataset, stats }
-    },
-    template: 'organisations/dataset-overview.html',
-    handlerName: 'datasetOverview'
-  }
-)
-
-const fetchOrgInfoWithStatGeo = fetchOne({
-  query: ({ params }) => {
-    return /* sql */ `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
-  },
-  result: 'orgInfo'
-})
-
-/**
- * Middleware.
- */
-const fetchResourceStatus = fetchOne({
-  query: ({ params }) => performanceDbApi.resourceStatusQuery(params.lpa, params.dataset),
-  result: 'resourceStatus'
-})
-
-/**
- * Middleware.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- * @returns {Promise<*>}
- */
-async function fetchLpaOverview (req, res, next) {
-  const { datasetsFilter } = config
-  try {
-    const overview = await performanceDbApi.getLpaOverview(req.params.lpa, {
-      datasetsFilter
-    })
-    req.lpaOverview = overview
-    next()
-  } catch (error) {
-    next(error)
-  }
-}
-
-/**
- * Middleware. Updates req with `resource`.
- */
-const fetchLatestResource = fetchOne({
-  query: ({ params }) => performanceDbApi.latestResourceQuery(params.lpa, params.dataset),
-  result: 'resource'
-})
-
-/**
- *
- * Middleware. Updates `req` with `issues`.
- *
- * Requires `resourceId` in request params or request (in that order).
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchIssues (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  if (!resourceId) {
-    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
-    throw Error('fetchIssues: missing resourceId')
-  }
-
-  try {
-    const issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
-    req.issues = issues
-    next()
-  } catch (error) {
-    next(error)
-  }
-}
-
-/**
- *
- * Middleware. Updates `req` with `issues`.
- *
- * Requires `issues` in request.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function reformatIssuesToBeByEntryNumber (req, res, next) {
-  const { issues } = req
-  const issuesByEntryNumber = issues.reduce((acc, current) => {
-    acc[current.entry_number] = acc[current.entry_number] || []
-    acc[current.entry_number].push(current)
-    return acc
-  }, {})
-  req.issuesByEntryNumber = issuesByEntryNumber
-  next()
-}
-
-/**
- *
- * Middleware. Updates `req` with `issueEntitiesCount` which is the count of entities that have issues.
- *
- * Requires `req.resource.resource`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchIssueEntitiesCount (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  console.assert(resourceId, 'missng resource id')
-  const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId)
-  req.issueEntitiesCount = parseInt(issueEntitiesCount)
-  next()
-}
-
-const fetchEntityCount = fetchOne({
-  query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource),
-  result: 'entityCount',
-  dataset: FetchOptions.fromParams,
-  fallbackPolicy: FetchOneFallbackPolicy.continue
-})
-
-/**
- *
- * Middleware. Updates `req` with `entryData`
- *
- * Requires `pageNumber`, `dataset` and
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- *
- */
-async function fetchEntry (req, res, next) {
-  const { dataset: datasetId, pageNumber } = req.params
-  const { issuesByEntryNumber } = req
-  const pageNum = pageNumber ? parseInt(pageNumber) : 1
-  req.pageNumber = pageNum
-
-  // look at issue Entries and get the index of that entry - 1
-
-  const entityNum =
-    Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
-
-  req.entryData = await performanceDbApi.getEntry(
-    req.resourceId,
-    entityNum,
-    datasetId
-  )
-  req.entryNumber = entityNum
-  next()
-}
-
-function prepareOverviewTemplateParams (req, res, next) {
-  const { lpaOverview, orgInfo: organisation } = req
-
-  const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => {
-    return {
-      slug: key,
-      ...value
-    }
-  })
-
-  // add in any of the missing key 8 datasets
-  const keys = Object.keys(lpaOverview.datasets)
-  availableDatasets.forEach((dataset) => {
-    if (!keys.includes(dataset)) {
-      datasets.push({
-        slug: dataset,
-        endpoint: null,
-        issue_count: 0,
-        status: 'Not submitted'
-      })
-    }
-  })
-
-  // re-sort the datasets to be in alphabetical order
-  datasets.sort((a, b) => a.slug.localeCompare(b.slug))
-
-  const totalDatasets = datasets.length
-  const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] =
-    datasets.reduce(
-      (accumulator, dataset) => {
-        if (dataset.endpoint) accumulator[0]++
-        if (dataset.status === 'Needs fixing') accumulator[1]++
-        if (dataset.status === 'Error') accumulator[2]++
-        return accumulator
-      },
-      [0, 0, 0]
-    )
-
-  req.templateParams = {
-    organisation,
-    datasets,
-    totalDatasets,
-    datasetsWithEndpoints,
-    datasetsWithIssues,
-    datasetsWithErrors
-  }
-
-  next()
-}
-
-const getOverview = renderTemplate({
-  templateParams (req) {
-    if (!req.templateParams) throw new Error('missing templateParams')
-    return req.templateParams
-  },
-  template: 'organisations/overview.html',
-  handlerName: 'getOverview'
-})
-
-/**
- * Middleware. Validates query params according to schema.
- * Short circuits with 400 error if validation fails
- *
- * `this` needs: `{ schema }`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-function validateQueryParams (req, res, next) {
-  try {
-    v.parse(this.schema || v.any(), req.params)
-    next()
-  } catch (error) {
-    res.status(400).render('errorPages/400', {})
-  }
-}
-
-export const IssueDetailsQueryParams = v.object({
-  lpa: v.string(),
-  dataset: v.string(),
-  issue_type: v.string(),
-  issue_field: v.string(),
-  pageNumber: v.optional(v.string()),
-  resourceId: v.optional(v.string())
-})
-
-const validateIssueDetailsQueryParams = validateQueryParams.bind({
-  schema: IssueDetailsQueryParams
-})
-
-/**
- *
- * @param {*} text
- * @param {*} html
- * @param {*} classes
- * @returns {{key: {text: string}, value: { html: string}, classes: string}}
- */
-const getIssueField = (text, html, classes) => {
-  return {
-    key: {
-      text
-    },
-    value: {
-      html
-    },
-    classes
-  }
-}
-
-/**
- *
- * @param {string} errorMessage
- * @param {{value: string}?} issue
- * @returns {string}
- */
-const issueErrorMessageHtml = (errorMessage, issue) =>
-  `<p class="govuk-error-message">${errorMessage}</p>${
-    issue ? issue.value ?? '' : ''
-  }`
-
-/**
- *
- * @param {*} issueType
- * @param {*} issuesByEntryNumber
- * @param {*} row
- * @returns {{key: {text: string}, value: { html: string}, classes: string}}
- */
-const processEntryRow = (issueType, issuesByEntryNumber, row) => {
-  const { entry_number: entryNumber } = row
-  console.assert(entryNumber, 'precessEntryRow(): entry_number not in row')
-  let hasError = false
-  let issueIndex
-  if (issuesByEntryNumber[entryNumber]) {
-    issueIndex = issuesByEntryNumber[entryNumber].findIndex(
-      (issue) => issue.field === row.field
-    )
-    hasError = issueIndex >= 0
-  }
-
-  let valueHtml = ''
-  let classes = ''
-  if (hasError) {
-    const message =
-      issuesByEntryNumber[entryNumber][issueIndex].message || issueType
-    valueHtml += issueErrorMessageHtml(message, null)
-    classes += 'dl-summary-card-list__row--error'
-  }
-  valueHtml += row.value
-
-  return getIssueField(row.field, valueHtml, classes)
-}
-
-/***
- * Middleware. Updates req with `templateParams`
- */
-function prepareIssueDetailsTemplateParams (req, res, next) {
-  const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount: entityCountRow } = req
-  const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
-
-  let errorHeading
-  let issueItems
-
-  const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/`
-
-  if (Object.keys(issuesByEntryNumber).length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
-    issueItems = Object.entries(issuesByEntryNumber).map(([entryNumber, issues], i) => {
-      const pageNum = i + 1
-      return {
-        html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
-        href: `${BaseSubpath}${pageNum}`
-      }
-    })
-  } else {
-    issueItems = [{
-      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true)
-    }]
-  }
-
-  const fields = entryData.map((row) => processEntryRow(issueType, issuesByEntryNumber, row))
-  const entityIssues = Object.values(issuesByEntryNumber)[pageNumber - 1] || []
-  for (const issue of entityIssues) {
-    if (!fields.find((field) => field.key.text === issue.field)) {
-      const errorMessage = issue.message || issueType
-      // TODO: pull the html out of here and into the template
-      const valueHtml = issueErrorMessageHtml(errorMessage, issue.value)
-      const classes = 'dl-summary-card-list__row--error'
-
-      fields.push(getIssueField(issue.field, valueHtml, classes))
-    }
-  }
-
-  const geometries = entryData
-    .filter((row) => row.field === 'geometry')
-    .map((row) => row.value)
-  const entry = {
-    title: `entry: ${entryNumber}`,
-    fields,
-    geometries
-  }
-
-  const paginationObj = {}
-  if (pageNumber > 1) {
-    paginationObj.previous = {
-      href: `${BaseSubpath}${pageNumber - 1}`
-    }
-  }
-
-  if (pageNumber < issueEntitiesCount) {
-    paginationObj.next = {
-      href: `${BaseSubpath}${pageNumber + 1}`
-    }
-  }
-
-  paginationObj.items = pagination(issueEntitiesCount, pageNumber).map(item => {
-    if (item === '...') {
-      return {
-        type: 'ellipsis',
-        ellipsis: true,
-        href: '#'
-      }
-    } else {
-      return {
-        type: 'item',
-        number: item,
-        href: `${BaseSubpath}${item}`,
-        current: pageNumber === parseInt(item)
-      }
-    }
-  })
-
-  // schema: OrgIssueDetails
-  req.templateParams = {
-    organisation: req.orgInfo,
-    dataset: req.dataset,
-    errorHeading,
-    issueItems,
-    entry,
-    issueType,
-    pagination: paginationObj,
-    issueEntitiesCount,
-    pageNumber
-  }
-
-  next()
-}
-
-/**
- * Middleware. Renders the issue details page with the list of issues, entry data,
- * and organisation and dataset details.
- */
-const getIssueDetails = renderTemplate({
-  templateParams: (req) => req.templateParams,
-  template: 'organisations/issueDetails.html',
-  handlerName: 'getIssueDetails'
-})
-
-const isResourceIdInParams = ({ params }) => !('resourceId' in params)
-
-const takeResourceIdFromParams = (req) => {
-  logger.debug('skipping resource fetch', { type: types.App, params: req.params })
-  req.resource = { resource: req.params.resourceId }
-}
-
-const getGetStartedMiddleware = [
-  fetchOrgInfo,
-  fetchDatasetInfo,
-  getGetStarted,
-  logPageError
-]
-
-const getDatasetOverviewMiddleware = [fetchOrgInfo, fetchDatasetInfo, fetchDatasetStats, getDatasetOverview, logPageError]
-
-const getOverviewMiddleware = [
-  fetchOrgInfo,
-  fetchLpaOverview,
-  prepareOverviewTemplateParams,
-  getOverview,
-  logPageError
-]
-
-const getIssueDetailsMiddleware = [
-  validateIssueDetailsQueryParams,
-  parallel([
-    fetchOrgInfo,
-    fetchDatasetInfo
-  ]),
-  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
-  fetchIssues,
-  reformatIssuesToBeByEntryNumber,
-  parallel([
-    fetchEntry,
-    fetchEntityCount,
-    fetchIssueEntitiesCount
-  ]),
-  prepareIssueDetailsTemplateParams,
-  getIssueDetails,
-  logPageError
-]
-
-const fetchOrganisations = fetchMany({
-  query: ({ req, params }) => 'select name, organisation from organisation',
-  result: 'organisations'
-})
-
-/**
- * Middleware. Updates req with `templateParams`.
- *
- * @param {{ organisations: {}[] }} req
- * @param {*} res
- * @param {*} next
- */
-const prepareGetOrganisationsTemplateParams = (req, res, next) => {
-  const sortedResults = req.organisations.sort((a, b) => {
-    return a.name.localeCompare(b.name)
-  })
-
-  const alphabetisedOrgs = sortedResults.reduce((acc, current) => {
-    const firstLetter = current.name.charAt(0).toUpperCase()
-    acc[firstLetter] = acc[firstLetter] || []
-    acc[firstLetter].push(current)
-    return acc
-  }, {})
-
-  req.templateParams = { alphabetisedOrgs }
-
-  next()
-}
-
-const getOrganisations = renderTemplate({
-  templateParams: (req) => req.templateParams,
-  template: 'organisations/find.html',
-  handlerName: 'getOrganisations'
-})
-
-/**
- * Was the resource accessed successfully via HTTP?
- *
- * @param {*} req
- * @returns {boolean}
- */
-const isResourceAccessible = (req) => req.resourceStatus.status === '200'
-const isResourceNotAccessible = (req) => !isResourceAccessible(req)
-
-const fetchLpaDatasetIssues = fetchMany({
-  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
-  result: 'issues'
-})
-
-const onlyIf = (condition, middlewareFn) => {
-  return async (req, res, next) => {
-    if (condition(req)) {
-      const result = middlewareFn(req, res, next)
-      if (result instanceof Promise) {
-        await result
-      }
-    } else {
-      next()
-    }
-  }
-}
-
-/**
- * Middleware. Updates req with `templateParams`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- * @returns { { templateParams: object }}
- */
-const prepareDatasetTaskListTemplateParams = (req, res, next) => {
-  const { issues, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req
-  const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
-  const { lpa, dataset: datasetId } = params
-  console.assert(req.resourceStatus.resource === req.resource.resource, 'mismatch between resourceStatus and resource data')
-  console.assert(typeof entityCount === 'number', 'entityCount should be a number')
-
-  const taskList = issues.map((issue) => {
-    return {
-      title: {
-        text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field })
-      },
-      href: `/organisations/${lpa}/${datasetId}/${issue.issue_type}/${issue.field}`,
-      status: getStatusTag(issue.status)
-    }
-  })
-
-  req.templateParams = {
-    taskList,
-    organisation,
-    dataset
-  }
-
-  next()
-}
-
-/**
- * Middleware. Updates req with `templateParams`
- *
- * @param {*} req
- * @param {*} res
- * @param {} next
- * @returns {{ templateParams: object }}
- */
-const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
-  const { orgInfo: organisation, dataset, resourceStatus: resource } = req
-
-  const daysSince200 = resource.days_since_200
-  const today = new Date()
-  const last200Date = new Date(
-    today.getTime() - daysSince200 * 24 * 60 * 60 * 1000
-  )
-  const last200Datetime = last200Date.toISOString().slice(0, 19) + 'Z'
-
-  req.templateParams = {
-    organisation,
-    dataset,
-    errorData: {
-      endpoint_url: resource.endpoint_url,
-      http_status: resource.status,
-      latest_log_entry_date: resource.latest_log_entry_date,
-      latest_200_date: last200Datetime
-    }
-  }
-
-  next()
-}
-
-const getDatasetTaskList = renderTemplate({
-  templateParams: (req) => req.templateParams,
-  template: 'organisations/datasetTaskList.html',
-  handlerName: 'getDatasetTaskList'
-})
-
-const getDatasetTaskListError = renderTemplate({
-  templateParams: (req) => req.templateParams,
-  template: 'organisations/http-error.html',
-  handlerName: 'getDatasetTaskListError'
-})
-
-const getOrganisationsMiddleware = [
-  fetchOrganisations,
-  prepareGetOrganisationsTemplateParams,
-  getOrganisations,
-  logPageError
-]
-
-const getDatasetTaskListMiddleware = [
-  fetchResourceStatus,
-  parallel([
-    fetchOrgInfoWithStatGeo,
-    fetchDatasetInfo]),
-  fetchIf(isResourceAccessible, fetchLatestResource),
-  parallel([
-    fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
-    fetchIf(isResourceAccessible, fetchEntityCount)
-  ]),
-  onlyIf(isResourceAccessible, prepareDatasetTaskListTemplateParams),
-  onlyIf(isResourceAccessible, getDatasetTaskList),
-
-  onlyIf(isResourceNotAccessible, prepareDatasetTaskListErrorTemplateParams),
-  onlyIf(isResourceNotAccessible, getDatasetTaskListError),
-  logPageError
-]
-
-/**
- * Returns a status tag object with a text label and a CSS class based on the status.
- *
- * @param {string} status - The status to generate a tag for (e.g. "Error", "Needs fixing", etc.)
- * @returns {object} - An object with a `tag` property containing the text label and CSS class.
- */
-function getStatusTag (status) {
-  return {
-    tag: {
-      text: status,
-      classes: statusToTagClass(status)
-    }
-  }
-}
+import getDatasetTaskListMiddleware from './datasetTasklist.middleware.js'
+import getDatasetOverviewMiddleware from './datasetOverview.middleware.js'
+import getIssueDetailsMiddleware from './issueDetails.middleware.js'
+import getOrganisationsMiddleware from './organisations.middleware.js'
+import getStartedMiddleware from './started.middleware.js'
+import getOverviewMiddleware from './overview.middleware.js'
 
 const organisationsController = {
-  prepareDatasetTaskListErrorTemplateParams,
-  prepareDatasetTaskListTemplateParams,
-  getDatasetTaskListError,
-  getDatasetTaskList,
+  getOrganisationsMiddleware,
   getDatasetTaskListMiddleware,
-
-  /**
-   * Middleware chain for GET requests for the "Get Started" page.
-   */
-  getGetStartedMiddleware,
-  getGetStarted,
-
   getDatasetOverviewMiddleware,
-  getDatasetOverview,
-
-  getOverviewMiddleware,
-  getOverview,
-  prepareOverviewTemplateParams,
-
   getIssueDetailsMiddleware,
-  getIssueDetails,
-  prepareIssueDetailsTemplateParams,
-  IssueDetailsQueryParams,
-
-  getOrganisationsMiddleware,
-  prepareGetOrganisationsTemplateParams,
-  fetchOrganisations,
-  getOrganisations
+  getStartedMiddleware,
+  getOverviewMiddleware
 }
 
 export default organisationsController

From b0524f50c695d29d39650af714cd6a495fe6cce7 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 12:23:04 +0100
Subject: [PATCH 31/65] fix naming

---
 src/controllers/OrganisationsController.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 130debf2..78816dba 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -1,16 +1,16 @@
-import getDatasetTaskListMiddleware from './datasetTasklist.middleware.js'
-import getDatasetOverviewMiddleware from './datasetOverview.middleware.js'
-import getIssueDetailsMiddleware from './issueDetails.middleware.js'
-import getOrganisationsMiddleware from './organisations.middleware.js'
-import getStartedMiddleware from './started.middleware.js'
-import getOverviewMiddleware from './overview.middleware.js'
+import getDatasetTaskListMiddleware from '../middleware/datasetTaskList.middleware.js'
+import getDatasetOverviewMiddleware from '../middleware/datasetOverview.middleware.js'
+import getIssueDetailsMiddleware from '../middleware/issueDetails.middleware.js'
+import getOrganisationsMiddleware from '../middleware/organisations.middleware.js'
+import getGetStartedMiddleware from '../middleware/getStarted.middleware.js'
+import getOverviewMiddleware from '../middleware/overview.middleware.js'
 
 const organisationsController = {
   getOrganisationsMiddleware,
   getDatasetTaskListMiddleware,
   getDatasetOverviewMiddleware,
   getIssueDetailsMiddleware,
-  getStartedMiddleware,
+  getGetStartedMiddleware,
   getOverviewMiddleware
 }
 

From cdef6b1fa47059fc425c931bc056816ef5c0e4b0 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Mon, 23 Sep 2024 12:23:51 +0100
Subject: [PATCH 32/65] fix links and delete old middleware file

---
 src/controllers/middleware.js                | 284 -------------------
 src/middleware/common.middleware.js          |  25 ++
 src/middleware/datasetOverview.middleware.js |   4 +-
 src/middleware/datasetTaskList.middleware.js |   4 +-
 src/middleware/getStarted.middleware.js      |   4 +-
 src/middleware/issueDetails.middleware.js    |  10 +-
 src/middleware/issuesTable.middleware.js     |  12 +
 src/middleware/organisations.middleware.js   |   4 +-
 src/middleware/overview.middleware.js        |   6 +-
 9 files changed, 53 insertions(+), 300 deletions(-)
 delete mode 100644 src/controllers/middleware.js
 create mode 100644 src/middleware/issuesTable.middleware.js

diff --git a/src/controllers/middleware.js b/src/controllers/middleware.js
deleted file mode 100644
index c9c04029..00000000
--- a/src/controllers/middleware.js
+++ /dev/null
@@ -1,284 +0,0 @@
-import logger from '../utils/logger.js'
-import { types } from '../utils/logging.js'
-import { templateSchema } from '../routes/schemas.js'
-import { render } from '../utils/custom-renderer.js'
-import datasette from '../services/datasette.js'
-import * as v from 'valibot'
-
-export const FetchOptions = {
-  /**
-   * Use 'dataset' from requets params.
-   */
-  fromParams: Symbol('from-params')
-}
-
-const datasetOverride = (val, req) => {
-  if (!val) {
-    return 'digital-land'
-  }
-  if (val === FetchOptions.fromParams) {
-    console.assert(req.params.dataset, 'no "dataset" in request params')
-    return req.params.dataset
-  } else {
-    return val(req)
-  }
-}
-
-const fetchOneFallbackPolicy = (req, res, next) => {
-  res.status(404).render('errorPages/404', {})
-}
-
-/**
- * Collection of fallback policies for the {@link fetchOneFn} middleware.
- * The policy is enacted when zero records is returned from the data source.
- */
-export const FetchOneFallbackPolicy = {
-  /**
-   * Renders a 404 response.
-   */
-  'not-found-error': fetchOneFallbackPolicy,
-
-  /**
-   * Proceeds by calling `next()`.
-   */
-  continue: (_req, _res, next) => next()
-}
-
-/**
- * Middleware. Attempts to fetch data from datasette and short-circuits with 404 when
- * data for given query does not exist. Meant to be used to fetch singular records.
- *
- * `this` needs `{ query({ req, params }) => any, result: string, dataset?: FetchParams | (req) => string, fallbackPolicy: (req, res, next) => void }`
- *
- * where the `result` is the key under which result of the query will be stored in `req`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchOneFn (req, res, next) {
-  logger.debug({ type: types.DataFetch, message: 'fetchOne', resultKey: this.result })
-  try {
-    const query = this.query({ req, params: req.params })
-    const result = await datasette.runQuery(query, datasetOverride(this.dataset, req))
-    const fallbackPolicy = this.fallbackPolicy ?? FetchOneFallbackPolicy['not-found-error']
-    if (result.formattedData.length === 0) {
-      // we can make the 404 more informative by informing the use what exactly was "not found"
-      fallbackPolicy(req, res, next)
-    } else {
-      req[this.result] = result.formattedData[0]
-      next()
-    }
-  } catch (error) {
-    logger.debug('fetchOne: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
-    req.handlerName = `fetching '${this.result}'`
-    next(error)
-  }
-}
-
-/**
- * Middleware. Attempts to fetch a collection of data from datasette.
- *
- * `this` needs `{ query( {req, params } ) => any, result: string, dataset?: FetchParams | (req) => string }`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-export async function fetchManyFn (req, res, next) {
-  try {
-    const query = this.query({ req, params: req.params })
-    const result = await datasette.runQuery(query, datasetOverride(this.dataset, req))
-    req[this.result] = result.formattedData
-    logger.debug({ type: types.DataFetch, message: 'fetchMany', resultKey: this.result, resultCount: result.formattedData.length })
-    next()
-  } catch (error) {
-    logger.debug('fetchMany: failed', { type: types.DataFetch, errorMessage: error.message, endpoint: req.originalUrl, resultKey: this.result })
-    req.handlerName = `fetching '${this.result}'`
-    next(error)
-  }
-}
-
-/**
- * Middleware. Does a conditional fetch. Optionally invokes `else` if condition is false.
- *
- * `this` needs: `{ fetchFn, condition: (req) => boolean, else?: (req) => void }`
- *
- * `fetchFn` should be a middleware fn. Can be async.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- */
-async function fetchIfFn (req, res, next) {
-  if (this.condition(req)) {
-    // `next` will be called in our fetchFn middleware
-    const result = this.fetchFn(req, res, next)
-    if (result instanceof Promise) {
-      await result
-    }
-  } else {
-    if (this.else) {
-      this.else(req)
-    }
-    next()
-  }
-}
-
-/**
- *
- * Returns a middleware that will fetch data if condition is satisfied,
- * invoke `elseFn` otherwise.
- *
- * @param {(req) => boolean} condition predicate fn
- * @param {*} fetchFn fetch middleware
- * @param {(req) => void} elseFn
- * @returns
- */
-export const fetchIf = (condition, fetchFn, elseFn) => {
-  return fetchIfFn.bind({
-    condition, fetchFn, else: elseFn
-  })
-}
-
-/**
- * Middleware. Set `req.handlerName` to a string that will identify
- * the function that threw the error.
- *
- * @param {Error} err
- * @param {{handlerName: string}} req
- * @param {*} res
- * @param {*} next
- */
-export const logPageError = (err, req, res, next) => {
-  console.assert(req.handlerName, 'handlerName missing ')
-  logger.warn({
-    message: `OrganisationsController.${req.handlerName}(): ${err.message}`,
-    endpoint: req.originalUrl,
-    errorStack: err.stack,
-    errorMessage: err.message,
-    type: types.App
-  })
-  next(err)
-}
-
-/**
- * Looks up schema for name in @{link templateSchema} (defaults to any()), validates and renders the template.
- *
- * @param {*} res
- * @param {string} name
- * @param {*} params
- * @returns {string}
- */
-export function validateAndRender (res, name, params) {
-  const schema = templateSchema.get(name) ?? v.any()
-  logger.info(
-    `rendering '${name}' with schema=<${schema ? 'defined' : 'any'}>`,
-    { type: types.App }
-  )
-  return render(res, name, schema, params)
-}
-
-/**
- * Middleware. Validates and renders the template.
- *
- * `this` needs: `{ templateParams(req), template,  handlerName }`
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- * @returns
- */
-export function renderTemplateFn (req, res, next) {
-  const templateParams = this.templateParams(req)
-  try {
-    validateAndRender(res, this.template, templateParams)
-    logger.info(`rendered ${this.template}`, { type: types.App })
-  } catch (err) {
-    req.handlerName = this.handlerName
-    next(err)
-  }
-}
-
-/**
- * Returns a middleware that executes the given sub-middlewares in parallel
- * and waits for all of them to complete.
- *
- * @param {*} req
- * @param {*} res
- * @param {*} next
- * @returns {(req, res, next) => void} middleware
- */
-async function parallelFn (req, res, next) {
-  const fns = this.middlewares
-  const nextParams = []
-  const nextFn = (val) => {
-    if (val) nextParams.push(val)
-  }
-  // We need to take care of explicit `next(value)`, so we hijack the 'next' callback.
-  // We also need to hadle any rejected promises in results
-  const results = await Promise.allSettled(fns.map(fn => fn(req, res, nextFn)))
-  /* eslint-disable no-unreachable-loop */
-  for (const param of nextParams) {
-    if (param instanceof Error) {
-      logger.debug('parallel: captured a "next" error', { type: types.App, errorMessage: param.message })
-    }
-    next(param)
-    return
-  }
-  for (const result of results) {
-    if (result.status === 'rejected') {
-      logger.debug('parallel: got a rejected promise', { type: types.App })
-      next(result.reason)
-      return
-    }
-  }
-
-  next()
-}
-
-/**
- * Returns a middleware that invokes the passed middlewares in parallel.
- *
- * Usage: when all sub-middlewre can be fetched independently, but we can't accept a partial success
- * (e.g. we require all middlewares to succeed).
- *
- * @param {((req,res,next) => void)[]} middlewares array of middleware functions
- * @returns {(req, res, next) => Promise<void>}
- */
-export function parallel (middlewares) {
-  return parallelFn.bind({ middlewares })
-}
-
-/**
- * Fetches a single entity and stores it in `req` under key specified by `result` entry.
- *
- * Use `fallbackPolicy` to handle zero record responses differently than the default 404 response.
- * See {@link FetchOneFallbackPolicy}
- *
- * @param {{ query: ({ req, params}) => object, result: string, dataset?: FetchParams | (req) => string, fallbackPolicy?: (req, res, next) => void}} context
- * @returns
- */
-export function fetchOne (context) {
-  return fetchOneFn.bind(context)
-}
-
-/**
- * Fetches a collection of records and stores them in `req` under key specified by `result` entry.
- *
- * @param {{query: ({req, params}) => object, result: string, dataset?: FetchParams | (req) => string}} context
- */
-export function fetchMany (context) {
-  return fetchManyFn.bind(context)
-}
-
-/**
- * Validates and renders the template.
- *
- *
- * @param {{templateParams: (req) => object, template: string,  handlerName: string}} context
- * @returns {(req, res, next) => void} middleware
- */
-export function renderTemplate (context) {
-  return renderTemplateFn.bind(context)
-}
diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index 85b786c1..af4a69d4 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -2,6 +2,7 @@ import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { fetchOne, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js'
+import * as v from 'valibot'
 
 /**
  * Middleware. Set `req.handlerName` to a string that will identify
@@ -50,6 +51,11 @@ export const fetchLatestResource = fetchOne({
   result: 'resource'
 })
 
+export const takeResourceIdFromParams = (req) => {
+  logger.debug('skipping resource fetch', { type: types.App, params: req.params })
+  req.resource = { resource: req.params.resourceId }
+}
+
 export const fetchEntityCount = fetchOne({
   query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource),
   result: 'entityCount',
@@ -63,3 +69,22 @@ export const fetchOrgInfo = fetchOne({
   },
   result: 'orgInfo'
 })
+
+/**
+ * Middleware. Validates query params according to schema.
+ * Short circuits with 400 error if validation fails
+ *
+ * `this` needs: `{ schema }`
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+export function validateQueryParams (req, res, next) {
+  try {
+    v.parse(this.schema || v.any(), req.params)
+    next()
+  } catch (error) {
+    res.status(400).render('errorPages/400', {})
+  }
+}
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index 29b6e864..e97c45c4 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,5 +1,5 @@
-import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware'
-import { renderTemplate } from './middleware.builders'
+import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware.js'
+import { renderTemplate } from './middleware.builders.js'
 import { getDatasetStats } from '../services/DatasetService.js'
 
 const fetchDatasetStats = async (req, res, next) => {
diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index ed32979b..30dc514a 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -1,5 +1,5 @@
-import { fetchDatasetInfo, isResourceAccessible, isResourceNotAccessible, fetchLatestResource, fetchEntityCount, logPageError } from './common.middleware'
-import { parallel, fetchOne, fetchIf, fetchMany, onlyIf, renderTemplate } from './middleware.builders'
+import { fetchDatasetInfo, isResourceAccessible, isResourceNotAccessible, fetchLatestResource, fetchEntityCount, logPageError } from './common.middleware.js'
+import { parallel, fetchOne, fetchIf, fetchMany, onlyIf, renderTemplate } from './middleware.builders.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { statusToTagClass } from '../filters/filters.js'
 
diff --git a/src/middleware/getStarted.middleware.js b/src/middleware/getStarted.middleware.js
index 55bbeeb7..9bfc123a 100644
--- a/src/middleware/getStarted.middleware.js
+++ b/src/middleware/getStarted.middleware.js
@@ -1,5 +1,5 @@
-import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware'
-import { renderTemplate } from './middleware.builders'
+import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware.js'
+import { renderTemplate } from './middleware.builders.js'
 
 const getGetStarted = renderTemplate({
   templateParams (req) {
diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index ca2af3b3..c9c61e4c 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -1,9 +1,9 @@
-import performanceDbApi from '../services/performanceDbApi'
-import logger from '../utils/logger'
+import performanceDbApi from '../services/performanceDbApi.js'
+import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware'
-import { fetchIf, parallel, renderTemplate } from './middleware.builders'
-import * as v from 'vali'
+import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
+import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 
 const IssueDetailsQueryParams = v.object({
diff --git a/src/middleware/issuesTable.middleware.js b/src/middleware/issuesTable.middleware.js
new file mode 100644
index 00000000..a3a39b34
--- /dev/null
+++ b/src/middleware/issuesTable.middleware.js
@@ -0,0 +1,12 @@
+// export default [
+//   validateIssueTableQueryParams,
+//   parallel([
+//     fetchOrgInfo,
+//     fetchDatasetInfo
+//   ]),
+//   fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+//   fetchEntitiesWithIssues,
+//   prepareIssueTableParams,
+//   getIssuesTable,
+//   logPageError
+// ]
diff --git a/src/middleware/organisations.middleware.js b/src/middleware/organisations.middleware.js
index 9a98df70..b9b67da5 100644
--- a/src/middleware/organisations.middleware.js
+++ b/src/middleware/organisations.middleware.js
@@ -1,5 +1,5 @@
-import { logPageError } from './common.middleware'
-import { fetchMany, renderTemplate } from './middleware.builders'
+import { logPageError } from './common.middleware.js'
+import { fetchMany, renderTemplate } from './middleware.builders.js'
 
 const fetchOrganisations = fetchMany({
   query: ({ req, params }) => 'select name, organisation from organisation',
diff --git a/src/middleware/overview.middleware.js b/src/middleware/overview.middleware.js
index 84c786ef..af951eb6 100644
--- a/src/middleware/overview.middleware.js
+++ b/src/middleware/overview.middleware.js
@@ -1,6 +1,6 @@
-import performanceDbApi from '../services/performanceDbApi'
-import { fetchOrgInfo, logPageError } from './common.middleware'
-import { renderTemplate } from './middleware.builders'
+import performanceDbApi from '../services/performanceDbApi.js'
+import { fetchOrgInfo, logPageError } from './common.middleware.js'
+import { renderTemplate } from './middleware.builders.js'
 import { dataSubjects } from '../utils/utils.js'
 import config from '../../config/index.js'
 

From f0f7730a22066f0256ee4dffed5c4a1f976f8a02 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Tue, 24 Sep 2024 14:44:40 +0100
Subject: [PATCH 33/65] Bump `govuk-frontend` to `5.6.0`

---
 package-lock.json | 6 ++++--
 package.json      | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index ba9d8aa6..df84ad42 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -24,7 +24,7 @@
         "dotenv": "^16.4.5",
         "express": "^4.19.2",
         "express-session": "^1.18.0",
-        "govuk-frontend": "^5.4.0",
+        "govuk-frontend": "^5.6.0",
         "hmpo-config": "^3.0.0",
         "hmpo-form-wizard": "^13.0.0",
         "hmpo-i18n": "^6.0.1",
@@ -7385,7 +7385,9 @@
       }
     },
     "node_modules/govuk-frontend": {
-      "version": "5.4.1",
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/govuk-frontend/-/govuk-frontend-5.6.0.tgz",
+      "integrity": "sha512-yNA4bL7i7mNrg36wPNZ3RctHo9mjl82Phs8MWs1lwovxJuQ4ogEo/XWn2uB1HxkXNqgMlW4wnd0iiKgRMfxYfw==",
       "license": "MIT",
       "engines": {
         "node": ">= 4.2.0"
diff --git a/package.json b/package.json
index ed0bf9a8..bb2d3adb 100644
--- a/package.json
+++ b/package.json
@@ -71,7 +71,7 @@
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "express-session": "^1.18.0",
-    "govuk-frontend": "^5.4.0",
+    "govuk-frontend": "^5.6.0",
     "hmpo-config": "^3.0.0",
     "hmpo-form-wizard": "^13.0.0",
     "hmpo-i18n": "^6.0.1",

From b92f0f4b32a59e98baa84e1020fd67922d8caf35 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Tue, 24 Sep 2024 14:49:43 +0100
Subject: [PATCH 34/65] Add dataset navigation component

---
 .../scss/components/_dataset-navigation.scss  |  4 +++
 src/assets/scss/index.scss                    |  3 +-
 src/views/components/dataset-navigation.html  | 29 +++++++++++++++++++
 3 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 src/assets/scss/components/_dataset-navigation.scss
 create mode 100644 src/views/components/dataset-navigation.html

diff --git a/src/assets/scss/components/_dataset-navigation.scss b/src/assets/scss/components/_dataset-navigation.scss
new file mode 100644
index 00000000..6cf45782
--- /dev/null
+++ b/src/assets/scss/components/_dataset-navigation.scss
@@ -0,0 +1,4 @@
+.app-c-dataset-navigation {
+  background-color: #fff;
+  border-top: 1px solid #bfc1c3;
+}
diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss
index 3e1b6a16..1a344e23 100644
--- a/src/assets/scss/index.scss
+++ b/src/assets/scss/index.scss
@@ -4,6 +4,7 @@ $govuk-global-styles: true;
 @import "node_modules/@x-govuk/govuk-prototype-components/x-govuk/all";
 @import "src/assets/scss/_scrollable-container.scss";
 @import "./step-by-step-nav.scss";
+@import "./components/dataset-navigation";
 
 
 .app-inset-text---error {
@@ -25,7 +26,7 @@ $govuk-global-styles: true;
 
 
 }
-  
+
 .app-inset-text---warning {
   border-left: 5px solid govuk-colour('blue');
   padding: govuk-spacing(1) govuk-spacing(2);
diff --git a/src/views/components/dataset-navigation.html b/src/views/components/dataset-navigation.html
new file mode 100644
index 00000000..039834a6
--- /dev/null
+++ b/src/views/components/dataset-navigation.html
@@ -0,0 +1,29 @@
+{% from "govuk/components/service-navigation/macro.njk" import govukServiceNavigation %}
+
+{% macro datasetNavigation(params) %}
+  {% set organisation = params.organisation %}
+  {% set dataset = params.dataset %}
+  {% set badge %}
+    {% if params.issue_count > 0 %}
+      <strong class="govuk-tag govuk-tag--red">
+        {{ params.issue_count }}
+      </strong>
+    {% endif %}
+  {% endset %}
+
+  {{ govukServiceNavigation({
+    classes: "app-c-dataset-navigation govuk-!-margin-bottom-7",
+    navigation: [
+      {
+        href: "/organisations/" + organisation.organisation + "/" + dataset.dataset + "/overview",
+        text: "Dataset overview",
+        active: params.active === "dataset-overview"
+      },
+      {
+        href: "/organisations/" + organisation.organisation + "/" + dataset.dataset,
+        html: "Task list " + badge,
+        active: params.active === "task-list"
+      }
+    ]
+  }) }}
+{% endmacro %}
\ No newline at end of file

From 1337df10bbe034895cd6a2d01bca2b2c5d083325 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 02:19:03 +0100
Subject: [PATCH 35/65] Add dataset navigation to overview and task list pages

---
 src/controllers/OrganisationsController.js    | 44 ++++++++++++-------
 src/routes/schemas.js                         |  3 +-
 src/views/organisations/dataset-overview.html | 11 ++++-
 src/views/organisations/datasetTaskList.html  | 16 +++++--
 4 files changed, 51 insertions(+), 23 deletions(-)

diff --git a/src/controllers/OrganisationsController.js b/src/controllers/OrganisationsController.js
index 019011a4..42495391 100644
--- a/src/controllers/OrganisationsController.js
+++ b/src/controllers/OrganisationsController.js
@@ -57,8 +57,8 @@ const fetchDatasetStats = async (req, res, next) => {
 const getDatasetOverview = renderTemplate(
   {
     templateParams (req) {
-      const { orgInfo: organisation, dataset, stats } = req
-      return { organisation, dataset, stats }
+      const { orgInfo: organisation, dataset, stats, issues } = req
+      return { organisation, dataset, stats, issueCount: issues.length }
     },
     template: 'organisations/dataset-overview.html',
     handlerName: 'datasetOverview'
@@ -473,8 +473,21 @@ const getIssueDetails = renderTemplate({
   handlerName: 'getIssueDetails'
 })
 
+/**
+ * Was the resource accessed successfully via HTTP?
+ *
+ * @param {*} req
+ * @returns {boolean}
+ */
+const isResourceAccessible = (req) => req.resourceStatus.status === '200'
+const isResourceNotAccessible = (req) => !isResourceAccessible(req)
 const isResourceIdInParams = ({ params }) => !('resourceId' in params)
 
+const fetchLpaDatasetIssues = fetchMany({
+  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
+  result: 'issues'
+})
+
 const takeResourceIdFromParams = (req) => {
   logger.debug('skipping resource fetch', { type: types.App, params: req.params })
   req.resource = { resource: req.params.resourceId }
@@ -487,7 +500,18 @@ const getGetStartedMiddleware = [
   logPageError
 ]
 
-const getDatasetOverviewMiddleware = [fetchOrgInfo, fetchDatasetInfo, fetchDatasetStats, getDatasetOverview, logPageError]
+const getDatasetOverviewMiddleware = [
+  parallel([
+    fetchOrgInfo,
+    fetchDatasetInfo
+  ]),
+  fetchResourceStatus,
+  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
+  fetchDatasetStats,
+  getDatasetOverview,
+  logPageError
+]
 
 const getOverviewMiddleware = [
   fetchOrgInfo,
@@ -551,20 +575,6 @@ const getOrganisations = renderTemplate({
   handlerName: 'getOrganisations'
 })
 
-/**
- * Was the resource accessed successfully via HTTP?
- *
- * @param {*} req
- * @returns {boolean}
- */
-const isResourceAccessible = (req) => req.resourceStatus.status === '200'
-const isResourceNotAccessible = (req) => !isResourceAccessible(req)
-
-const fetchLpaDatasetIssues = fetchMany({
-  query: ({ params, req }) => performanceDbApi.datasetIssuesQuery(req.resourceStatus.resource, params.dataset),
-  result: 'issues'
-})
-
 const onlyIf = (condition, middlewareFn) => {
   return async (req, res, next) => {
     if (condition(req)) {
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 9f763f67..feb1308f 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -76,7 +76,8 @@ export const OrgDatasetOverview = v.strictObject({
   stats: v.strictObject({
     numberOfRecords: v.integer(),
     numberOfFieldsSupplied: v.integer()
-  })
+  }),
+  issueCount: v.integer()
 })
 
 export const OrgDatasetTaskList = v.strictObject({
diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index a2b83750..b38f9cf2 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -1,5 +1,6 @@
 {% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %}
 {% from "govuk/components/summary-list/macro.njk" import govukSummaryList %}
+{% from "../components/dataset-navigation.html" import datasetNavigation %}
 {% extends "layouts/main.html" %}
 
 {% set showMap = [
@@ -51,16 +52,22 @@ <h1 class="govuk-heading-xl">{{ dataset.name }}</h1>
     </div>
 </div>
 
-{% if showMap %}
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-full">
+    {{ datasetNavigation({
+      active: "dataset-overview",
+      dataset: dataset,
+      organisation: organisation,
+      issue_count: issueCount
+    }) }}
+    {% if showMap %}
       <section>
         <h2 class="govuk-heading-m govuk-visually-hidden">Map of dataset</h2>
         <div id="map" class="app-map" role="region" aria-label="Map illustrating {{ dataset.name }} geometries. Use your keyboard to interact with the map. For screen reader users, use the arrow keys to navigate the map and the plus and minus keys to zoom in and out."></div>
       </section>
+    {% endif %}
   </div>
 </div>
-{% endif %}
 
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
diff --git a/src/views/organisations/datasetTaskList.html b/src/views/organisations/datasetTaskList.html
index 34b11192..b61dbc1e 100644
--- a/src/views/organisations/datasetTaskList.html
+++ b/src/views/organisations/datasetTaskList.html
@@ -1,11 +1,10 @@
 {% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %}
+{% from 'govuk/components/task-list/macro.njk' import govukTaskList %}
+{% from "../components/dataset-navigation.html" import datasetNavigation %}
 
 {% extends "layouts/main.html" %}
 
-{% from 'govuk/components/task-list/macro.njk' import govukTaskList %}
-
 {% set pageName %}{{organisation.name}} - {{dataset.name}} - Task list{% endset %}
-
 {% set serviceType = 'Submit' %}
 
 {% block beforeContent %}
@@ -40,6 +39,17 @@
   {% include "includes/_dataset-page-header.html" %}
 </div>
 
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-full">
+    {{ datasetNavigation({
+      active: "task-list",
+      dataset: dataset,
+      organisation: organisation,
+      issue_count: taskList.length
+    }) }}
+  </div>
+</div>
+
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
     <h2 class="govuk-heading-l">

From 4f1a87507b963e1484b986b91698c3c3bd827451 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 02:19:32 +0100
Subject: [PATCH 36/65] Make map border red to highlight boundaries

Better contrast
---
 src/assets/js/map.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/assets/js/map.js b/src/assets/js/map.js
index e3363cc0..7386452d 100644
--- a/src/assets/js/map.js
+++ b/src/assets/js/map.js
@@ -3,7 +3,8 @@ import maplibregl from 'maplibre-gl'
 
 const fillColor = '#008'
 const lineColor = '#000000'
-const boundaryLineColor = '#000000'
+const boundaryLineColor = '#f00'
+const boundaryLineOpacity = 1
 const opacity = 0.4
 
 /**
@@ -161,7 +162,7 @@ class Map {
       paint: {
         'line-color': boundaryLineColor,
         'line-width': 2,
-        'line-opacity': opacity
+        'line-opacity': boundaryLineOpacity
       }
     }, this.firstMapLayerId)
   }

From 8fb9e146216d458cfa465b3c17f845063fca65f9 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 02:37:43 +0100
Subject: [PATCH 37/65] Add tests to ensure navigation is showing

---
 test/unit/datasetTaskListPage.test.js                  | 9 +++++++++
 test/unit/views/organisations/dataset-overview.test.js | 9 +++++++++
 2 files changed, 18 insertions(+)

diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js
index 99db0894..2fa67da1 100644
--- a/test/unit/datasetTaskListPage.test.js
+++ b/test/unit/datasetTaskListPage.test.js
@@ -83,6 +83,15 @@ describe('Dataset Task List Page', () => {
     expect(document.querySelector('h1').textContent).toContain(params.dataset.name)
   })
 
+  it('Renders the dataset navigation links correctly', () => {
+    const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+
+    expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
+    expect(links.length).toEqual(2)
+    expect(links[0].textContent).toContain('Dataset overview')
+    expect(links[1].textContent).toContain('Task list')
+  })
+
   const taskListItems = document.querySelectorAll('.govuk-task-list__item')
   it('Task list items are rendered correctly', () => {
     expect(taskListItems.length).toEqual(params.taskList.length)
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index 867d88dd..ede7a12a 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -35,6 +35,15 @@ describe('Dataset Overview Page', () => {
     expect(document.querySelector('h1').textContent).toContain('World heritage site buffer zone')
   })
 
+  it('Renders the dataset navigation links correctly', () => {
+    const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+
+    expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
+    expect(links.length).toEqual(2)
+    expect(links[0].textContent).toContain('Dataset overview')
+    expect(links[1].textContent).toContain('Task list')
+  })
+
   it('Renders dataset details correctly', () => {
     expect(document.querySelector('h2.govuk-heading-m').textContent).toContain('Dataset details')
     const summaryListValues = document.querySelectorAll('dd.govuk-summary-list__value')

From 342bc081d20d45feed7988b533050ca8c54d783c Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 02:55:41 +0100
Subject: [PATCH 38/65] Add tests for dataset navigation component

---
 test/unit/datasetTaskListPage.test.js         |  6 +-
 .../components/dataset-navigation.test.js     | 93 +++++++++++++++++++
 .../organisations/dataset-overview.test.js    |  4 +-
 3 files changed, 99 insertions(+), 4 deletions(-)
 create mode 100644 test/unit/views/components/dataset-navigation.test.js

diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js
index 2fa67da1..99a0b3b4 100644
--- a/test/unit/datasetTaskListPage.test.js
+++ b/test/unit/datasetTaskListPage.test.js
@@ -85,11 +85,13 @@ describe('Dataset Task List Page', () => {
 
   it('Renders the dataset navigation links correctly', () => {
     const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+    const activeLink = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
+    const issueCount = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .govuk-tag')
 
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(links.length).toEqual(2)
-    expect(links[0].textContent).toContain('Dataset overview')
-    expect(links[1].textContent).toContain('Task list')
+    expect(activeLink.textContent).toContain('Task list')
+    expect(issueCount.textContent).toContain('3')
   })
 
   const taskListItems = document.querySelectorAll('.govuk-task-list__item')
diff --git a/test/unit/views/components/dataset-navigation.test.js b/test/unit/views/components/dataset-navigation.test.js
new file mode 100644
index 00000000..383d6f16
--- /dev/null
+++ b/test/unit/views/components/dataset-navigation.test.js
@@ -0,0 +1,93 @@
+import { describe, it, expect } from 'vitest'
+import jsdom from 'jsdom'
+import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
+
+describe('Dataset Navigation component', () => {
+  const params = {
+    organisation: {
+      name: 'Mock org',
+      organisation: 'mock-org'
+    },
+    dataset: {
+      dataset: 'world-heritage-site-buffer-zone',
+      name: 'World heritage site buffer zone'
+    },
+    issueCount: 0
+  }
+
+  const htmlString = `
+    {% from "components/dataset-navigation.html" import datasetNavigation %}
+    {{ datasetNavigation({
+      active: "dataset-overview",
+      dataset: dataset,
+      organisation: organisation,
+      issue_count: issueCount
+    }) }}
+  `
+
+  const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
+  const html = nunjucks.renderString(htmlString, params)
+  const dom = new jsdom.JSDOM(html)
+  const document = dom.window.document
+
+  it('Renders the dataset navigation links correctly', () => {
+    const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+    const activeLink = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
+    const issueCount = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .govuk-tag')
+
+    expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
+    expect(activeLink.textContent).toContain('Dataset overview')
+    expect(links.length).toEqual(2)
+    expect(issueCount).toBeNull()
+  })
+
+  it('Renders the active dataset navigation links correctly', () => {
+    const htmlString = `
+      {% from "components/dataset-navigation.html" import datasetNavigation %}
+      {{ datasetNavigation({
+        active: "task-list",
+        dataset: dataset,
+        organisation: organisation,
+        issue_count: issueCount
+      }) }}
+    `
+
+    const htmlWithActiveLink = nunjucks.renderString(htmlString, params)
+    const domWithActiveLink = new jsdom.JSDOM(htmlWithActiveLink)
+    const documentWithActiveLink = domWithActiveLink.window.document
+    const links = documentWithActiveLink.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+    const activeLink = documentWithActiveLink.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
+
+    expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
+    expect(activeLink.textContent).toContain('Task list')
+    expect(links.length).toEqual(2)
+  })
+
+  it('Renders the issue count correctly', () => {
+    const paramsWithIssues = {
+      ...params,
+      issueCount: 3
+    }
+    const htmlString = `
+      {% from "components/dataset-navigation.html" import datasetNavigation %}
+      {{ datasetNavigation({
+        active: "task-list",
+        dataset: dataset,
+        organisation: organisation,
+        issue_count: issueCount
+      }) }}
+    `
+
+    const htmlWithIssues = nunjucks.renderString(htmlString, paramsWithIssues)
+    const domWithIssues = new jsdom.JSDOM(htmlWithIssues)
+    const documentWithIssues = domWithIssues.window.document
+    const links = documentWithIssues.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+    const activeLink = documentWithIssues.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
+    const issueCount = documentWithIssues.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .govuk-tag')
+
+    expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
+    expect(activeLink.textContent).toContain('Task list')
+    expect(issueCount.textContent).toContain('3')
+    expect(links.length).toEqual(2)
+  })
+})
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index ede7a12a..33594824 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -37,11 +37,11 @@ describe('Dataset Overview Page', () => {
 
   it('Renders the dataset navigation links correctly', () => {
     const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
+    const activeLink = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
 
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(links.length).toEqual(2)
-    expect(links[0].textContent).toContain('Dataset overview')
-    expect(links[1].textContent).toContain('Task list')
+    expect(activeLink.textContent).toContain('Dataset overview')
   })
 
   it('Renders dataset details correctly', () => {

From a823ae7627331d75c90bcf89c0cbf4dc45f75154 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 09:57:48 +0100
Subject: [PATCH 39/65] added middleware builders test file

---
 .../middleware.builders.test.js}                          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
 rename test/unit/{middleware.test.js => middleware/middleware.builders.test.js} (95%)

diff --git a/test/unit/middleware.test.js b/test/unit/middleware/middleware.builders.test.js
similarity index 95%
rename from test/unit/middleware.test.js
rename to test/unit/middleware/middleware.builders.test.js
index dc54c84d..456028a0 100644
--- a/test/unit/middleware.test.js
+++ b/test/unit/middleware/middleware.builders.test.js
@@ -1,14 +1,14 @@
 import { describe, it, vi, expect, beforeEach } from 'vitest'
-import datasette from '../../src/services/datasette.js'
-import * as middleware from '../../src/controllers/middleware.js'
+import datasette from '../../../src/services/datasette.js'
+import * as middleware from '../../../src/middleware/middleware.builders.js'
 
-vi.mock('../../src/services/datasette.js', () => ({
+vi.mock('../../../src/services/datasette.js', () => ({
   default: {
     runQuery: vi.fn()
   }
 }))
 
-describe('Middleware', () => {
+describe('middleware.builders', () => {
   beforeEach(() => {
     vi.resetAllMocks()
   })

From 223ec7c297d4f5df33bdfe6f62ddd25eb7fb3953 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 11:24:36 +0100
Subject: [PATCH 40/65] move datasettaskList middleware tests to their own file

---
 .../datasetTaskList.middleware.test.js        | 109 ++++++++++++++++++
 test/unit/organisationsController.test.js     | 104 -----------------
 2 files changed, 109 insertions(+), 104 deletions(-)
 create mode 100644 test/unit/middleware/datasetTaskList.middleware.test.js

diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js
new file mode 100644
index 00000000..d176617b
--- /dev/null
+++ b/test/unit/middleware/datasetTaskList.middleware.test.js
@@ -0,0 +1,109 @@
+import { describe, it, vi, expect } from 'vitest'
+import performanceDbApi from '../../../src/services/performanceDbApi.js'
+import { prepareDatasetTaskListTemplateParams, prepareDatasetTaskListErrorTemplateParams } from '../../../src/middleware/datasetTaskList.middleware.js'
+
+vi.mock('../../../src/services/performanceDbApi.js')
+
+describe('datasetTaskList.middleware.js', () => {
+  describe('prepareDatasetTaskListParams', () => {
+    it('sets the correct template params on the request object', async () => {
+      const req = {
+        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
+        resourceStatus: {
+          resource: 'mock-resource',
+          endpoint_url: 'http://example.com/resource',
+          status: '200',
+          latest_log_entry_date: '',
+          days_since_200: 0
+        },
+        orgInfo: { name: 'Example Organisation', organisation: 'ORG' },
+        dataset: { name: 'Example Dataset' },
+        resource: { resource: 'mock-resource' },
+        issues: [
+          {
+            issue: 'Example issue 1',
+            issue_type: 'Example issue type 1',
+            field: 'Example issue field 1',
+            num_issues: 1,
+            status: 'Error'
+          },
+          {
+            issue: 'Example issue 2',
+            issue_type: 'Example issue type 2',
+            field: 'Example issue field 2',
+            num_issues: 1,
+            status: 'Needs fixing'
+          }
+        ]
+      }
+      const res = { render: vi.fn() }
+      const next = vi.fn()
+
+      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('task message 1').mockReturnValueOnce('task message 2')
+
+      prepareDatasetTaskListTemplateParams(req, res, next)
+
+      const templateParams = {
+        taskList: [
+          {
+            title: {
+              text: 'task message 1'
+            },
+            href: '/organisations/example-lpa/example-dataset/Example issue type 1/Example issue field 1',
+            status: {
+              tag: {
+                classes: 'govuk-tag--red',
+                text: 'Error'
+              }
+            }
+          },
+          {
+            title: {
+              text: 'task message 2'
+            },
+            href: '/organisations/example-lpa/example-dataset/Example issue type 2/Example issue field 2',
+            status: {
+              tag: {
+                classes: 'govuk-tag--yellow',
+                text: 'Needs fixing'
+              }
+            }
+          }
+        ],
+        organisation: { name: 'Example Organisation', organisation: 'ORG' },
+        dataset: { name: 'Example Dataset' }
+      }
+
+      expect(req.templateParams).toEqual(templateParams)
+    })
+  })
+
+  describe('prepareDatasetTaskListErrorTemplateParams', () => {
+    it('sets the correct template params on the request object', async () => {
+      const resourceStatus = { status: '404', days_since_200: 3, endpoint_url: 'https://example.com', latest_log_entry_date: '2022-01-01T12:00:00.000Z' }
+      const organisation = { name: 'Example Organisation', organisation: 'ORG' }
+      const dataset = { name: 'Example Dataset', dataset: 'example-dataset' }
+      const req = {
+        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
+        resourceStatus,
+        orgInfo: organisation,
+        dataset
+      }
+      const res = { render: vi.fn() }
+      const next = vi.fn()
+
+      prepareDatasetTaskListErrorTemplateParams(req, res, next)
+
+      const templateParams = req.templateParams
+
+      const dataTimeRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z/
+
+      expect(templateParams.organisation).toEqual(organisation)
+      expect(templateParams.dataset).toEqual(dataset)
+      expect(templateParams.errorData.endpoint_url).toEqual('https://example.com')
+      expect(templateParams.errorData.http_status).toEqual('404')
+      expect(templateParams.errorData.latest_log_entry_date).toMatch(dataTimeRegex)
+      expect(templateParams.errorData.latest_200_date).toMatch(dataTimeRegex)
+    })
+  })
+})
diff --git a/test/unit/organisationsController.test.js b/test/unit/organisationsController.test.js
index a3ca6e5a..bf5d9bcd 100644
--- a/test/unit/organisationsController.test.js
+++ b/test/unit/organisationsController.test.js
@@ -166,79 +166,6 @@ describe('OrganisationsController.js', () => {
     })
   })
 
-  describe('dataset task list', () => {
-    it('should fetch the dataset tasks and correctly pass them on to the dataset task list page', async () => {
-      const req = {
-        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
-        resourceStatus: {
-          resource: 'mock-resource',
-          endpoint_url: 'http://example.com/resource',
-          status: '200',
-          latest_log_entry_date: '',
-          days_since_200: 0
-        },
-        orgInfo: { name: 'Example Organisation', organisation: 'ORG' },
-        dataset: { name: 'Example Dataset' },
-        resource: { resource: 'mock-resource' },
-        issues: [
-          {
-            issue: 'Example issue 1',
-            issue_type: 'Example issue type 1',
-            field: 'Example issue field 1',
-            num_issues: 1,
-            status: 'Error'
-          },
-          {
-            issue: 'Example issue 2',
-            issue_type: 'Example issue type 2',
-            field: 'Example issue field 2',
-            num_issues: 1,
-            status: 'Needs fixing'
-          }
-        ]
-      }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('task message 1').mockReturnValueOnce('task message 2')
-
-      organisationsController.prepareDatasetTaskListTemplateParams(req, res, next)
-      organisationsController.getDatasetTaskList(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/datasetTaskList.html', {
-        taskList: [
-          {
-            title: {
-              text: 'task message 1'
-            },
-            href: '/organisations/example-lpa/example-dataset/Example issue type 1/Example issue field 1',
-            status: {
-              tag: {
-                classes: 'govuk-tag--red',
-                text: 'Error'
-              }
-            }
-          },
-          {
-            title: {
-              text: 'task message 2'
-            },
-            href: '/organisations/example-lpa/example-dataset/Example issue type 2/Example issue field 2',
-            status: {
-              tag: {
-                classes: 'govuk-tag--yellow',
-                text: 'Needs fixing'
-              }
-            }
-          }
-        ],
-        organisation: { name: 'Example Organisation', organisation: 'ORG' },
-        dataset: { name: 'Example Dataset' }
-      })
-    })
-  })
-
   describe('issue details', () => {
     const orgInfo = { name: 'mock lpa', organisation: 'ORG' }
     const dataset = { name: 'mock dataset', dataset: 'mock-dataset' }
@@ -462,35 +389,4 @@ describe('OrganisationsController.js', () => {
       })
     })
   })
-
-  describe('getDatasetTaskListError', () => {
-    it('should render http-error.html template with correct params', async () => {
-      const resourceStatus = { status: '404', days_since_200: 3, endpoint_url: 'https://example.com', latest_log_entry_date: '2022-01-01T12:00:00.000Z' }
-      const organisation = { name: 'Example Organisation', organisation: 'ORG' }
-      const dataset = { name: 'Example Dataset', dataset: 'example-dataset' }
-      const req = {
-        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
-        resourceStatus,
-        orgInfo: organisation,
-        dataset
-      }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      organisationsController.prepareDatasetTaskListErrorTemplateParams(req, res, next)
-      organisationsController.getDatasetTaskListError(req, res, next)
-
-      const dataTimeRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z/
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      const renderArgs = res.render.mock.calls[0]
-      expect(renderArgs[0]).toEqual('organisations/http-error.html')
-      expect(renderArgs[1].organisation).toEqual(organisation)
-      expect(renderArgs[1].dataset).toEqual(dataset)
-      expect(renderArgs[1].errorData.endpoint_url).toEqual('https://example.com')
-      expect(renderArgs[1].errorData.http_status).toEqual('404')
-      expect(renderArgs[1].errorData.latest_log_entry_date).toMatch(dataTimeRegex)
-      expect(renderArgs[1].errorData.latest_200_date).toMatch(dataTimeRegex)
-    })
-  })
 })

From 05df90c61f52261dc6c7a434a5769b7ecf012217 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 11:32:43 +0100
Subject: [PATCH 41/65] make sure to export function for testing

---
 src/middleware/datasetTaskList.middleware.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js
index 30dc514a..c8420503 100644
--- a/src/middleware/datasetTaskList.middleware.js
+++ b/src/middleware/datasetTaskList.middleware.js
@@ -46,7 +46,7 @@ function getStatusTag (status) {
  * @param {*} next
  * @returns { { templateParams: object }}
  */
-const prepareDatasetTaskListTemplateParams = (req, res, next) => {
+export const prepareDatasetTaskListTemplateParams = (req, res, next) => {
   const { issues, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
   const { lpa, dataset: datasetId } = params
@@ -86,7 +86,7 @@ const getDatasetTaskList = renderTemplate({
  * @param {} next
  * @returns {{ templateParams: object }}
  */
-const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
+export const prepareDatasetTaskListErrorTemplateParams = (req, res, next) => {
   const { orgInfo: organisation, dataset, resourceStatus: resource } = req
 
   const daysSince200 = resource.days_since_200

From a1f91104ecabcec185ad5fb58640ed299e0a752b Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 11:32:59 +0100
Subject: [PATCH 42/65] move get started middleware tests to their own file

---
 src/middleware/getStarted.middleware.js       |  2 +-
 .../middleware/getStarted.middleware.test.js  | 44 +++++++++++++++++++
 test/unit/organisationsController.test.js     | 42 ------------------
 3 files changed, 45 insertions(+), 43 deletions(-)
 create mode 100644 test/unit/middleware/getStarted.middleware.test.js

diff --git a/src/middleware/getStarted.middleware.js b/src/middleware/getStarted.middleware.js
index 9bfc123a..4d2d351b 100644
--- a/src/middleware/getStarted.middleware.js
+++ b/src/middleware/getStarted.middleware.js
@@ -1,7 +1,7 @@
 import { fetchDatasetInfo, fetchOrgInfo, logPageError } from './common.middleware.js'
 import { renderTemplate } from './middleware.builders.js'
 
-const getGetStarted = renderTemplate({
+export const getGetStarted = renderTemplate({
   templateParams (req) {
     const { orgInfo: organisation, dataset } = req
     return { organisation, dataset }
diff --git a/test/unit/middleware/getStarted.middleware.test.js b/test/unit/middleware/getStarted.middleware.test.js
new file mode 100644
index 00000000..5a31ec96
--- /dev/null
+++ b/test/unit/middleware/getStarted.middleware.test.js
@@ -0,0 +1,44 @@
+import { describe, it, vi, expect } from 'vitest'
+import { getGetStarted } from '../../../src/middleware/getStarted.middleware.js'
+
+describe('get-started', () => {
+  const exampleLpa = {
+    formattedData: [
+      { name: 'Example LPA', organisation: 'LPA' }
+    ]
+  }
+  const exampleDataset = { name: 'Example Dataset', dataset: 'example-dataset' }
+
+  it('should render the get-started template with the correct params', async () => {
+    const req = {
+      params: { lpa: 'example-lpa', dataset: 'example-dataset' },
+      orgInfo: exampleLpa.formattedData[0],
+      dataset: exampleDataset
+    }
+    const res = { render: vi.fn() }
+    const next = vi.fn()
+
+    getGetStarted(req, res, next)
+
+    expect(res.render).toHaveBeenCalledTimes(1)
+    expect(res.render).toHaveBeenCalledWith('organisations/get-started.html', {
+      organisation: { name: 'Example LPA', organisation: 'LPA' },
+      dataset: exampleDataset
+    })
+  })
+
+  it('should catch and pass errors to the next function', async () => {
+    const req = {
+      params: { lpa: 'example-lpa', dataset: 'example-dataset' },
+      orgInfo: undefined, // this should fail validation
+      dataset: exampleDataset
+    }
+    const res = { render: vi.fn() }
+    const next = vi.fn()
+
+    getGetStarted(req, res, next)
+
+    expect(next).toHaveBeenCalledTimes(1)
+    expect(res.render).toHaveBeenCalledTimes(0)
+  })
+})
diff --git a/test/unit/organisationsController.test.js b/test/unit/organisationsController.test.js
index bf5d9bcd..f67861db 100644
--- a/test/unit/organisationsController.test.js
+++ b/test/unit/organisationsController.test.js
@@ -124,48 +124,6 @@ describe('OrganisationsController.js', () => {
     })
   })
 
-  describe('get-started', () => {
-    const exampleLpa = {
-      formattedData: [
-        { name: 'Example LPA', organisation: 'LPA' }
-      ]
-    }
-    const exampleDataset = { name: 'Example Dataset', dataset: 'example-dataset' }
-
-    it('should render the get-started template with the correct params', async () => {
-      const req = {
-        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
-        orgInfo: exampleLpa.formattedData[0],
-        dataset: exampleDataset
-      }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      organisationsController.getGetStarted(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/get-started.html', {
-        organisation: { name: 'Example LPA', organisation: 'LPA' },
-        dataset: exampleDataset
-      })
-    })
-
-    it('should catch and pass errors to the next function', async () => {
-      const req = {
-        params: { lpa: 'example-lpa', dataset: 'example-dataset' },
-        orgInfo: undefined, // this should fail validation
-        dataset: exampleDataset
-      }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      organisationsController.getGetStarted(req, res, next)
-
-      expect(next).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledTimes(0)
-    })
-  })
-
   describe('issue details', () => {
     const orgInfo = { name: 'mock lpa', organisation: 'ORG' }
     const dataset = { name: 'mock dataset', dataset: 'mock-dataset' }

From 18cefbdad54823d128d09155e3973be7ea2c8d41 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 12:04:31 +0100
Subject: [PATCH 43/65] move issue details middleware tests to their own file

---
 src/middleware/issueDetails.middleware.js     |   6 +-
 .../issueDetails.middleware.test.js           | 287 ++++++++++++++++++
 test/unit/organisationsController.test.js     | 226 --------------
 3 files changed, 290 insertions(+), 229 deletions(-)
 create mode 100644 test/unit/middleware/issueDetails.middleware.test.js

diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js
index cd44788b..3fee650b 100644
--- a/src/middleware/issueDetails.middleware.js
+++ b/src/middleware/issueDetails.middleware.js
@@ -6,7 +6,7 @@ import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
 
-const IssueDetailsQueryParams = v.object({
+export const IssueDetailsQueryParams = v.object({
   lpa: v.string(),
   dataset: v.string(),
   issue_type: v.string(),
@@ -182,7 +182,7 @@ const processEntryRow = (issueType, issuesByEntryNumber, row) => {
 /***
  * Middleware. Updates req with `templateParams`
  */
-function prepareIssueDetailsTemplateParams (req, res, next) {
+export function prepareIssueDetailsTemplateParams (req, res, next) {
   const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount: entityCountRow } = req
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
@@ -279,7 +279,7 @@ function prepareIssueDetailsTemplateParams (req, res, next) {
  * Middleware. Renders the issue details page with the list of issues, entry data,
  * and organisation and dataset details.
  */
-const getIssueDetails = renderTemplate({
+export const getIssueDetails = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/issueDetails.html',
   handlerName: 'getIssueDetails'
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
new file mode 100644
index 00000000..9e79e561
--- /dev/null
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -0,0 +1,287 @@
+import { describe, it, vi, expect } from 'vitest'
+import * as v from 'valibot'
+
+import performanceDbApi from '../../../src/services/performanceDbApi.js'
+import { getIssueDetails, IssueDetailsQueryParams, prepareIssueDetailsTemplateParams } from '../../../src/middleware/issueDetails.middleware.js'
+
+vi.mock('../../../src/services/performanceDbApi.js')
+
+describe('issueDetails.middleware.js', () => {
+  const orgInfo = { name: 'mock lpa', organisation: 'ORG' }
+  const dataset = { name: 'mock dataset', dataset: 'mock-dataset' }
+  const entryData = [
+    {
+      field: 'start-date',
+      value: '02-02-2022',
+      entry_number: 1
+    }
+  ]
+  const issues = [
+    {
+      entry_number: 0,
+      field: 'start-date',
+      value: '02-02-2022'
+    }
+  ]
+
+  describe('prepareIssueDetailsTemplateParams', () => {
+    it('should correctly set the template params', async () => {
+      const requestParams = {
+        lpa: 'test-lpa',
+        dataset: 'test-dataset',
+        issue_type: 'test-issue-type',
+        issue_field: 'test-issue-field',
+        resourceId: 'test-resource-id',
+        entityNumber: '1'
+      }
+      const req = {
+        params: requestParams,
+        // middleware supplies the below
+        entryNumber: 1,
+        entityCount: { entity_count: 3 },
+        issueEntitiesCount: 1,
+        pageNumber: 1,
+        orgInfo,
+        dataset,
+        entryData,
+        issues,
+        resource: { resource: requestParams.resourceId },
+        issuesByEntryNumber: {
+          1: [
+            {
+              field: 'start-date',
+              value: '02-02-2022',
+              line_number: 1,
+              entry_number: 1,
+              message: 'mock message',
+              issue_type: 'mock type'
+            }
+          ]
+        }
+        // errorHeading -- set  in prepare* fn
+      }
+      v.parse(IssueDetailsQueryParams, req.params)
+
+      issues.forEach(issue => {
+        vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
+      })
+      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
+
+      prepareIssueDetailsTemplateParams(req, {}, () => {})
+
+      const expectedTempalteParams = {
+        organisation: {
+          name: 'mock lpa',
+          organisation: 'ORG'
+        },
+        dataset: {
+          name: 'mock dataset',
+          dataset: 'mock-dataset'
+        },
+        errorHeading: 'mockMessageFor: 0',
+        issueItems: [
+          {
+            html: 'mock task message 1 in record 1',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
+          }
+        ],
+        entry: {
+          title: 'entry: 1',
+          fields: [
+            {
+              key: { text: 'start-date' },
+              value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
+              classes: 'dl-summary-card-list__row--error'
+            }
+          ],
+          geometries: []
+        },
+        issueType: 'test-issue-type',
+        pagination: {
+          items: [{
+            current: true,
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
+            number: 1,
+            type: 'item'
+          }]
+        },
+        issueEntitiesCount: 1,
+        pageNumber: 1
+      }
+
+      expect(req.templateParams).toEqual(expectedTempalteParams)
+    })
+
+    it('should correctly set the template params with the correct geometry params', async () => {
+      const entryData = [
+        {
+          field: 'start-date',
+          value: '02-02-2022',
+          entry_number: 1
+        },
+        {
+          field: 'geometry',
+          value: 'POINT(0 0)',
+          entry_number: 1
+        }
+      ]
+      const requestParams = {
+        lpa: 'test-lpa',
+        dataset: 'test-dataset',
+        issue_type: 'test-issue-type',
+        issue_field: 'test-issue-field',
+        resourceId: 'test-resource-id',
+        entityNumber: '1'
+      }
+      const req = {
+        params: requestParams,
+        // middleware supplies the below
+        entryNumber: 1,
+        entityCount: { entity_count: 3 },
+        issueEntitiesCount: 1,
+        pageNumber: 1,
+        orgInfo,
+        dataset,
+        entryData,
+        issues,
+        resource: { resource: requestParams.resourceId },
+        issuesByEntryNumber: {
+          1: [
+            {
+              field: 'start-date',
+              value: '02-02-2022',
+              line_number: 1,
+              entry_number: 1,
+              message: 'mock message',
+              issue_type: 'mock type'
+            }
+          ]
+        }
+        // errorHeading -- set  in prepare* fn
+      }
+      v.parse(IssueDetailsQueryParams, req.params)
+
+      issues.forEach(issue => {
+        vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
+      })
+      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
+
+      prepareIssueDetailsTemplateParams(req, {}, () => {})
+
+      const expectedTemplateParams = {
+        organisation: {
+          name: 'mock lpa',
+          organisation: 'ORG'
+        },
+        dataset: {
+          name: 'mock dataset',
+          dataset: 'mock-dataset'
+        },
+        errorHeading: 'mockMessageFor: 0',
+        issueItems: [
+          {
+            html: 'mock task message 1 in record 1',
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
+          }
+        ],
+        entry: {
+          title: 'entry: 1',
+          fields: [
+            {
+              key: { text: 'start-date' },
+              value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
+              classes: 'dl-summary-card-list__row--error'
+            },
+            {
+              classes: '',
+              key: {
+                text: 'geometry'
+              },
+              value: {
+                html: 'POINT(0 0)'
+              }
+            }
+          ],
+          geometries: ['POINT(0 0)']
+        },
+        issueType: 'test-issue-type',
+        pagination: {
+          items: [{
+            current: true,
+            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
+            number: 1,
+            type: 'item'
+          }]
+        },
+        issueEntitiesCount: 1,
+        pageNumber: 1
+      }
+
+      expect(req.templateParams).toEqual(expectedTemplateParams)
+    })
+  })
+
+  describe('getIssueDetails', () => {
+    it('should call render using the provided template params and correct view', () => {
+      const req = {
+        templateParams: {
+          organisation: {
+            name: 'mock lpa',
+            organisation: 'ORG'
+          },
+          dataset: {
+            name: 'mock dataset',
+            dataset: 'mock-dataset'
+          },
+          errorHeading: 'mockMessageFor: 0',
+          issueItems: [
+            {
+              html: 'mock task message 1 in record 1',
+              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
+            }
+          ],
+          entry: {
+            title: 'entry: 1',
+            fields: [
+              {
+                key: { text: 'start-date' },
+                value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
+                classes: 'dl-summary-card-list__row--error'
+              },
+              {
+                classes: '',
+                key: {
+                  text: 'geometry'
+                },
+                value: {
+                  html: 'POINT(0 0)'
+                }
+              }
+            ],
+            geometries: ['POINT(0 0)']
+          },
+          issueType: 'test-issue-type',
+          pagination: {
+            items: [{
+              current: true,
+              href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
+              number: 1,
+              type: 'item'
+            }]
+          },
+          issueEntitiesCount: 1,
+          pageNumber: 1
+        }
+      }
+
+      const res = {
+        render: vi.fn()
+      }
+
+      getIssueDetails(req, res, () => {})
+
+      expect(res.render).toHaveBeenCalledTimes(1)
+      expect(res.render).toHaveBeenCalledWith('organisations/issueDetails.html', req.templateParams)
+    })
+  })
+})
diff --git a/test/unit/organisationsController.test.js b/test/unit/organisationsController.test.js
index f67861db..d1a6c6df 100644
--- a/test/unit/organisationsController.test.js
+++ b/test/unit/organisationsController.test.js
@@ -1,7 +1,5 @@
 import { describe, it, vi, expect, beforeEach } from 'vitest'
-import * as v from 'valibot'
 import organisationsController from '../../src/controllers/OrganisationsController.js'
-import performanceDbApi from '../../src/services/performanceDbApi.js'
 
 vi.mock('../../src/services/performanceDbApi.js')
 vi.mock('../../src/utils/utils.js', () => {
@@ -123,228 +121,4 @@ describe('OrganisationsController.js', () => {
       }))
     })
   })
-
-  describe('issue details', () => {
-    const orgInfo = { name: 'mock lpa', organisation: 'ORG' }
-    const dataset = { name: 'mock dataset', dataset: 'mock-dataset' }
-    const entryData = [
-      {
-        field: 'start-date',
-        value: '02-02-2022',
-        entry_number: 1
-      }
-    ]
-    const issues = [
-      {
-        entry_number: 0,
-        field: 'start-date',
-        value: '02-02-2022'
-      }
-    ]
-
-    it('should call render with the issue details page and the correct params', async () => {
-      const requestParams = {
-        lpa: 'test-lpa',
-        dataset: 'test-dataset',
-        issue_type: 'test-issue-type',
-        issue_field: 'test-issue-field',
-        resourceId: 'test-resource-id',
-        entityNumber: '1'
-      }
-      const req = {
-        params: requestParams,
-        // middleware supplies the below
-        entryNumber: 1,
-        entityCount: { entity_count: 3 },
-        issueEntitiesCount: 1,
-        pageNumber: 1,
-        orgInfo,
-        dataset,
-        entryData,
-        issues,
-        resource: { resource: requestParams.resourceId },
-        issuesByEntryNumber: {
-          1: [
-            {
-              field: 'start-date',
-              value: '02-02-2022',
-              line_number: 1,
-              entry_number: 1,
-              message: 'mock message',
-              issue_type: 'mock type'
-            }
-          ]
-        }
-        // errorHeading -- set  in prepare* fn
-      }
-      v.parse(organisationsController.IssueDetailsQueryParams, req.params)
-
-      const res = {
-        render: vi.fn()
-      }
-      const next = vi.fn()
-
-      issues.forEach(issue => {
-        vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
-      })
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
-
-      organisationsController.prepareIssueDetailsTemplateParams(req, {}, () => {})
-      organisationsController.getIssueDetails(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/issueDetails.html', {
-        organisation: {
-          name: 'mock lpa',
-          organisation: 'ORG'
-        },
-        dataset: {
-          name: 'mock dataset',
-          dataset: 'mock-dataset'
-        },
-        errorHeading: 'mockMessageFor: 0',
-        issueItems: [
-          {
-            html: 'mock task message 1 in record 1',
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
-          }
-        ],
-        entry: {
-          title: 'entry: 1',
-          fields: [
-            {
-              key: { text: 'start-date' },
-              value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
-              classes: 'dl-summary-card-list__row--error'
-            }
-          ],
-          geometries: []
-        },
-        issueType: 'test-issue-type',
-        pagination: {
-          items: [{
-            current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
-            number: 1,
-            type: 'item'
-          }]
-        },
-        issueEntitiesCount: 1,
-        pageNumber: 1
-      })
-    })
-
-    it('should call render with the issue details page with the correct geometry params', async () => {
-      const entryData = [
-        {
-          field: 'start-date',
-          value: '02-02-2022',
-          entry_number: 1
-        },
-        {
-          field: 'geometry',
-          value: 'POINT(0 0)',
-          entry_number: 1
-        }
-      ]
-      const requestParams = {
-        lpa: 'test-lpa',
-        dataset: 'test-dataset',
-        issue_type: 'test-issue-type',
-        issue_field: 'test-issue-field',
-        resourceId: 'test-resource-id',
-        entityNumber: '1'
-      }
-      const req = {
-        params: requestParams,
-        // middleware supplies the below
-        entryNumber: 1,
-        entityCount: { entity_count: 3 },
-        issueEntitiesCount: 1,
-        pageNumber: 1,
-        orgInfo,
-        dataset,
-        entryData,
-        issues,
-        resource: { resource: requestParams.resourceId },
-        issuesByEntryNumber: {
-          1: [
-            {
-              field: 'start-date',
-              value: '02-02-2022',
-              line_number: 1,
-              entry_number: 1,
-              message: 'mock message',
-              issue_type: 'mock type'
-            }
-          ]
-        }
-        // errorHeading -- set  in prepare* fn
-      }
-      v.parse(organisationsController.IssueDetailsQueryParams, req.params)
-
-      const res = {
-        render: vi.fn()
-      }
-      const next = vi.fn()
-
-      issues.forEach(issue => {
-        vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce(`mockMessageFor: ${issue.entry_number}`)
-      })
-      vi.mocked(performanceDbApi.getTaskMessage).mockReturnValueOnce('mock task message 1')
-
-      organisationsController.prepareIssueDetailsTemplateParams(req, {}, () => {})
-      organisationsController.getIssueDetails(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/issueDetails.html', {
-        organisation: {
-          name: 'mock lpa',
-          organisation: 'ORG'
-        },
-        dataset: {
-          name: 'mock dataset',
-          dataset: 'mock-dataset'
-        },
-        errorHeading: 'mockMessageFor: 0',
-        issueItems: [
-          {
-            html: 'mock task message 1 in record 1',
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1'
-          }
-        ],
-        entry: {
-          title: 'entry: 1',
-          fields: [
-            {
-              key: { text: 'start-date' },
-              value: { html: '<p class="govuk-error-message">mock message</p>02-02-2022' },
-              classes: 'dl-summary-card-list__row--error'
-            },
-            {
-              classes: '',
-              key: {
-                text: 'geometry'
-              },
-              value: {
-                html: 'POINT(0 0)'
-              }
-            }
-          ],
-          geometries: ['POINT(0 0)']
-        },
-        issueType: 'test-issue-type',
-        pagination: {
-          items: [{
-            current: true,
-            href: '/organisations/test-lpa/test-dataset/test-issue-type/test-issue-field/1',
-            number: 1,
-            type: 'item'
-          }]
-        },
-        issueEntitiesCount: 1,
-        pageNumber: 1
-      })
-    })
-  })
 })

From 1cbaaf47f884386b90c438c49e453103c55053f7 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 12:16:16 +0100
Subject: [PATCH 44/65] moved organisations middleware tests to their own file

---
 src/middleware/organisations.middleware.js    |  4 +-
 .../organisations.middleware.test.js          | 80 +++++++++++++++++++
 test/unit/organisationsController.test.js     | 58 --------------
 3 files changed, 82 insertions(+), 60 deletions(-)
 create mode 100644 test/unit/middleware/organisations.middleware.test.js

diff --git a/src/middleware/organisations.middleware.js b/src/middleware/organisations.middleware.js
index b9b67da5..9efcc4a5 100644
--- a/src/middleware/organisations.middleware.js
+++ b/src/middleware/organisations.middleware.js
@@ -13,7 +13,7 @@ const fetchOrganisations = fetchMany({
  * @param {*} res
  * @param {*} next
  */
-const prepareGetOrganisationsTemplateParams = (req, res, next) => {
+export const prepareGetOrganisationsTemplateParams = (req, res, next) => {
   const sortedResults = req.organisations.sort((a, b) => {
     return a.name.localeCompare(b.name)
   })
@@ -30,7 +30,7 @@ const prepareGetOrganisationsTemplateParams = (req, res, next) => {
   next()
 }
 
-const getOrganisations = renderTemplate({
+export const getOrganisations = renderTemplate({
   templateParams: (req) => req.templateParams,
   template: 'organisations/find.html',
   handlerName: 'getOrganisations'
diff --git a/test/unit/middleware/organisations.middleware.test.js b/test/unit/middleware/organisations.middleware.test.js
new file mode 100644
index 00000000..e9b456a0
--- /dev/null
+++ b/test/unit/middleware/organisations.middleware.test.js
@@ -0,0 +1,80 @@
+import { describe, it, vi, expect } from 'vitest'
+import { getOrganisations, prepareGetOrganisationsTemplateParams } from '../../../src/middleware/organisations.middleware'
+
+describe('organisations.middleware.js', () => {
+  describe('prepareGetOrganisationsTemplateParams', () => {
+    it('should correctly sort and restructure the data received from datasette, then set them on req.template params', async () => {
+      const req = {}
+      const res = { render: vi.fn() }
+      const next = vi.fn()
+
+      const datasetteResponse = [
+        { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' },
+        { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' },
+        { name: 'Bristol Hospital', organisation: 'Bristol Hospital' },
+        { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' },
+        { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' },
+        { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' }
+      ]
+
+      req.organisations = datasetteResponse
+      prepareGetOrganisationsTemplateParams(req, res, next)
+
+      const expectedTemplatePrams = {
+        alphabetisedOrgs: {
+          A: [
+            { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' }
+          ],
+          B: [
+            { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' },
+            { name: 'Bristol Hospital', organisation: 'Bristol Hospital' }
+          ],
+          C: [
+            { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' }
+          ],
+          D: [
+            { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' }
+          ],
+          E: [
+            { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' }
+          ]
+        }
+      }
+
+      expect(req.templateParams).toEqual(expectedTemplatePrams)
+
+      expect(next).toHaveBeenCalledOnce()
+    })
+  })
+  it('should call render with the find page', async () => {
+    const req = {}
+    const res = { render: vi.fn() }
+    const next = vi.fn()
+
+    req.templateParams = {
+      alphabetisedOrgs: {
+        A: [
+          { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' }
+        ],
+        B: [
+          { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' },
+          { name: 'Bristol Hospital', organisation: 'Bristol Hospital' }
+        ],
+        C: [
+          { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' }
+        ],
+        D: [
+          { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' }
+        ],
+        E: [
+          { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' }
+        ]
+      }
+    }
+
+    getOrganisations(req, res, next)
+
+    expect(res.render).toHaveBeenCalledTimes(1)
+    expect(res.render).toHaveBeenCalledWith('organisations/find.html', req.templateParams)
+  })
+})
diff --git a/test/unit/organisationsController.test.js b/test/unit/organisationsController.test.js
index d1a6c6df..b933478e 100644
--- a/test/unit/organisationsController.test.js
+++ b/test/unit/organisationsController.test.js
@@ -63,62 +63,4 @@ describe('OrganisationsController.js', () => {
       }))
     })
   })
-
-  describe('find', () => {
-    it('should call render with the find page', async () => {
-      const req = {}
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      req.organisations = []
-      organisationsController.prepareGetOrganisationsTemplateParams(req, res, next)
-      organisationsController.getOrganisations(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/find.html', expect.objectContaining({
-        alphabetisedOrgs: {}
-      }))
-    })
-
-    it('should correctly sort and restructure the data recieved from datasette, then pass it on to the template', async () => {
-      const req = {}
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      const datasetteResponse = [
-        { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' },
-        { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' },
-        { name: 'Bristol Hospital', organisation: 'Bristol Hospital' },
-        { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' },
-        { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' },
-        { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' }
-      ]
-
-      req.organisations = datasetteResponse
-      organisationsController.prepareGetOrganisationsTemplateParams(req, res, next)
-      organisationsController.getOrganisations(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/find.html', expect.objectContaining({
-        alphabetisedOrgs: {
-          A: [
-            { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' }
-          ],
-          B: [
-            { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' },
-            { name: 'Bristol Hospital', organisation: 'Bristol Hospital' }
-          ],
-          C: [
-            { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' }
-          ],
-          D: [
-            { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' }
-          ],
-          E: [
-            { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' }
-          ]
-        }
-      }))
-    })
-  })
 })

From b949105f4ad8f8446fca99e400446786102601cf Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 15:06:40 +0100
Subject: [PATCH 45/65] moved overview middleware tests to their own file

---
 src/middleware/overview.middleware.js         |  4 +-
 .../organisations.middleware.test.js          |  2 +-
 .../middleware/overview.middleware.test.js    | 81 +++++++++++++++++++
 test/unit/organisationsController.test.js     | 66 ---------------
 4 files changed, 84 insertions(+), 69 deletions(-)
 create mode 100644 test/unit/middleware/overview.middleware.test.js
 delete mode 100644 test/unit/organisationsController.test.js

diff --git a/src/middleware/overview.middleware.js b/src/middleware/overview.middleware.js
index af951eb6..483927e1 100644
--- a/src/middleware/overview.middleware.js
+++ b/src/middleware/overview.middleware.js
@@ -32,7 +32,7 @@ async function fetchLpaOverview (req, res, next) {
   }
 }
 
-function prepareOverviewTemplateParams (req, res, next) {
+export function prepareOverviewTemplateParams (req, res, next) {
   const { lpaOverview, orgInfo: organisation } = req
 
   const datasets = Object.entries(lpaOverview.datasets).map(([key, value]) => {
@@ -82,7 +82,7 @@ function prepareOverviewTemplateParams (req, res, next) {
   next()
 }
 
-const getOverview = renderTemplate({
+export const getOverview = renderTemplate({
   templateParams (req) {
     if (!req.templateParams) throw new Error('missing templateParams')
     return req.templateParams
diff --git a/test/unit/middleware/organisations.middleware.test.js b/test/unit/middleware/organisations.middleware.test.js
index e9b456a0..9650e533 100644
--- a/test/unit/middleware/organisations.middleware.test.js
+++ b/test/unit/middleware/organisations.middleware.test.js
@@ -46,7 +46,7 @@ describe('organisations.middleware.js', () => {
       expect(next).toHaveBeenCalledOnce()
     })
   })
-  it('should call render with the find page', async () => {
+  it('should call render with the find page and correct template params', async () => {
     const req = {}
     const res = { render: vi.fn() }
     const next = vi.fn()
diff --git a/test/unit/middleware/overview.middleware.test.js b/test/unit/middleware/overview.middleware.test.js
new file mode 100644
index 00000000..2b538bf8
--- /dev/null
+++ b/test/unit/middleware/overview.middleware.test.js
@@ -0,0 +1,81 @@
+import { describe, it, vi, expect } from 'vitest'
+import { getOverview, prepareOverviewTemplateParams } from '../../../src/middleware/overview.middleware'
+
+// vi.mock('../../../src/services/performanceDbApi.js')
+
+vi.mock('../../../src/utils/utils.js', () => {
+  return {
+    dataSubjects: {}
+  }
+})
+
+describe('overview.middleware', () => {
+  const exampleLpa = {
+    formattedData: [
+      { name: 'Example LPA', organisation: 'LPA' }
+    ]
+  }
+
+  const perfDbApiResponse = {
+    name: 'test LPA',
+    datasets: {
+      dataset1: { endpoint: 'https://example.com', status: 'Live' },
+      dataset2: { endpoint: null, status: 'Needs fixing' },
+      dataset3: { endpoint: 'https://example.com', status: 'Error' }
+    }
+  }
+
+  describe('prepareOverviewTemplateParams', () => {
+    it('should render the overview page', async () => {
+      const req = {
+        params: { lpa: 'LPA' },
+        orgInfo: exampleLpa.formattedData[0],
+        lpaOverview: perfDbApiResponse
+      }
+      const res = { render: vi.fn() }
+
+      prepareOverviewTemplateParams(req, res, () => {})
+
+      const expectedTemplateParams = {
+        organisation: { name: 'Example LPA', organisation: 'LPA' },
+        datasets: expect.arrayContaining([
+          { endpoint: 'https://example.com', status: 'Live', slug: 'dataset1' },
+          { endpoint: null, status: 'Needs fixing', slug: 'dataset2' },
+          { endpoint: 'https://example.com', status: 'Error', slug: 'dataset3' }
+        ]),
+        totalDatasets: 3,
+        datasetsWithEndpoints: 2,
+        datasetsWithIssues: 1,
+        datasetsWithErrors: 1
+      }
+
+      expect(req.templateParams).toEqual(expectedTemplateParams)
+    })
+  })
+
+  describe('getOverview', () => {
+    it('should call render with the correct template and params', () => {
+      const req = {}
+      const res = { render: vi.fn() }
+      const next = vi.fn()
+
+      req.templateParams = {
+        organisation: { name: 'Example LPA', organisation: 'LPA' },
+        datasets: [
+          { endpoint: 'https://example.com', status: 'Live', slug: 'dataset1' },
+          { endpoint: null, status: 'Needs fixing', slug: 'dataset2' },
+          { endpoint: 'https://example.com', status: 'Error', slug: 'dataset3' }
+        ],
+        totalDatasets: 3,
+        datasetsWithEndpoints: 2,
+        datasetsWithIssues: 1,
+        datasetsWithErrors: 1
+      }
+
+      getOverview(req, res, next)
+
+      expect(res.render).toHaveBeenCalledTimes(1)
+      expect(res.render).toHaveBeenCalledWith('organisations/overview.html', req.templateParams)
+    })
+  })
+})
diff --git a/test/unit/organisationsController.test.js b/test/unit/organisationsController.test.js
deleted file mode 100644
index b933478e..00000000
--- a/test/unit/organisationsController.test.js
+++ /dev/null
@@ -1,66 +0,0 @@
-import { describe, it, vi, expect, beforeEach } from 'vitest'
-import organisationsController from '../../src/controllers/OrganisationsController.js'
-
-vi.mock('../../src/services/performanceDbApi.js')
-vi.mock('../../src/utils/utils.js', () => {
-  return {
-    dataSubjects: {}
-  }
-})
-
-vi.mock('../../src/services/datasette.js', () => ({
-  default: {
-    runQuery: vi.fn()
-  }
-}))
-vi.mock('../../src/utils/issueMessages/getDatasetTaskList.js')
-
-describe('OrganisationsController.js', () => {
-  beforeEach(() => {
-    vi.resetAllMocks()
-  })
-
-  describe('overview', () => {
-    const exampleLpa = {
-      formattedData: [
-        { name: 'Example LPA', organisation: 'LPA' }
-      ]
-    }
-
-    const perfDbApiResponse = {
-      name: 'test LPA',
-      datasets: {
-        dataset1: { endpoint: 'https://example.com', status: 'Live' },
-        dataset2: { endpoint: null, status: 'Needs fixing' },
-        dataset3: { endpoint: 'https://example.com', status: 'Error' }
-      }
-    }
-
-    it('should render the overview page', async () => {
-      const req = {
-        params: { lpa: 'LPA' },
-        orgInfo: exampleLpa.formattedData[0],
-        lpaOverview: perfDbApiResponse
-      }
-      const res = { render: vi.fn() }
-      const next = vi.fn()
-
-      organisationsController.prepareOverviewTemplateParams(req, res, () => {})
-      organisationsController.getOverview(req, res, next)
-
-      expect(res.render).toHaveBeenCalledTimes(1)
-      expect(res.render).toHaveBeenCalledWith('organisations/overview.html', expect.objectContaining({
-        organisation: { name: 'Example LPA', organisation: 'LPA' },
-        datasets: expect.arrayContaining([
-          { endpoint: 'https://example.com', status: 'Live', slug: 'dataset1' },
-          { endpoint: null, status: 'Needs fixing', slug: 'dataset2' },
-          { endpoint: 'https://example.com', status: 'Error', slug: 'dataset3' }
-        ]),
-        totalDatasets: 3,
-        datasetsWithEndpoints: 2,
-        datasetsWithIssues: 1,
-        datasetsWithErrors: 1
-      }))
-    })
-  })
-})

From b0d99c58344404503e6a415c26faa6596184461a Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 15:34:28 +0100
Subject: [PATCH 46/65] format dates with govuk datetime

---
 src/views/organisations/dataset-overview.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index 608b868f..add912b9 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -118,7 +118,7 @@
         text: 'Data URL last accessed'
       },
       value: {
-        text: endpoint.lastAccessed
+        text: endpoint.lastAccessed | govukDateTime
       }
     } %}
   {% endif %}
@@ -131,7 +131,7 @@
       text: 'Data URL last updated'
     },
     value: {
-      text: endpoint.lastUpdated
+      text: endpoint.lastUpdated | govukDateTime
     }
   } %}
   {{ rows.push(lastUpdatedRow) }}

From 8992a3fc5cdb38d16a5aa43798b046f2bedd5075 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 15:38:43 +0100
Subject: [PATCH 47/65] Show overview page unless not started

---
 src/views/organisations/overview.html | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html
index bd200bcd..a0862a24 100644
--- a/src/views/organisations/overview.html
+++ b/src/views/organisations/overview.html
@@ -69,16 +69,11 @@ <h2 class="govuk-heading-m">Datasets</h2>
         <li class="govuk-task-list__item govuk-task-list__item--with-link">
           <div class="govuk-task-list__name-and-hint">
             <h2 class="govuk-heading-m">
-                {% if dataset.status === "Live" %}
+              {% if dataset.endpoint %}
                 <a class="govuk-link govuk-task-list__link"
                   href="/organisations/{{ organisation.organisation }}/{{dataset.slug}}/overview">
                   {{dataset.slug | datasetSlugToReadableName}}
                 </a>
-              {% elseif dataset.endpoint %}
-                <a class="govuk-link govuk-task-list__link"
-                  href="/organisations/{{ organisation.organisation }}/{{dataset.slug}}">
-                  {{dataset.slug | datasetSlugToReadableName}}
-                </a>
               {% else %}
                 <a class="govuk-link govuk-task-list__link"
                   href="/organisations/{{ organisation.organisation }}/{{dataset.slug}}/get-started">

From bf5b1f6e60d71ec21664c1073b1e8056b2208eb9 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 15:45:27 +0100
Subject: [PATCH 48/65] temp truncating of urls

---
 src/views/organisations/dataset-overview.html | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index add912b9..d46202dd 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -16,6 +16,8 @@
     {{organisation.name}} - {{dataset.name}} - Dataset overview
 {% endset %}
 
+{% set urlStyle = 'text-overflow: ellipsis; overflow: hidden; white-space: nowrap; width: 400px; display: block;' %}
+
 {% block beforeContent %}
 
 {{ super() }}
@@ -84,7 +86,7 @@
       classes: 'padding-top'
     },
     value: {
-      html: '<a href="'+endpoint.endpoint+'">'+endpoint.endpoint+'</a>'
+      html: '<a  style="'+urlStyle+'" href="'+endpoint.endpoint+'">'+endpoint.endpoint+'</a>'
     },
     classes: 'padding-top'
   } %}
@@ -96,7 +98,7 @@
         text: 'Documentation URL'
       },
       value: {
-        html: '<a href="'+endpoint.documentation_url+'">'+endpoint.documentation_url+'</a>'
+        html: '<a style="'+urlStyle+'" href="'+endpoint.documentation_url+'">'+endpoint.documentation_url+'</a>'
       }
     } %}p
     {{ rows.push(documentationRow) }}

From 91c07ee9294680d3a67a49be170135a4901187bb Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 15:46:41 +0100
Subject: [PATCH 49/65] fix typo

---
 src/views/organisations/dataset-overview.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index d46202dd..22e8210f 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -71,7 +71,7 @@
   },
   {
     key: {
-      text: "License"
+      text: "Licence"
     },
     value: {
       text: "Open Government Licence" 

From a14d09e63b9b060ec1847aa308042c875762583f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 15:47:12 +0100
Subject: [PATCH 50/65] make sure error type is correct

---
 src/services/DatasetService.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
index 5b20c02b..51edc68f 100644
--- a/src/services/DatasetService.js
+++ b/src/services/DatasetService.js
@@ -20,7 +20,7 @@ async function getFieldStats (lpa, dataset) {
   const columnSummary = await datasetService.getColumnSummary(dataset, lpa)
   const specifications = await datasetService.getSpecifications()
   if (!(dataset in specifications)) {
-    logger.warn(`services/datasetService.getFieldStats(): cannot find specification for dataset: ${dataset}`, { type: types.app, dataset })
+    logger.warn(`services/datasetService.getFieldStats(): cannot find specification for dataset: ${dataset}`, { type: types.App, dataset })
     return null
   }
 

From a3f350cb8e6a8f43046b40675193c280b1aceb77 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 15:53:49 +0100
Subject: [PATCH 51/65] Add notification badge

---
 .../scss/components/_dataset-navigation.scss       | 14 ++++++++++++--
 src/views/components/dataset-navigation.html       |  4 ++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/assets/scss/components/_dataset-navigation.scss b/src/assets/scss/components/_dataset-navigation.scss
index 6cf45782..852c3532 100644
--- a/src/assets/scss/components/_dataset-navigation.scss
+++ b/src/assets/scss/components/_dataset-navigation.scss
@@ -1,4 +1,14 @@
 .app-c-dataset-navigation {
-  background-color: #fff;
-  border-top: 1px solid #bfc1c3;
+  background-color: govuk-colour('white');
+  border-top: 1px solid govuk-colour('mid-grey');
+}
+
+.app-c-dataset-navigation__notification-badge {
+  background-color: govuk-colour('red');
+  border-radius: 7%;
+  color: govuk-colour('white');
+  text-align: center;
+  padding: 2px 5px;
+  font-size: 1rem;
+  margin-left: 5px;
 }
diff --git a/src/views/components/dataset-navigation.html b/src/views/components/dataset-navigation.html
index 039834a6..56e86b91 100644
--- a/src/views/components/dataset-navigation.html
+++ b/src/views/components/dataset-navigation.html
@@ -5,7 +5,7 @@
   {% set dataset = params.dataset %}
   {% set badge %}
     {% if params.issue_count > 0 %}
-      <strong class="govuk-tag govuk-tag--red">
+      <strong class="app-c-dataset-navigation__notification-badge">
         {{ params.issue_count }}
       </strong>
     {% endif %}
@@ -26,4 +26,4 @@
       }
     ]
   }) }}
-{% endmacro %}
\ No newline at end of file
+{% endmacro %}

From 29d34476af911a7235605a49d823645742368858 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 15:58:24 +0100
Subject: [PATCH 52/65] add visually hidden text

---
 src/views/components/dataset-navigation.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/views/components/dataset-navigation.html b/src/views/components/dataset-navigation.html
index 56e86b91..6bda664f 100644
--- a/src/views/components/dataset-navigation.html
+++ b/src/views/components/dataset-navigation.html
@@ -6,7 +6,7 @@
   {% set badge %}
     {% if params.issue_count > 0 %}
       <strong class="app-c-dataset-navigation__notification-badge">
-        {{ params.issue_count }}
+        {{ params.issue_count }} <span class="govuk-visually-hidden">{{ "issue" | pluralise(params.issue_count) }}</span>
       </strong>
     {% endif %}
   {% endset %}

From 91c16e8aca1437f996f95325683be0c7fc151ab5 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Wed, 25 Sep 2024 16:17:49 +0100
Subject: [PATCH 53/65] update docs

---
 docs/Middleware Guidelines.md         | 9 ++++++---
 src/middleware/middleware.builders.js | 9 +++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/docs/Middleware Guidelines.md b/docs/Middleware Guidelines.md
index 8e67c1af..2fc743a0 100644
--- a/docs/Middleware Guidelines.md	
+++ b/docs/Middleware Guidelines.md	
@@ -2,10 +2,13 @@
 
 This document outlines convetions/rules of thumb for adding new routes/middleware to the app.
 
-Relevant modules: 
-- [controllers/middleware.js](../src/controllers/middleware.js)
+Relevant modules:
+- [middleware/*](../src/middleware)
+  - [common.middleware.js](../src/middleware/common.middleware.js) (common middlewares that are used by more than one middleware chain)
+  - [middleware.builders.js](../src/middleware/middleware.builders.js) (Middleware builders for data fetching, conditional execution, and template rendering.)
+  - middleware chains and any middleware that is used by only that chain should be put into its own file for example [middleware/issueDetails.middleware.js](../src/middleware/issueDetails.middleware.js)
 - [routes/schemas.js](../src/routes/schemas.js)
-- [OrganisationsController.js](../src/controllers/OrganisationsController.js) - contains usage examples of the two above
+
 
 ## Middleware chains
 
diff --git a/src/middleware/middleware.builders.js b/src/middleware/middleware.builders.js
index 45dbd70d..3340afd7 100644
--- a/src/middleware/middleware.builders.js
+++ b/src/middleware/middleware.builders.js
@@ -1,3 +1,12 @@
+/**
+ * Middleware builders for data fetching, conditional execution, and template rendering.
+ *
+ * This file provides a set of reusable middleware functions and utility functions that
+ * can be composed together to create custom workflows for a web application.
+ * Includes functions for fetching data from a dataset, conditionally executing
+ * middleware, and rendering templates with validation.
+ */
+
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
 import { templateSchema } from '../routes/schemas.js'

From dd72f325523a21821151a04f8bc159ff6cbd3249 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 16:20:31 +0100
Subject: [PATCH 54/65] Force hide underline text decorations

---
 src/assets/scss/components/_dataset-navigation.scss | 13 +++++++++++--
 src/views/components/dataset-navigation.html        |  4 ++--
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/assets/scss/components/_dataset-navigation.scss b/src/assets/scss/components/_dataset-navigation.scss
index 852c3532..50812134 100644
--- a/src/assets/scss/components/_dataset-navigation.scss
+++ b/src/assets/scss/components/_dataset-navigation.scss
@@ -1,14 +1,23 @@
+$badgeBgColour: govuk-colour('red');
+
 .app-c-dataset-navigation {
   background-color: govuk-colour('white');
   border-top: 1px solid govuk-colour('mid-grey');
 }
 
 .app-c-dataset-navigation__notification-badge {
-  background-color: govuk-colour('red');
-  border-radius: 7%;
+  background-color: $badgeBgColour;
+  border-radius: 10%;
   color: govuk-colour('white');
   text-align: center;
   padding: 2px 5px;
   font-size: 1rem;
   margin-left: 5px;
 }
+
+/* Hides text underline decoration for badge */
+.app-c-dataset-navigation .govuk-service-navigation__link:hover .app-c-dataset-navigation__notification-badge {
+  text-decoration: underline;
+  text-decoration-color: $badgeBgColour;
+  text-decoration-thickness: $govuk-link-hover-underline-thickness;
+}
diff --git a/src/views/components/dataset-navigation.html b/src/views/components/dataset-navigation.html
index 6bda664f..9cd05f72 100644
--- a/src/views/components/dataset-navigation.html
+++ b/src/views/components/dataset-navigation.html
@@ -5,9 +5,9 @@
   {% set dataset = params.dataset %}
   {% set badge %}
     {% if params.issue_count > 0 %}
-      <strong class="app-c-dataset-navigation__notification-badge">
+      <span class="app-c-dataset-navigation__notification-badge">
         {{ params.issue_count }} <span class="govuk-visually-hidden">{{ "issue" | pluralise(params.issue_count) }}</span>
-      </strong>
+      </span>
     {% endif %}
   {% endset %}
 

From 4ecfa0da0aa82d7ac4e31d4899549c4c80adcdfe Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 16:26:28 +0100
Subject: [PATCH 55/65] increase horizontal padding for badge

---
 src/assets/scss/components/_dataset-navigation.scss | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/assets/scss/components/_dataset-navigation.scss b/src/assets/scss/components/_dataset-navigation.scss
index 50812134..68c77caf 100644
--- a/src/assets/scss/components/_dataset-navigation.scss
+++ b/src/assets/scss/components/_dataset-navigation.scss
@@ -10,7 +10,7 @@ $badgeBgColour: govuk-colour('red');
   border-radius: 10%;
   color: govuk-colour('white');
   text-align: center;
-  padding: 2px 5px;
+  padding: 2px 6px;
   font-size: 1rem;
   margin-left: 5px;
 }

From a97dae122fcb3ed11e96c1c93437c2a665df9502 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 16:40:46 +0100
Subject: [PATCH 56/65] update tests to include visually hidden text

---
 test/unit/datasetTaskListPage.test.js                 | 2 +-
 test/unit/views/components/dataset-navigation.test.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js
index 99a0b3b4..e153e53f 100644
--- a/test/unit/datasetTaskListPage.test.js
+++ b/test/unit/datasetTaskListPage.test.js
@@ -91,7 +91,7 @@ describe('Dataset Task List Page', () => {
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(links.length).toEqual(2)
     expect(activeLink.textContent).toContain('Task list')
-    expect(issueCount.textContent).toContain('3')
+    expect(issueCount.textContent).toContain('3 issues')
   })
 
   const taskListItems = document.querySelectorAll('.govuk-task-list__item')
diff --git a/test/unit/views/components/dataset-navigation.test.js b/test/unit/views/components/dataset-navigation.test.js
index 383d6f16..3332479f 100644
--- a/test/unit/views/components/dataset-navigation.test.js
+++ b/test/unit/views/components/dataset-navigation.test.js
@@ -87,7 +87,7 @@ describe('Dataset Navigation component', () => {
 
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(activeLink.textContent).toContain('Task list')
-    expect(issueCount.textContent).toContain('3')
+    expect(issueCount.textContent).toContain('3 issues')
     expect(links.length).toEqual(2)
   })
 })

From f1c3fb1b392c9edb3a52f14db0f1e19fddd031e0 Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 17:04:36 +0100
Subject: [PATCH 57/65] Move the badge outside of the link

---
 src/assets/scss/components/_dataset-navigation.scss | 7 -------
 src/views/components/dataset-navigation.html        | 9 +++++++--
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/assets/scss/components/_dataset-navigation.scss b/src/assets/scss/components/_dataset-navigation.scss
index 68c77caf..7dea91d3 100644
--- a/src/assets/scss/components/_dataset-navigation.scss
+++ b/src/assets/scss/components/_dataset-navigation.scss
@@ -14,10 +14,3 @@ $badgeBgColour: govuk-colour('red');
   font-size: 1rem;
   margin-left: 5px;
 }
-
-/* Hides text underline decoration for badge */
-.app-c-dataset-navigation .govuk-service-navigation__link:hover .app-c-dataset-navigation__notification-badge {
-  text-decoration: underline;
-  text-decoration-color: $badgeBgColour;
-  text-decoration-thickness: $govuk-link-hover-underline-thickness;
-}
diff --git a/src/views/components/dataset-navigation.html b/src/views/components/dataset-navigation.html
index 9cd05f72..4624d559 100644
--- a/src/views/components/dataset-navigation.html
+++ b/src/views/components/dataset-navigation.html
@@ -10,6 +10,12 @@
       </span>
     {% endif %}
   {% endset %}
+  {% set taskLink %}
+    <a href="/organisations/{{ organisation.organisation }}/{{ dataset.dataset }}" class="govuk-service-navigation__link">
+      Task list
+    </a>
+    {{ badge | safe }}
+  {% endset %}
 
   {{ govukServiceNavigation({
     classes: "app-c-dataset-navigation govuk-!-margin-bottom-7",
@@ -20,8 +26,7 @@
         active: params.active === "dataset-overview"
       },
       {
-        href: "/organisations/" + organisation.organisation + "/" + dataset.dataset,
-        html: "Task list " + badge,
+        html: taskLink,
         active: params.active === "task-list"
       }
     ]

From f7eebc8c28ffc785f45f26467f774e43dbbc8e1b Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Wed, 25 Sep 2024 17:10:13 +0100
Subject: [PATCH 58/65] fix tests

---
 test/unit/datasetTaskListPage.test.js                 | 2 +-
 test/unit/lpaOverviewPage.test.js                     | 6 ++----
 test/unit/views/components/dataset-navigation.test.js | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js
index e153e53f..cffa7b6e 100644
--- a/test/unit/datasetTaskListPage.test.js
+++ b/test/unit/datasetTaskListPage.test.js
@@ -86,7 +86,7 @@ describe('Dataset Task List Page', () => {
   it('Renders the dataset navigation links correctly', () => {
     const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
     const activeLink = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
-    const issueCount = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .govuk-tag')
+    const issueCount = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .app-c-dataset-navigation__notification-badge')
 
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(links.length).toEqual(2)
diff --git a/test/unit/lpaOverviewPage.test.js b/test/unit/lpaOverviewPage.test.js
index 665cf65d..d3f6c8c2 100644
--- a/test/unit/lpaOverviewPage.test.js
+++ b/test/unit/lpaOverviewPage.test.js
@@ -164,12 +164,10 @@ describe('LPA Overview Page', () => {
     it(`Renders the correct link on each dataset card for dataset='${dataset.slug}'`, () => {
       const expectedLink = datasetCards[i].querySelector('.govuk-task-list__link').href
 
-      if (dataset.status === 'Live') {
-        expect(expectedLink).toEqual(`/organisations/mock-org/${dataset.slug}/overview`)
-      } else if (dataset.status === 'Not submitted') {
+      if (dataset.status === 'Not submitted') {
         expect(expectedLink).toEqual(`/organisations/mock-org/${dataset.slug}/get-started`)
       } else {
-        expect(expectedLink).toEqual(`/organisations/mock-org/${dataset.slug}`)
+        expect(expectedLink).toEqual(`/organisations/mock-org/${dataset.slug}/overview`)
       }
 
       const link = datasetCards[i].querySelector('.govuk-link')
diff --git a/test/unit/views/components/dataset-navigation.test.js b/test/unit/views/components/dataset-navigation.test.js
index 3332479f..e8575514 100644
--- a/test/unit/views/components/dataset-navigation.test.js
+++ b/test/unit/views/components/dataset-navigation.test.js
@@ -33,7 +33,7 @@ describe('Dataset Navigation component', () => {
   it('Renders the dataset navigation links correctly', () => {
     const links = document.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
     const activeLink = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
-    const issueCount = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .govuk-tag')
+    const issueCount = document.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .app-c-dataset-navigation__notification-badge')
 
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(activeLink.textContent).toContain('Dataset overview')
@@ -83,7 +83,7 @@ describe('Dataset Navigation component', () => {
     const documentWithIssues = domWithIssues.window.document
     const links = documentWithIssues.querySelectorAll('.app-c-dataset-navigation .govuk-service-navigation__link')
     const activeLink = documentWithIssues.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active')
-    const issueCount = documentWithIssues.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .govuk-tag')
+    const issueCount = documentWithIssues.querySelector('.app-c-dataset-navigation .govuk-service-navigation__item.govuk-service-navigation__item--active .app-c-dataset-navigation__notification-badge')
 
     expect(document.querySelector('.app-c-dataset-navigation')).not.toBeNull()
     expect(activeLink.textContent).toContain('Task list')

From 341b05354718889c0466ff2a7d240b2dc245556f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 26 Sep 2024 11:43:49 +0100
Subject: [PATCH 59/65] fixed test around date formatting

---
 src/views/organisations/dataset-overview.html         |  2 +-
 .../unit/views/organisations/dataset-overview.test.js | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html
index 44748c37..6a462d78 100644
--- a/src/views/organisations/dataset-overview.html
+++ b/src/views/organisations/dataset-overview.html
@@ -112,7 +112,7 @@
         classes: 'app-inset-text---error'
       },
       value: {
-        html: endpoint.lastAccessed + '<br><p class="govuk-error-message">There was a '+endpoint.error.code+' error accessing the data URL</p> <p class="app-inset-text__error"></p>'
+        html: (endpoint.lastAccessed | govukDateTime) + '<br><p class="govuk-error-message">There was a '+endpoint.error.code+' error accessing the data URL</p> <p class="app-inset-text__error"></p>'
       }
     } %}
   {% else %}
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index 98ab24c1..3c7f4749 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -4,6 +4,9 @@ import { runGenericPageTests } from '../../generic-page.js'
 import { stripWhitespace } from '../../../utils/stripWhiteSpace.js'
 import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
 
+import xGovFilters from '@x-govuk/govuk-prototype-filters'
+const { govukDateTime } = xGovFilters
+
 describe('Dataset Overview Page', () => {
   const params = {
     organisation: {
@@ -74,12 +77,12 @@ describe('Dataset Overview Page', () => {
     expect(summaryListValues[3].textContent.trim()).toEqual('Open Government Licence')
     expect(summaryListValues[4].textContent).toContain(params.stats.endpoints[0].endpoint)
     expect(summaryListValues[5].textContent).toContain(params.stats.endpoints[0].documentation_url)
-    expect(summaryListValues[6].textContent).toContain(params.stats.endpoints[0].lastAccessed)
-    expect(summaryListValues[7].textContent).toContain(params.stats.endpoints[0].lastUpdated)
+    expect(summaryListValues[6].textContent).toContain(govukDateTime(params.stats.endpoints[0].lastAccessed))
+    expect(summaryListValues[7].textContent).toContain(govukDateTime(params.stats.endpoints[0].lastUpdated))
     expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].endpoint)
-    expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].lastAccessed)
+    expect(summaryListValues[9].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastAccessed))
     expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].error.code)
-    expect(summaryListValues[10].textContent).toContain(params.stats.endpoints[1].lastUpdated)
+    expect(summaryListValues[10].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastUpdated))
   })
 
   it('Renders breadcrumbs correctly', () => {

From a6acb433615db4bde388bc85bb3c3b71d290318c Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 26 Sep 2024 16:43:15 +0100
Subject: [PATCH 60/65] change to use correct middleware methods

---
 src/middleware/common.middleware.js          |   2 +-
 src/middleware/datasetOverview.middleware.js | 121 +++++++++++++++--
 src/routes/schemas.js                        |  17 +--
 src/services/DatasetService.js               | 134 -------------------
 4 files changed, 112 insertions(+), 162 deletions(-)
 delete mode 100644 src/services/DatasetService.js

diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js
index b3945d18..97e480ec 100644
--- a/src/middleware/common.middleware.js
+++ b/src/middleware/common.middleware.js
@@ -27,7 +27,7 @@ export const logPageError = (err, req, res, next) => {
 
 export const fetchDatasetInfo = fetchOne({
   query: ({ params }) => {
-    return `SELECT name, dataset FROM dataset WHERE dataset = '${params.dataset}'`
+    return `SELECT name, dataset, collection FROM dataset WHERE dataset = '${params.dataset}'`
   },
   result: 'dataset'
 })
diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index 35be7fdb..ed20b221 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -1,23 +1,109 @@
 import { fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, isResourceAccessible, isResourceIdInParams, logPageError, takeResourceIdFromParams } from './common.middleware.js'
-import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
-import { getDatasetStats } from '../services/DatasetService.js'
+import { fetchOne, fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
+import performanceDbApi from '../services/performanceDbApi.js'
+import json5 from 'json5'
 
-const fetchDatasetStats = async (req, res, next) => {
-  req.stats = await getDatasetStats({
-    lpa: req.params.lpa,
-    dataset: req.params.dataset,
-    organisation: req.orgInfo
+const fetchColumnSummary = fetchMany({
+  query: ({ params }) => `select * from column_field_summary
+    where resource != ''
+    and pipeline = '${params.dataset}'
+    AND organisation = '${params.lpa}'
+    limit 1000`,
+  result: 'columnSummary',
+  dataset: () => 'performance'
+})
+
+const fetchSpecification = fetchOne({
+  query: ({ req }) => `select * from specification WHERE specification = '${req.dataset.collection}'`,
+  result: 'specification'
+})
+
+const pullOutDatasetSpecification = (req, res, next) => {
+  const { specification } = req
+  const collectionSpecifications = json5.parse(specification.json)
+  const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
+  req.specification = datasetSpecification
+  next()
+}
+
+const fetchSources = fetchMany({
+  query: ({ params }) => `
+    select rhe.endpoint, rhe.endpoint_url, rhe.status, rhe.exception, rhe.resource, rhe.latest_log_entry_date, rhe.endpoint_entry_date, rhe.endpoint_end_date, rhe.resource_start_date, rhe.resource_end_date, s.documentation_url
+    from reporting_historic_endpoints rhe
+    LEFT JOIN source s ON rhe.endpoint = s.endpoint
+    where REPLACE(rhe.organisation, '-eng', '') = '${params.lpa}' and rhe.pipeline = '${params.dataset}'
+    AND (rhe.resource_end_date >= current_timestamp OR rhe.resource_end_date is null)
+  `,
+  result: 'sources'
+})
+
+const fetchEntityCount = fetchOne({
+  query: ({ req }) => performanceDbApi.entityCountQuery(req.orgInfo.entity),
+  result: 'entityCount',
+  dataset: FetchOptions.fromParams
+})
+
+const prepareDatasetOverviewTemplateParams = (req, res, next) => {
+  const { orgInfo, specification, columnSummary, entityCount, sources, dataset } = req
+
+  const matchingFields = columnSummary[0].matching_field?.split(',') ?? []
+  const nonMatchingFields = columnSummary[0].non_matching_field?.split(',') ?? []
+  const allFields = [...matchingFields, ...nonMatchingFields]
+
+  const numberOfFieldsSupplied = specification.fields.map(field => field.field).reduce((acc, current) => {
+    return allFields.includes(current) ? acc + 1 : acc
+  }, 0)
+
+  const numberOfFieldsMatched = specification.fields.map(field => field.field).reduce((acc, current) => {
+    return matchingFields.includes(current) ? acc + 1 : acc
+  }, 0)
+
+  const numberOfExpectedFields = specification.fields.length
+
+  // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
+  const endpoints = sources.sort((a, b) => {
+    if (a.status >= 200 && a.status < 300) return -1
+    if (b.status >= 200 && b.status < 300) return 1
+    return 0
+  }).map((source, index) => {
+    let error
+
+    if (parseInt(source.status) < 200 || parseInt(source.status) >= 300) {
+      error = {
+        code: parseInt(source.status),
+        exception: source.exception
+      }
+    }
+
+    return {
+      name: `Data Url ${index}`,
+      endpoint: source.endpoint_url,
+      documentation_url: source.documentation_url,
+      lastAccessed: source.latest_log_entry_date,
+      lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
+      error
+    }
   })
+
+  req.templateParams = {
+    organisation: orgInfo,
+    dataset,
+    stats: {
+      numberOfFieldsSupplied: numberOfFieldsSupplied ?? 0,
+      numberOfFieldsMatched: numberOfFieldsMatched ?? 0,
+      numberOfExpectedFields: numberOfExpectedFields ?? 0,
+      numberOfRecords: entityCount.entity_count,
+      endpoints
+    }
+  }
+
   next()
 }
 
 const getDatasetOverview = renderTemplate(
   {
-    templateParams (req) {
-      const { orgInfo: organisation, dataset, stats, issues } = req
-      return { organisation, dataset, stats, issueCount: issues.length }
-    },
+    templateParams: (req) => req.templateParams,
     template: 'organisations/dataset-overview.html',
     handlerName: 'datasetOverview'
   }
@@ -28,10 +114,17 @@ export default [
     fetchOrgInfo,
     fetchDatasetInfo
   ]),
-  fetchResourceStatus,
-  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  parallel([
+    fetchColumnSummary,
+    fetchResourceStatus,
+    fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams)
+  ]),
   fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
-  fetchDatasetStats,
+  fetchSpecification,
+  pullOutDatasetSpecification,
+  fetchSources,
+  fetchEntityCount,
+  prepareDatasetOverviewTemplateParams,
   getDatasetOverview,
   logPageError
 ]
diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 38e6f443..62acf49c 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -39,7 +39,7 @@ const datasetStatusEnum = {
 }
 
 const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
-const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString })
+const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
 
 export const OrgOverviewPage = v.strictObject({
   organisation: OrgField,
@@ -69,10 +69,7 @@ export const OrgGetStarted = v.strictObject({
 
 export const OrgDatasetOverview = v.strictObject({
   organisation: OrgField,
-  dataset: v.strictObject({
-    name: NonEmptyString,
-    dataset: NonEmptyString
-  }),
+  dataset: DatasetNameField,
   stats: v.strictObject({
     numberOfRecords: v.integer(),
     numberOfFieldsSupplied: v.integer(),
@@ -112,10 +109,7 @@ export const OrgDatasetTaskList = v.strictObject({
 
 export const OrgEndpointError = v.strictObject({
   organisation: OrgField,
-  dataset: v.object({
-    name: NonEmptyString,
-    dataset: NonEmptyString
-  }),
+  dataset: DatasetNameField,
   errorData: v.strictObject({
     endpoint_url: v.url(),
     http_status: v.integer(),
@@ -191,10 +185,7 @@ export const ChooseDataset = v.strictObject({
 
 export const DatasetDetails = v.strictObject({
   organisation: OrgField,
-  dataset: v.strictObject({
-    name: NonEmptyString,
-    dataset: NonEmptyString
-  }),
+  dataset: DatasetNameField,
   values: v.strictObject({
     dataset: NonEmptyString
   }),
diff --git a/src/services/DatasetService.js b/src/services/DatasetService.js
deleted file mode 100644
index 51edc68f..00000000
--- a/src/services/DatasetService.js
+++ /dev/null
@@ -1,134 +0,0 @@
-import datasette from './datasette.js'
-import logger from '../utils/logger.js'
-import { types } from '../utils/logging.js'
-import JSON5 from 'json5' // need to use this as spec is written with single quotes. need to pass this onto data designers team to fix
-import performanceDbApi from './performanceDbApi.js'
-
-async function getColumnSummary (dataset, lpa) {
-  const sql = `select * from column_field_summary
-    where resource != ''
-    and pipeline = '${dataset}'
-    AND organisation = '${lpa}'
-    limit 1000`
-
-  const { formattedData } = await datasette.runQuery(sql, 'performance')
-
-  return formattedData
-}
-
-async function getFieldStats (lpa, dataset) {
-  const columnSummary = await datasetService.getColumnSummary(dataset, lpa)
-  const specifications = await datasetService.getSpecifications()
-  if (!(dataset in specifications)) {
-    logger.warn(`services/datasetService.getFieldStats(): cannot find specification for dataset: ${dataset}`, { type: types.App, dataset })
-    return null
-  }
-
-  const datasetSpecification = specifications[dataset]
-
-  const matchingFields = columnSummary[0].matching_field.split(',')
-  const nonMatchingFields = columnSummary[0].non_matching_field.split(',')
-  const allFields = [...matchingFields, ...nonMatchingFields]
-
-  const numberOfFieldsSupplied = datasetSpecification.fields.map(field => field.field).reduce((acc, current) => {
-    return allFields.includes(current) ? acc + 1 : acc
-  }, 0)
-
-  const numberOfFieldsMatched = datasetSpecification.fields.map(field => field.field).reduce((acc, current) => {
-    return matchingFields.includes(current) ? acc + 1 : acc
-  }, 0)
-
-  const numberOfExpectedFields = datasetSpecification.fields.length
-
-  return {
-    numberOfFieldsSupplied,
-    numberOfFieldsMatched,
-    numberOfExpectedFields
-  }
-}
-
-async function getSpecifications () {
-  const sql = 'select * from specification order by specification'
-  const result = await datasette.runQuery(sql)
-
-  const resultWithParsedJson = result.formattedData.filter(dataset => dataset.datasets !== '').map((dataset) => {
-    const { json, ...rest } = dataset
-    const formattedJson = JSON5.parse(json)
-    return {
-      ...rest,
-      json: formattedJson
-    }
-  })
-
-  const specifications = resultWithParsedJson.reduce((accumulator, current) => {
-    current.json.forEach((spec) => {
-      accumulator[spec.dataset] = spec
-    })
-    return accumulator
-  }, {})
-
-  return specifications
-}
-
-export async function getSources (lpa, dataset) {
-  const sql = `
-    select rhe.endpoint, rhe.endpoint_url, rhe.status, rhe.exception, rhe.resource, rhe.latest_log_entry_date, rhe.endpoint_entry_date, rhe.endpoint_end_date, rhe.resource_start_date, rhe.resource_end_date, s.documentation_url
-    from reporting_historic_endpoints rhe
-    LEFT JOIN source s ON rhe.endpoint = s.endpoint
-    where REPLACE(rhe.organisation, '-eng', '') = '${lpa}' and rhe.pipeline = '${dataset}'
-    AND (rhe.resource_end_date >= current_timestamp OR rhe.resource_end_date is null)
-  `
-
-  const { formattedData } = await datasette.runQuery(sql)
-
-  return formattedData
-}
-
-export async function getDatasetStats ({ dataset, lpa, organisation }) {
-  const stats = await datasetService.getFieldStats(lpa, dataset)
-
-  const sources = await datasetService.getSources(lpa, dataset)
-
-  // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
-  const endpoints = sources.sort((a, b) => {
-    if (a.status >= 200 && a.status < 300) return -1
-    if (b.status >= 200 && b.status < 300) return 1
-    return 0
-  }).map((source, index) => {
-    let error
-
-    if (parseInt(source.status) < 200 || parseInt(source.status) >= 300) {
-      error = {
-        code: parseInt(source.status),
-        exception: source.exception
-      }
-    }
-
-    return {
-      name: `Data Url ${index}`,
-      endpoint: source.endpoint_url,
-      documentation_url: source.documentation_url,
-      lastAccessed: source.latest_log_entry_date,
-      lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
-      error
-    }
-  })
-
-  const numberOfRecords = await performanceDbApi.getEntityCount(organisation.entity, dataset)
-
-  return {
-    numberOfFieldsSupplied: stats?.numberOfFieldsSupplied ?? 0,
-    numberOfFieldsMatched: stats?.numberOfFieldsMatched ?? 0,
-    numberOfExpectedFields: stats?.numberOfExpectedFields ?? 0,
-    numberOfRecords,
-    endpoints
-  }
-}
-
-export const datasetService = {
-  getColumnSummary,
-  getFieldStats,
-  getSpecifications,
-  getSources,
-  getDatasetStats
-}

From 8bf0efb17e10db8775253706f80efe19a9cf486e Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Thu, 26 Sep 2024 17:01:14 +0100
Subject: [PATCH 61/65] remove unused file

---
 src/middleware/issuesTable.middleware.js | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 src/middleware/issuesTable.middleware.js

diff --git a/src/middleware/issuesTable.middleware.js b/src/middleware/issuesTable.middleware.js
deleted file mode 100644
index a3a39b34..00000000
--- a/src/middleware/issuesTable.middleware.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// export default [
-//   validateIssueTableQueryParams,
-//   parallel([
-//     fetchOrgInfo,
-//     fetchDatasetInfo
-//   ]),
-//   fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
-//   fetchEntitiesWithIssues,
-//   prepareIssueTableParams,
-//   getIssuesTable,
-//   logPageError
-// ]

From d9e98c7a1c387f5c7aef87f32ee64e77befaa77e Mon Sep 17 00:00:00 2001
From: Dilwoar Hussain <dilwoar.hussain@gmail.com>
Date: Fri, 27 Sep 2024 10:38:37 +0100
Subject: [PATCH 62/65] Allow files with undefined `content-length` headers

This change allows undefined `content-length` headers to be accepted.

We cannot rely on the content length header to always return or be accurate. Therefore, it's best this check is just a soft check and let the API enforce restrictions on uploaded dataset sizes.

https://github.com/digital-land/submit/issues/450
---
 src/controllers/submitUrlController.js | 7 +++++++
 test/unit/submitUrlController.test.js  | 4 ++++
 2 files changed, 11 insertions(+)

diff --git a/src/controllers/submitUrlController.js b/src/controllers/submitUrlController.js
index a7626a19..7493f1d1 100644
--- a/src/controllers/submitUrlController.js
+++ b/src/controllers/submitUrlController.js
@@ -116,7 +116,14 @@ class SubmitUrlController extends UploadController {
    */
   static urlResponseIsNotTooLarge (response) {
     const contentLength = (response?.headers ?? {})['content-length']
+
     try {
+      // if the content length header is not provided, return true
+      if (!contentLength) {
+        console.warn(`urlResponseIsNotTooLarge(): response.config.url=${response.config.url}`, { type: types.App, errorMessage: 'Content-Length header not provided' })
+        return true
+      }
+
       return contentLength <= config.validations.maxFileSize
     } catch (err) {
       console.warn('urlResponseIsNotTooLarge()', { type: types.App, errorMessage: err.message, errorStack: err.stack, contentLength })
diff --git a/test/unit/submitUrlController.test.js b/test/unit/submitUrlController.test.js
index 2c7f3df4..8322ab63 100644
--- a/test/unit/submitUrlController.test.js
+++ b/test/unit/submitUrlController.test.js
@@ -165,6 +165,10 @@ describe('SubmitUrlController', async () => {
       it('should return false for URLs with a response larger than the max file size', async () => {
         expect(SubmitUrlController.urlResponseIsNotTooLarge({ headers: { 'content-length': config.validations.maxFileSize + 21 } })).toBe(false)
       })
+
+      it('should return true for URLs without a content length header', async () => {
+        expect(SubmitUrlController.urlResponseIsNotTooLarge({ headers: { } })).toBe(true)
+      })
     })
 
     describe('urlExists', () => {

From a7ae56341cd804b824a6b0af73b2fca70cf22fa0 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 27 Sep 2024 14:22:36 +0100
Subject: [PATCH 63/65] fix tests

---
 src/middleware/datasetOverview.middleware.js          |  2 +-
 src/middleware/middleware.builders.js                 |  5 ++++-
 test/unit/dataset-details.test.js                     |  3 ++-
 test/unit/middleware/getStarted.middleware.test.js    |  2 +-
 test/unit/middleware/issueDetails.middleware.test.js  | 11 +++++++----
 .../unit/views/organisations/dataset-overview.test.js |  6 ++++--
 6 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index ed20b221..bd33bc43 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -11,7 +11,7 @@ const fetchColumnSummary = fetchMany({
     AND organisation = '${params.lpa}'
     limit 1000`,
   result: 'columnSummary',
-  dataset: () => 'performance'
+  dataset: FetchOptions.performanceDb
 })
 
 const fetchSpecification = fetchOne({
diff --git a/src/middleware/middleware.builders.js b/src/middleware/middleware.builders.js
index 3340afd7..f641fc35 100644
--- a/src/middleware/middleware.builders.js
+++ b/src/middleware/middleware.builders.js
@@ -18,7 +18,8 @@ export const FetchOptions = {
   /**
      * Use 'dataset' from requets params.
      */
-  fromParams: Symbol('from-params')
+  fromParams: Symbol('from-params'),
+  performanceDb: Symbol('performance-database')
 }
 
 const datasetOverride = (val, req) => {
@@ -28,6 +29,8 @@ const datasetOverride = (val, req) => {
   if (val === FetchOptions.fromParams) {
     console.assert(req.params.dataset, 'no "dataset" in request params')
     return req.params.dataset
+  } else if (val === FetchOptions.performanceDb) {
+    return 'performance'
   } else {
     return val(req)
   }
diff --git a/test/unit/dataset-details.test.js b/test/unit/dataset-details.test.js
index add2d819..5d802230 100644
--- a/test/unit/dataset-details.test.js
+++ b/test/unit/dataset-details.test.js
@@ -46,7 +46,8 @@ describe('dataset details View', () => {
     },
     dataset: {
       name: 'mock dataset',
-      dataset: 'mock-dataset'
+      dataset: 'mock-dataset',
+      collection: 'mock-collection'
     },
     values: {
       dataset: 'mockDataset'
diff --git a/test/unit/middleware/getStarted.middleware.test.js b/test/unit/middleware/getStarted.middleware.test.js
index 5a31ec96..9edb1eef 100644
--- a/test/unit/middleware/getStarted.middleware.test.js
+++ b/test/unit/middleware/getStarted.middleware.test.js
@@ -7,7 +7,7 @@ describe('get-started', () => {
       { name: 'Example LPA', organisation: 'LPA' }
     ]
   }
-  const exampleDataset = { name: 'Example Dataset', dataset: 'example-dataset' }
+  const exampleDataset = { name: 'Example Dataset', dataset: 'example-dataset', collection: 'example-collection' }
 
   it('should render the get-started template with the correct params', async () => {
     const req = {
diff --git a/test/unit/middleware/issueDetails.middleware.test.js b/test/unit/middleware/issueDetails.middleware.test.js
index 9e79e561..befa4b24 100644
--- a/test/unit/middleware/issueDetails.middleware.test.js
+++ b/test/unit/middleware/issueDetails.middleware.test.js
@@ -8,7 +8,7 @@ vi.mock('../../../src/services/performanceDbApi.js')
 
 describe('issueDetails.middleware.js', () => {
   const orgInfo = { name: 'mock lpa', organisation: 'ORG' }
-  const dataset = { name: 'mock dataset', dataset: 'mock-dataset' }
+  const dataset = { name: 'mock dataset', dataset: 'mock-dataset', collection: 'mock-collection' }
   const entryData = [
     {
       field: 'start-date',
@@ -76,7 +76,8 @@ describe('issueDetails.middleware.js', () => {
         },
         dataset: {
           name: 'mock dataset',
-          dataset: 'mock-dataset'
+          dataset: 'mock-dataset',
+          collection: 'mock-collection'
         },
         errorHeading: 'mockMessageFor: 0',
         issueItems: [
@@ -175,7 +176,8 @@ describe('issueDetails.middleware.js', () => {
         },
         dataset: {
           name: 'mock dataset',
-          dataset: 'mock-dataset'
+          dataset: 'mock-dataset',
+          collection: 'mock-collection'
         },
         errorHeading: 'mockMessageFor: 0',
         issueItems: [
@@ -231,7 +233,8 @@ describe('issueDetails.middleware.js', () => {
           },
           dataset: {
             name: 'mock dataset',
-            dataset: 'mock-dataset'
+            dataset: 'mock-dataset',
+            collection: 'mock-collection'
           },
           errorHeading: 'mockMessageFor: 0',
           issueItems: [
diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js
index 3c7f4749..bbb5c1bb 100644
--- a/test/unit/views/organisations/dataset-overview.test.js
+++ b/test/unit/views/organisations/dataset-overview.test.js
@@ -15,7 +15,8 @@ describe('Dataset Overview Page', () => {
     },
     dataset: {
       dataset: 'world-heritage-site-buffer-zone',
-      name: 'World heritage site buffer zone'
+      name: 'World heritage site buffer zone',
+      collection: 'world-heritage-site'
     },
     stats: {
       numberOfRecords: 10,
@@ -103,7 +104,8 @@ describe('Dataset Overview Page', () => {
       ...params,
       dataset: {
         dataset: 'article-4-direction-area',
-        name: 'Article 4 direction area'
+        name: 'Article 4 direction area',
+        collection: 'article-4-direction'
       }
     }
     const htmlWithGeometries = stripWhitespace(nunjucks.render('organisations/dataset-overview.html', paramsWithGeometries))

From aaeb916fde0e936073e005426a4ba191994fd1d2 Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 27 Sep 2024 15:10:28 +0100
Subject: [PATCH 64/65] move tests over to test new datasetOverview middleware

---
 src/middleware/datasetOverview.middleware.js  |   4 +-
 test/unit/http-errorPage.test.js              |   3 +-
 .../datasetOverview.middleware.test.js        |  63 ++++++
 test/unit/services/DatasetService.test.js     | 186 ------------------
 4 files changed, 67 insertions(+), 189 deletions(-)
 create mode 100644 test/unit/middleware/datasetOverview.middleware.test.js
 delete mode 100644 test/unit/services/DatasetService.test.js

diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js
index bd33bc43..10e8fe81 100644
--- a/src/middleware/datasetOverview.middleware.js
+++ b/src/middleware/datasetOverview.middleware.js
@@ -19,7 +19,7 @@ const fetchSpecification = fetchOne({
   result: 'specification'
 })
 
-const pullOutDatasetSpecification = (req, res, next) => {
+export const pullOutDatasetSpecification = (req, res, next) => {
   const { specification } = req
   const collectionSpecifications = json5.parse(specification.json)
   const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
@@ -44,7 +44,7 @@ const fetchEntityCount = fetchOne({
   dataset: FetchOptions.fromParams
 })
 
-const prepareDatasetOverviewTemplateParams = (req, res, next) => {
+export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
   const { orgInfo, specification, columnSummary, entityCount, sources, dataset } = req
 
   const matchingFields = columnSummary[0].matching_field?.split(',') ?? []
diff --git a/test/unit/http-errorPage.test.js b/test/unit/http-errorPage.test.js
index d908019c..19536ea9 100644
--- a/test/unit/http-errorPage.test.js
+++ b/test/unit/http-errorPage.test.js
@@ -13,7 +13,8 @@ describe('http-error.html', () => {
     },
     dataset: {
       name: 'mock-dataset',
-      dataset: 'mock-dataset'
+      dataset: 'mock-dataset',
+      collection: 'mock-collection'
     },
     errorData: {
       endpoint_url: 'https://example.com/data-url',
diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js
new file mode 100644
index 00000000..4807205b
--- /dev/null
+++ b/test/unit/middleware/datasetOverview.middleware.test.js
@@ -0,0 +1,63 @@
+import { describe, it, expect } from 'vitest'
+import { prepareDatasetOverviewTemplateParams, pullOutDatasetSpecification } from '../../../src/middleware/datasetOverview.middleware.js'
+
+describe('Dataset Overview Middleware', () => {
+  const req = {
+    params: {
+      lpa: 'mock-lpa',
+      dataset: 'mock-dataset'
+    },
+    dataset: {
+      name: 'mock dataset',
+      dataset: 'mock-dataset',
+      collection: 'mock-collection'
+    }
+  }
+  const res = {}
+
+  describe('pullOutDatasetSpecification', () => {
+    it('', () => {
+      const reqWithSpecification = {
+        ...req,
+        specification: {
+          json: JSON.stringify([
+            { dataset: 'mock-dataset', foo: 'bar' }
+          ])
+        }
+      }
+      pullOutDatasetSpecification(reqWithSpecification, res, () => {})
+      expect(reqWithSpecification.specification).toEqual({ dataset: 'mock-dataset', foo: 'bar' })
+    })
+  })
+
+  describe('prepareDatasetOverviewTemplateParams', () => {
+    it('should prepare template params for dataset overview', async () => {
+      const reqWithResults = {
+        ...req,
+        orgInfo: { name: 'mock-org' },
+        specification: { fields: [{ field: 'field1' }, { field: 'field2' }] },
+        columnSummary: [{ matching_field: 'field1', non_matching_field: 'field3' }],
+        entityCount: { entity_count: 10 },
+        sources: [
+          { endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: '200', endpoint_entry_date: 'LU1', latest_log_entry_date: 'LA1' },
+          { endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: '404', exception: 'exception', endpoint_entry_date: 'LU2', latest_log_entry_date: 'LA2' }
+        ]
+      }
+      prepareDatasetOverviewTemplateParams(reqWithResults, res, () => {})
+      expect(reqWithResults.templateParams).toEqual({
+        organisation: { name: 'mock-org' },
+        dataset: reqWithResults.dataset,
+        stats: {
+          numberOfFieldsSupplied: 1,
+          numberOfFieldsMatched: 1,
+          numberOfExpectedFields: 2,
+          numberOfRecords: 10,
+          endpoints: [
+            { name: 'Data Url 0', endpoint: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: 'LU1' },
+            { name: 'Data Url 1', endpoint: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: 'LU2' }
+          ]
+        }
+      })
+    })
+  })
+})
diff --git a/test/unit/services/DatasetService.test.js b/test/unit/services/DatasetService.test.js
deleted file mode 100644
index 05d5fcf2..00000000
--- a/test/unit/services/DatasetService.test.js
+++ /dev/null
@@ -1,186 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest'
-import { datasetService } from '../../../src/services/DatasetService.js'
-import datasette from '../../../src/services/datasette.js'
-import performanceDbApi from '../../../src/services/performanceDbApi.js'
-
-vi.mock('../../../src/services/datasette')
-vi.mock('../../../src/services/performanceDbApi.js')
-
-describe('DatasetService', () => {
-  describe('getSpecifications', () => {
-    beforeEach(() => {
-      datasette.runQuery.mockReturnValue({
-        formattedData: [
-          {
-            datasets: 'dataset1',
-            json: "[{'dataset': 'dataset1', 'key': 'value'}]"
-          },
-          {
-            datasets: 'dataset2;dataset3',
-            json: "[{'dataset': 'dataset2', 'key2': 'value2'},{'dataset': 'dataset3', 'key3': 'value3'}]"
-          },
-          {
-            datasets: 'dataset4',
-            json: "[{'dataset': 'dataset4', 'key4': 'value4'}]"
-          }
-        ]
-      })
-    })
-
-    const expectedSpecifications = {
-      dataset1: { dataset: 'dataset1', key: 'value' },
-      dataset2: { dataset: 'dataset2', key2: 'value2' },
-      dataset3: { dataset: 'dataset3', key3: 'value3' },
-      dataset4: { dataset: 'dataset4', key4: 'value4' }
-    }
-
-    it('returns specifications in the correct format', async () => {
-      const specifications = await datasetService.getSpecifications()
-      expect(specifications).toEqual(expectedSpecifications)
-    })
-  })
-
-  describe('getFieldStats', () => {
-    const columnSummary = [
-      {
-        matching_field: 'field1,field2',
-        non_matching_field: 'field3,field4'
-      }
-    ]
-
-    const specifications = [
-      {
-        datasets: 'dataset1',
-        json: '[{dataset: "dataset1", fields: [{field: "field1"},{field: "field3"}]}]'
-      }
-    ]
-
-    beforeEach(() => {
-      vi.mocked(datasette.runQuery).mockResolvedValueOnce({ formattedData: columnSummary })
-      vi.mocked(datasette.runQuery).mockResolvedValueOnce({ formattedData: specifications })
-    })
-
-    it('returns field stats', async () => {
-      const fieldStats = await datasetService.getFieldStats('lpa', 'dataset1')
-      expect(fieldStats).toEqual({
-        numberOfFieldsSupplied: 2,
-        numberOfFieldsMatched: 1,
-        numberOfExpectedFields: 2
-      })
-    })
-
-    it('returns null if dataset specification is missing', async () => {
-      const fieldStats = await datasetService.getFieldStats('lpa', 'nonExistentDataset')
-      expect(fieldStats).toBeNull()
-    })
-  })
-
-  describe('getDatasetStats', async () => {
-    const lpa = 'lpa-example'
-    const dataset = 'dataset-example'
-    const organisation = 'organisation-example'
-
-    const getFieldStatsSpy = vi.spyOn(datasetService, 'getFieldStats')
-    const getSourcesSpy = vi.spyOn(datasetService, 'getSources')
-
-    beforeEach(() => {
-      getFieldStatsSpy.mockImplementation(() => {
-        return {
-          numberOfFieldsSupplied: 2,
-          numberOfFieldsMatched: 1,
-          numberOfExpectedFields: 2
-        }
-      })
-      getSourcesSpy.mockImplementation(() => [
-        {
-          status: 200,
-          endpoint_url: 'https://example.com/endpoint1',
-          documentation_url: 'https://example.com/doc1',
-          latest_log_entry_date: '2023-02-20T14:30:00Z',
-          endpoint_entry_date: '2023-02-19T12:00:00Z'
-        },
-        {
-          status: 404,
-          endpoint_url: 'https://example.com/endpoint2',
-          documentation_url: 'https://example.com/doc2',
-          latest_log_entry_date: '2023-02-20T14:30:00Z',
-          endpoint_entry_date: '2023-02-19T12:00:00Z',
-          exception: 'Not Found'
-        }
-      ])
-      performanceDbApi.getEntityCount = vi.fn().mockResolvedValueOnce(100)
-    })
-
-    it('returns dataset stats with field stats and sources', async () => {
-      const datasetStats = await datasetService.getDatasetStats({ dataset, lpa, organisation })
-
-      expect(datasetStats).toEqual({
-        numberOfFieldsSupplied: 2,
-        numberOfFieldsMatched: 1,
-        numberOfExpectedFields: 2,
-        numberOfRecords: 100,
-        endpoints: [
-          {
-            name: 'Data Url 0',
-            endpoint: 'https://example.com/endpoint1',
-            documentation_url: 'https://example.com/doc1',
-            lastAccessed: '2023-02-20T14:30:00Z',
-            lastUpdated: '2023-02-19T12:00:00Z'
-          },
-          {
-            name: 'Data Url 1',
-            endpoint: 'https://example.com/endpoint2',
-            documentation_url: 'https://example.com/doc2',
-            lastAccessed: '2023-02-20T14:30:00Z',
-            lastUpdated: '2023-02-19T12:00:00Z',
-            error: {
-              code: 404,
-              exception: 'Not Found'
-            }
-          }
-        ]
-      })
-    })
-
-    it('returns 0s if getFieldStats returns null', async () => {
-      getFieldStatsSpy.mockResolvedValueOnce(null)
-      const datasetStats = await datasetService.getDatasetStats({ dataset, lpa, organisation })
-      expect(datasetStats).toEqual({
-        numberOfFieldsSupplied: 0,
-        numberOfFieldsMatched: 0,
-        numberOfExpectedFields: 0,
-        numberOfRecords: 100,
-        endpoints: [
-          {
-            name: 'Data Url 0',
-            endpoint: 'https://example.com/endpoint1',
-            documentation_url: 'https://example.com/doc1',
-            lastAccessed: '2023-02-20T14:30:00Z',
-            lastUpdated: '2023-02-19T12:00:00Z'
-          },
-          {
-            name: 'Data Url 1',
-            endpoint: 'https://example.com/endpoint2',
-            documentation_url: 'https://example.com/doc2',
-            lastAccessed: '2023-02-20T14:30:00Z',
-            lastUpdated: '2023-02-19T12:00:00Z',
-            error: {
-              code: 404,
-              exception: 'Not Found'
-            }
-          }
-        ]
-      })
-    })
-
-    it('throws an error if getFieldStats throws an error', async () => {
-      getFieldStatsSpy.mockRejectedValueOnce(new Error('Error fetching sources'))
-      await expect(datasetService.getDatasetStats({ dataset, lpa, organisation })).rejects.toThrowError('Error fetching sources')
-    })
-
-    it('throws an error if getSources throws an error', async () => {
-      getSourcesSpy.mockRejectedValueOnce(new Error('Error fetching sources'))
-      await expect(datasetService.getDatasetStats({ dataset, lpa, organisation })).rejects.toThrowError('Error fetching sources')
-    })
-  })
-})

From 44b0609c11ed9ea6d69c9184bf14a8ef6b02b27f Mon Sep 17 00:00:00 2001
From: George Goodall <ggoodall@scottlogic.com>
Date: Fri, 27 Sep 2024 15:19:44 +0100
Subject: [PATCH 65/65] update schema

---
 src/routes/schemas.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/routes/schemas.js b/src/routes/schemas.js
index 62acf49c..a9484b0a 100644
--- a/src/routes/schemas.js
+++ b/src/routes/schemas.js
@@ -103,7 +103,8 @@ export const OrgDatasetTaskList = v.strictObject({
   organisation: OrgField,
   dataset: v.strictObject({
     dataset: v.optional(NonEmptyString),
-    name: NonEmptyString
+    name: NonEmptyString,
+    collection: NonEmptyString
   })
 })