Merge pull request #451 from digital-land/407-dataset-details-summary…

…-table

407 dataset details summary table

package.json

@@ -76,6 +76,7 @@
     "hmpo-form-wizard": "^13.0.0",
     "hmpo-i18n": "^6.0.1",
     "js-yaml": "^4.1.0",
+    "json5": "^2.2.3",
     "lodash": "^4.17.21",
     "maplibre-gl": "^4.1.0",
     "multer": "^1.4.5-lts.1",

src/assets/scss/index.scss

@@ -108,4 +108,8 @@ $govuk-global-styles: true;
 code,
 code * {
   font-family: monospace;
-}
+}
+
+.padding-top {
+  padding-top: 40px;
+}

src/middleware/common.middleware.js

@@ -27,7 +27,7 @@ export const logPageError = (err, req, res, next) => {
 
 export const fetchDatasetInfo = fetchOne({
   query: ({ params }) => {
-    return `SELECT name, dataset FROM dataset WHERE dataset = '${params.dataset}'`
+    return `SELECT name, dataset, collection FROM dataset WHERE dataset = '${params.dataset}'`
   },
   result: 'dataset'
 })
@@ -64,7 +64,7 @@ export const fetchEntityCount = fetchOne({
 
 export const fetchOrgInfo = fetchOne({
   query: ({ params }) => {
-    return `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
+    return `SELECT name, organisation, entity, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'`
   },
   result: 'orgInfo'
 })

src/middleware/datasetOverview.middleware.js

@@ -1,20 +1,109 @@
 import { fetchDatasetInfo, fetchLatestResource, fetchLpaDatasetIssues, fetchOrgInfo, isResourceAccessible, isResourceIdInParams, logPageError, takeResourceIdFromParams } from './common.middleware.js'
-import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
-import { getDatasetStats } from '../services/DatasetService.js'
+import { fetchOne, fetchIf, fetchMany, parallel, renderTemplate, FetchOptions } from './middleware.builders.js'
 import { fetchResourceStatus } from './datasetTaskList.middleware.js'
+import performanceDbApi from '../services/performanceDbApi.js'
+import json5 from 'json5'
 
-const fetchDatasetStats = async (req, res, next) => {
-  req.stats = await getDatasetStats(req.params.dataset, req.params.lpa)
+const fetchColumnSummary = fetchMany({
+  query: ({ params }) => `select * from column_field_summary
+    where resource != ''
+    and pipeline = '${params.dataset}'
+    AND organisation = '${params.lpa}'
+    limit 1000`,
+  result: 'columnSummary',
+  dataset: FetchOptions.performanceDb
+})
+
+const fetchSpecification = fetchOne({
+  query: ({ req }) => `select * from specification WHERE specification = '${req.dataset.collection}'`,
+  result: 'specification'
+})
+
+export const pullOutDatasetSpecification = (req, res, next) => {
+  const { specification } = req
+  const collectionSpecifications = json5.parse(specification.json)
+  const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
+  req.specification = datasetSpecification
+  next()
+}
+
+const fetchSources = fetchMany({
+  query: ({ params }) => `
+    select rhe.endpoint, rhe.endpoint_url, rhe.status, rhe.exception, rhe.resource, rhe.latest_log_entry_date, rhe.endpoint_entry_date, rhe.endpoint_end_date, rhe.resource_start_date, rhe.resource_end_date, s.documentation_url
+    from reporting_historic_endpoints rhe
+    LEFT JOIN source s ON rhe.endpoint = s.endpoint
+    where REPLACE(rhe.organisation, '-eng', '') = '${params.lpa}' and rhe.pipeline = '${params.dataset}'
+    AND (rhe.resource_end_date >= current_timestamp OR rhe.resource_end_date is null)
+  `,
+  result: 'sources'
+})
+
+const fetchEntityCount = fetchOne({
+  query: ({ req }) => performanceDbApi.entityCountQuery(req.orgInfo.entity),
+  result: 'entityCount',
+  dataset: FetchOptions.fromParams
+})
+
+export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
+  const { orgInfo, specification, columnSummary, entityCount, sources, dataset } = req
+
+  const matchingFields = columnSummary[0].matching_field?.split(',') ?? []
+  const nonMatchingFields = columnSummary[0].non_matching_field?.split(',') ?? []
+  const allFields = [...matchingFields, ...nonMatchingFields]
+
+  const numberOfFieldsSupplied = specification.fields.map(field => field.field).reduce((acc, current) => {
+    return allFields.includes(current) ? acc + 1 : acc
+  }, 0)
+
+  const numberOfFieldsMatched = specification.fields.map(field => field.field).reduce((acc, current) => {
+    return matchingFields.includes(current) ? acc + 1 : acc
+  }, 0)
+
+  const numberOfExpectedFields = specification.fields.length
+
+  // I'm pretty sure every endpoint has a separate documentation-url, but this isn't currently represented in the performance db. need to double check this and update if so
+  const endpoints = sources.sort((a, b) => {
+    if (a.status >= 200 && a.status < 300) return -1
+    if (b.status >= 200 && b.status < 300) return 1
+    return 0
+  }).map((source, index) => {
+    let error
+
+    if (parseInt(source.status) < 200 || parseInt(source.status) >= 300) {
+      error = {
+        code: parseInt(source.status),
+        exception: source.exception
+      }
+    }
+
+    return {
+      name: `Data Url ${index}`,
+      endpoint: source.endpoint_url,
+      documentation_url: source.documentation_url,
+      lastAccessed: source.latest_log_entry_date,
+      lastUpdated: source.endpoint_entry_date, // not sure if this is the lastupdated
+      error
+    }
+  })
+
+  req.templateParams = {
+    organisation: orgInfo,
+    dataset,
+    stats: {
+      numberOfFieldsSupplied: numberOfFieldsSupplied ?? 0,
+      numberOfFieldsMatched: numberOfFieldsMatched ?? 0,
+      numberOfExpectedFields: numberOfExpectedFields ?? 0,
+      numberOfRecords: entityCount.entity_count,
+      endpoints
+    }
+  }
 
   next()
 }
 
 const getDatasetOverview = renderTemplate(
   {
-    templateParams (req) {
-      const { orgInfo: organisation, dataset, stats, issues } = req
-      return { organisation, dataset, stats, issueCount: issues.length }
-    },
+    templateParams: (req) => req.templateParams,
     template: 'organisations/dataset-overview.html',
     handlerName: 'datasetOverview'
   }
@@ -25,10 +114,17 @@ export default [
     fetchOrgInfo,
     fetchDatasetInfo
   ]),
-  fetchResourceStatus,
-  fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams),
+  parallel([
+    fetchColumnSummary,
+    fetchResourceStatus,
+    fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams)
+  ]),
   fetchIf(isResourceAccessible, fetchLpaDatasetIssues),
-  fetchDatasetStats,
+  fetchSpecification,
+  pullOutDatasetSpecification,
+  fetchSources,
+  fetchEntityCount,
+  prepareDatasetOverviewTemplateParams,
   getDatasetOverview,
   logPageError
 ]

src/middleware/issueDetails.middleware.js

@@ -86,8 +86,7 @@ async function fetchEntry (req, res, next) {
 
   // look at issue Entries and get the index of that entry - 1
 
-  const entityNum =
-      Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
+  const entityNum = Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number
 
   req.entryData = await performanceDbApi.getEntry(
     req.resource.resource,

src/middleware/middleware.builders.js

@@ -18,7 +18,8 @@ export const FetchOptions = {
   /**
      * Use 'dataset' from requets params.
      */
-  fromParams: Symbol('from-params')
+  fromParams: Symbol('from-params'),
+  performanceDb: Symbol('performance-database')
 }
 
 const datasetOverride = (val, req) => {
@@ -28,6 +29,8 @@ const datasetOverride = (val, req) => {
   if (val === FetchOptions.fromParams) {
     console.assert(req.params.dataset, 'no "dataset" in request params')
     return req.params.dataset
+  } else if (val === FetchOptions.performanceDb) {
+    return 'performance'
   } else {
     return val(req)
   }

src/routes/schemas.js

@@ -38,8 +38,8 @@ const datasetStatusEnum = {
   'Not submitted': 'Not submitted'
 }
 
-const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(NonEmptyString) })
-const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString })
+const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) })
+const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmptyString, collection: NonEmptyString })
 
 export const OrgOverviewPage = v.strictObject({
   organisation: OrgField,
@@ -69,15 +69,24 @@ export const OrgGetStarted = v.strictObject({
 
 export const OrgDatasetOverview = v.strictObject({
   organisation: OrgField,
-  dataset: v.strictObject({
-    name: NonEmptyString,
-    dataset: NonEmptyString
-  }),
+  dataset: DatasetNameField,
   stats: v.strictObject({
     numberOfRecords: v.integer(),
-    numberOfFieldsSupplied: v.integer()
-  }),
-  issueCount: v.integer()
+    numberOfFieldsSupplied: v.integer(),
+    numberOfFieldsMatched: v.integer(),
+    numberOfExpectedFields: v.integer(),
+    endpoints: v.array(v.strictObject({
+      name: v.string(),
+      documentation_url: v.optional(v.string()),
+      endpoint: v.string(),
+      lastAccessed: v.string(),
+      lastUpdated: v.string(),
+      error: v.optional(v.strictObject({
+        code: v.integer(),
+        exception: v.string()
+      }))
+    }))
+  })
 })
 
 export const OrgDatasetTaskList = v.strictObject({
@@ -94,16 +103,14 @@ export const OrgDatasetTaskList = v.strictObject({
   organisation: OrgField,
   dataset: v.strictObject({
     dataset: v.optional(NonEmptyString),
-    name: NonEmptyString
+    name: NonEmptyString,
+    collection: NonEmptyString
   })
 })
 
 export const OrgEndpointError = v.strictObject({
   organisation: OrgField,
-  dataset: v.object({
-    name: NonEmptyString,
-    dataset: NonEmptyString
-  }),
+  dataset: DatasetNameField,
   errorData: v.strictObject({
     endpoint_url: v.url(),
     http_status: v.integer(),
@@ -179,10 +186,7 @@ export const ChooseDataset = v.strictObject({
 
 export const DatasetDetails = v.strictObject({
   organisation: OrgField,
-  dataset: v.strictObject({
-    name: NonEmptyString,
-    dataset: NonEmptyString
-  }),
+  dataset: DatasetNameField,
   values: v.strictObject({
     dataset: NonEmptyString
   }),

src/services/performanceDbApi.js

@@ -383,23 +383,23 @@ ORDER BY
     return result.formattedData
   },
 
-  entityCountQuery (resource) {
+  entityCountQuery (orgEntity) {
     return /* sql */ `
-    select dataset, entity_count, resource
-    from dataset_resource
-    WHERE resource = '${resource}'
-  `
+      select count(entity) as entity_count 
+      from entity
+      WHERE organisation_entity = '${orgEntity}'
+    `
   },
 
   /**
-     * Retrieves the entity count for a given resource and dataset.
-     *
-     * @param {string} resource - The resource to retrieve the entity count for.
-     * @param {string} dataset - The dataset to retrieve the entity count from.
-     * @returns {Promise<number>} The entity count for the given resource and dataset.
-     */
-  async getEntityCount (resource, dataset) {
-    const query = this.entityCountQuery(resource)
+   * Retrieves the entity count for a given organisation and dataset.
+   *
+   * @param {string} orgEntity - The organisation entity to retrieve the entity count for.
+   * @param {string} dataset - The dataset to retrieve the entity count from.
+   * @returns {number} The entity count for the given resource and dataset.
+   */
+  async getEntityCount (orgEntity, dataset) {
+    const query = this.entityCountQuery(orgEntity)
     const result = await datasette.runQuery(query, dataset)
     return result.formattedData[0].entity_count
   }