query overhall after entity vs entry talk

src/middleware/common.middleware.js

@@ -126,25 +126,21 @@ export async function fetchIssueEntitiesCount (req, res, next) {
 }
 
 /**
-*
-* Middleware. Updates `req` with `issues`.
-*
-* Requires `resourceId` in request params or request (in that order).
-*
-* @param {*} req
-* @param {*} res
-* @param {*} next
-*/
+ * Fetches issues from the performance database and updates the request object with the result.
+ *
+ * This middleware requires the `resourceId` to be present in the request params or request object.
+ *
+ * @param {object} req - The HTTP request object
+ * @param {object} res - The HTTP response object
+ * @param {function} next - The next middleware function in the stack
+ *
+ * @throws {Error} If `resourceId` is missing from the request
+ */
 export async function fetchIssues (req, res, next) {
-  const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { resource: resourceId } = req.resource
-  if (!resourceId) {
-    logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource })
-    throw Error('fetchIssues: missing resourceId')
-  }
+  const { dataset, issue_type: issueType, issue_field: issueField, lpa } = req.params
 
   try {
-    req.issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId)
+    req.issues = await performanceDbApi.getIssues({ organisation: lpa, dataset, issueType, issueField })
     next()
   } catch (error) {
     next(error)
@@ -203,3 +199,79 @@ export function formatErrorSummaryParams (req, res, next) {
   }
   next()
 }
+
+export const getEntryNumbersWithIssues = (req, res, next) => {
+  const { issues } = req
+
+  const entryNumbersWithIssues = [...new Set(issues.map(issue => issue.entry_number))]
+
+  req.entryNumbersWithIssues = entryNumbersWithIssues
+
+  next()
+}
+
+export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
+  query: ({ req, params }) => performanceDbApi.fetchEntityNumbersFromEntryNumbers({ entryNumbers: req.entryNumbersWithIssues, organisationEntity: req.orgInfo.entity }),
+  result: 'entities',
+  dataset: FetchOptions.fromParams
+})
+
+export const extractJsonFieldFromEntities = (req, res, next) => {
+  const { entities } = req
+
+  req.entities = entities.map(entity => {
+    const jsonField = entity.json
+    delete entity.json
+    const parsedJson = JSON.parse(jsonField)
+    entity = { ...entity, ...parsedJson }
+    return entity
+  })
+
+  next()
+}
+
+export const replaceUnderscoreWithHyphenForEntities = (req, res, next) => {
+  const { entities } = req
+
+  entities.forEach(entity => {
+    Object.keys(entity).forEach(key => {
+      if (key.includes('_')) {
+        const newKey = key.replace(/_/g, '-')
+        entity[newKey] = entity[key]
+        delete entity[key]
+      }
+    })
+  })
+
+  next()
+}
+
+export const nestEntityFields = (req, res, next) => {
+  const { entities, specification } = req
+
+  req.entities = entities.map(entity => {
+    specification.fields.forEach(field => {
+      entity[field.field] = { value: entity[field.field] }
+    })
+    return entity
+  })
+
+  next()
+}
+
+export const addIssuesToEntities = (req, res, next) => {
+  const { entities, issues } = req
+
+  req.entitiesWithIssues = entities.map(entity => {
+    const entityIssues = issues.filter(issue => issue.entryNumber === entity.entryNumber)
+
+    entityIssues.forEach(issue => {
+      entity[issue.field].value = issue.value
+      entity[issue.field].issue = issue
+    })
+
+    return entity
+  })
+
+  next()
+}

src/middleware/issueTable.middleware.js

@@ -1,8 +1,27 @@
-import performanceDbApi from '../services/performanceDbApi.js'
-import logger from '../utils/logger.js'
 import { pagination } from '../utils/pagination.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, fetchSpecification, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, pullOutDatasetSpecification, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
-import { fetchIf, fetchMany, FetchOptions, parallel, renderTemplate } from './middleware.builders.js'
+import {
+  addIssuesToEntities,
+  extractJsonFieldFromEntities,
+  fetchDatasetInfo,
+  fetchEntitiesFromOrganisationAndEntryNumbers,
+  fetchEntityCount,
+  fetchIssueEntitiesCount,
+  fetchIssues,
+  fetchLatestResource,
+  fetchOrgInfo,
+  fetchSpecification,
+  formatErrorSummaryParams,
+  getEntryNumbersWithIssues,
+  isResourceIdNotInParams,
+  logPageError,
+  nestEntityFields,
+  pullOutDatasetSpecification,
+  reformatIssuesToBeByEntryNumber,
+  replaceUnderscoreWithHyphenForEntities,
+  takeResourceIdFromParams,
+  validateQueryParams
+} from './common.middleware.js'
+import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 
 const paginationPageLength = 50
@@ -27,23 +46,6 @@ export const setDefaultQueryParams = (req, res, next) => {
   next()
 }
 
-const fetchEntitiesWithIssues = fetchMany({
-  query: ({ req, params }) => {
-    const pagination = {
-      limit: paginationPageLength,
-      offset: paginationPageLength * (params.pageNumber - 1)
-    }
-    return performanceDbApi.entitiesAndIssuesQuery({
-      resource: req.resource.resource,
-      issueType: req.params.issue_type,
-      issueField: req.params.issue_field,
-      pagination
-    })
-  },
-  result: 'entitiesWithIssues',
-  dataset: FetchOptions.fromParams
-})
-
 /**
  * Middleware function to prepare issue table template params
  *
@@ -53,42 +55,27 @@ const fetchEntitiesWithIssues = fetchMany({
  */
 export const prepareIssueTableTemplateParams = (req, res, next) => {
   const { issue_type: issueType, issue_field: issueField, lpa, dataset: datasetId } = req.params
-  const { entitiesWithIssues, specification, pagination, errorSummary } = req
+  const { entities, specification, pagination, errorSummary } = req
 
   const tableParams = {
     columns: specification.fields.map(field => field.field),
     fields: specification.fields.map(field => field.field),
-    rows: entitiesWithIssues.map((entity, index) => {
+    rows: entities.map((entity, index) => {
       const columns = {}
 
       specification.fields.forEach(fieldObject => {
         const { field } = fieldObject
         if (field === 'reference') {
           const pageNumber = index + 1
           const entityLink = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/entry/${pageNumber}`
-          columns[field] = { html: `<a href="${entityLink}">${entity[field]}</a>` }
+          columns[field] = { html: `<a href="${entityLink}">${entity[field].value}</a>`, error: entity[field].issue }
         } else if (entity[field]) {
-          columns[field] = { value: entity[field] }
+          columns[field] = { value: entity[field].value, error: entity[field].issue }
         } else {
           columns[field] = { value: '' }
         }
       })
 
-      let issues = {}
-      try {
-        issues = JSON.parse(entity.issues)
-      } catch (e) {
-        logger.warn('issueTableMiddleware:prepareIssueTableParams - entity issues is not valid json', { entityIssues: entity.issues })
-      }
-
-      Object.entries(issues).forEach(([field, issueType]) => {
-        if (columns[field]) {
-          columns[field].error = { message: issueType }
-        } else {
-          columns[field] = { value: '', error: { message: issueType } }
-        }
-      })
-
       return {
         columns
       }
@@ -177,13 +164,18 @@ export default [
     fetchDatasetInfo
   ]),
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
-  fetchEntitiesWithIssues,
-  fetchIssueEntitiesCount,
   fetchSpecification,
-  fetchIssues,
-  reformatIssuesToBeByEntryNumber,
   pullOutDatasetSpecification,
+  fetchIssues,
+  getEntryNumbersWithIssues,
+  fetchEntitiesFromOrganisationAndEntryNumbers,
+  extractJsonFieldFromEntities,
+  replaceUnderscoreWithHyphenForEntities,
+  nestEntityFields,
+  addIssuesToEntities,
+  fetchIssueEntitiesCount,
   fetchEntityCount,
+  reformatIssuesToBeByEntryNumber,
   formatErrorSummaryParams,
   createPaginationTemplatePrams,
   prepareIssueTableTemplateParams,

src/services/performanceDbApi.js

@@ -364,18 +364,28 @@ export default {
      * @param {string} [database="digital-land"] - Database to query (defaults to "digital-land")
      * @returns {Promise<Object>} - Promise resolving to an object with formatted data
      */
-  async getIssues ({
-    resource,
-    issueType,
-    issueField
-  }, database = 'digital-land') {
-    const sql = `
-    SELECT i.field, i.line_number, entry_number, message, issue_type, value
-    FROM issue i
-    WHERE resource = '${resource}'
-    AND issue_type = '${issueType}'
-    AND field = '${issueField}'
-  `
+  async getIssues ({ organisation, dataset, resource, issueType, issueField }, database = 'digital-land') {
+    let sql = `
+      SELECT i.field, i.line_number, entry_number, message, issue_type, value
+      FROM issue i
+      LEFT JOIN reporting_historic_endpoints rhe ON rhe.resource = i.resource
+      WHERE REPLACE(rhe.organisation, '-eng', '') = '${organisation}'
+      AND rhe.pipeline = '${dataset}'
+    `
+
+    if (resource) {
+      sql += ` AND i.resource = '${resource}'`
+    }
+
+    if (issueType) {
+      sql += ` AND i.issue_type = '${issueType}'`
+    }
+
+    if (issueField) {
+      sql += ` AND i.field = '${issueField}'`
+    }
+
+    // (no changes below this line)
 
     const result = await datasette.runQuery(sql, database)
 
@@ -471,5 +481,15 @@ export default {
         LIMIT ${pagination.limit}
         OFFSET ${pagination.offset}
       `
+  },
+
+  fetchEntityNumbersFromEntryNumbers ({ entryNumbers, organisationEntity }) {
+    return /* sql */ `
+      select DISTINCT f.entity, fr.entry_number, fr.resource, e.* from fact f
+      LEFT JOIN fact_resource fr ON f.fact = fr.fact
+      LEFT JOIN entity e ON f.entity = e.entity
+      WHERE e.organisation_entity = ${organisationEntity}
+      AND entry_number in (${entryNumbers.join(', ')})
+    `
   }
 }