now get data by getting all resources and working from there

src/middleware/common.middleware.js

@@ -50,6 +50,11 @@ export const fetchLatestResource = fetchOne({
   result: 'resource'
 })
 
+export const fetchActiveResourcesForOrganisationAndDataset = fetchMany({
+  query: ({ params }) => performanceDbApi.activeResourcesForOrganisationAndDatasetQuery(params.lpa, params.dataset),
+  result: 'resources'
+})
+
 export const takeResourceIdFromParams = (req) => {
   logger.debug('skipping resource fetch', { type: types.App, params: req.params })
   req.resource = { resource: req.params.resourceId }
@@ -158,8 +163,8 @@ export async function fetchIssues (req, res, next) {
  * @param {*} next
  */
 export async function reformatIssuesToBeByEntryNumber (req, res, next) {
-  const { issues } = req
-  const issuesByEntryNumber = issues.reduce((acc, current) => {
+  const { issuesWithReferences } = req
+  const issuesByEntryNumber = issuesWithReferences.reduce((acc, current) => {
     acc[current.entry_number] = acc[current.entry_number] || []
     acc[current.entry_number].push(current)
     return acc
@@ -170,7 +175,7 @@ export async function reformatIssuesToBeByEntryNumber (req, res, next) {
 
 export function formatErrorSummaryParams (req, res, next) {
   const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params
-  const { issuesByEntryNumber, entityCount: entityCountRow, issues } = req
+  const { issuesByEntryNumber, entityCount: entityCountRow, issuesWithReferences } = req
 
   const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 }
 
@@ -180,7 +185,7 @@ export function formatErrorSummaryParams (req, res, next) {
   let issueItems
 
   if (Object.keys(issuesByEntryNumber).length < entityCount) {
-    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true)
+    errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true)
     issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => {
       return {
         html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`,
@@ -189,7 +194,7 @@ export function formatErrorSummaryParams (req, res, next) {
     })
   } else {
     issueItems = [{
-      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true)
+      html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true)
     }]
   }
 
@@ -218,10 +223,13 @@ export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({
 
 export const paginateEntitiesAndPullOutCount = (req, res, next) => {
   const { entities, pagination } = req
+  const { pageNumber } = req.params
+
+  const paginationIndex = pageNumber - 1
 
   req.entitiesWithIssuesCount = entities.length
 
-  req.entities = entities.slice(pagination.offset, pagination.offset + pagination.limit)
+  req.entities = entities.slice(pagination.offset * paginationIndex, pagination.offset * paginationIndex + pagination.limit)
 
   next()
 }
@@ -278,10 +286,10 @@ export const nestEntityFields = (req, res, next) => {
 }
 
 export const addIssuesToEntities = (req, res, next) => {
-  const { entities, issues } = req
+  const { entities, issuesWithReferences } = req
 
   req.entitiesWithIssues = entities.map(entity => {
-    const entityIssues = issues.filter(issue => issue.entryNumber === entity.entryNumber)
+    const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber)
 
     entityIssues.forEach(issue => {
       entity[issue.field].value = issue.value
@@ -293,3 +301,27 @@ export const addIssuesToEntities = (req, res, next) => {
 
   next()
 }
+
+export const hasEntities = (req, res, next) => req.entities !== undefined
+
+export const fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield = fetchMany({
+  query: ({ req, params }) => performanceDbApi.issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery({
+    resources: req.resources.map(resourceObj => resourceObj.resource),
+    dataset: params.dataset,
+    issueType: params.issue_type,
+    issueField: params.issue_field
+  }),
+  result: 'issuesWithReferences',
+  dataset: FetchOptions.fromParams
+})
+
+export const fetchEntitiesFromIssuesWithReferences = fetchMany({
+  query: ({ req }) => performanceDbApi.fetchEntitiesFromReferencesAndOrganisationEntity({
+    references: req.issuesWithReferences.map(issueWithReference => issueWithReference.reference),
+    organisationEntity: req.orgInfo.entity
+  }),
+  result: 'entities',
+  dataset: FetchOptions.fromParams
+})
+
+// export const getReferencesOfIssueEntities

src/middleware/issueDetails.middleware.js

@@ -1,5 +1,5 @@
 import performanceDbApi from '../services/performanceDbApi.js'
-import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
+import { fetchDatasetInfo, fetchEntitiesFromOrganisationAndEntryNumbers, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, getEntryNumbersWithIssues, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { pagination } from '../utils/pagination.js'
@@ -245,6 +245,8 @@ export default [
   fetchDatasetInfo,
   fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams),
   fetchIssues,
+  getEntryNumbersWithIssues,
+  fetchEntitiesFromOrganisationAndEntryNumbers,
   reformatIssuesToBeByEntryNumber,
   getEntryNumberFromPageNumber,
   parallel([

src/middleware/issueTable.middleware.js

@@ -3,15 +3,13 @@ import {
   addIssuesToEntities,
   extractJsonFieldFromEntities,
   fetchDatasetInfo,
-  fetchEntitiesFromOrganisationAndEntryNumbers,
   fetchEntityCount,
-  fetchIssues,
   fetchLatestResource,
   fetchOrgInfo,
   fetchSpecification,
   formatErrorSummaryParams,
-  getEntryNumbersWithIssues,
   getPaginationOptions,
+  hasEntities,
   isResourceIdNotInParams,
   logPageError,
   nestEntityFields,
@@ -20,7 +18,10 @@ import {
   reformatIssuesToBeByEntryNumber,
   replaceUnderscoreWithHyphenForEntities,
   takeResourceIdFromParams,
-  validateQueryParams
+  validateQueryParams,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
+  fetchEntitiesFromIssuesWithReferences
 } from './common.middleware.js'
 import { fetchIf, parallel, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -159,6 +160,14 @@ export const getIssueTable = renderTemplate({
   handlerName: 'getIssueTable'
 })
 
+// const getEntitiesWithIssuesMiddlewareChain = [
+//   fetchResourcesFromOrganisationAndDataset,
+//   fetchIssuesFromResourcesDatasetIssuetypefield,
+//   // have a list of issues with resource, and entry number
+//   getReferencesOfIssueEntities,
+//   getEntitiesFromRefernces
+// ]
+
 export default [
   validateIssueTableQueryParams,
   setDefaultQueryParams,
@@ -170,14 +179,14 @@ export default [
   fetchSpecification,
   pullOutDatasetSpecification,
   getPaginationOptions(paginationPageLength),
-  fetchIssues,
-  getEntryNumbersWithIssues,
-  fetchEntitiesFromOrganisationAndEntryNumbers,
-  paginateEntitiesAndPullOutCount,
-  extractJsonFieldFromEntities,
-  replaceUnderscoreWithHyphenForEntities,
-  nestEntityFields,
-  addIssuesToEntities,
+  fetchActiveResourcesForOrganisationAndDataset,
+  fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield,
+  fetchEntitiesFromIssuesWithReferences,
+  fetchIf(hasEntities, paginateEntitiesAndPullOutCount),
+  fetchIf(hasEntities, extractJsonFieldFromEntities),
+  fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities),
+  fetchIf(hasEntities, nestEntityFields),
+  fetchIf(hasEntities, addIssuesToEntities),
   fetchEntityCount,
   reformatIssuesToBeByEntryNumber,
   formatErrorSummaryParams,

src/services/performanceDbApi.js

@@ -263,6 +263,20 @@ export default {
     AND rle.pipeline = '${dataset}'`
   },
 
+  activeResourcesForOrganisationAndDatasetQuery: (lpa, dataset) => {
+    return /* sql */`
+      select
+        rhe.endpoint, rhe.endpoint_url, rhe.resource, rhe.status
+      from
+        reporting_historic_endpoints rhe
+      LEFT JOIN resource_organisation ro ON rhe.resource = ro.resource
+      LEFT JOIN organisation o ON REPLACE(ro.organisation, '-eng', '') = o.organisation
+      WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}'
+      AND pipeline = '${dataset}'
+      AND rhe.endpoint_end_date == ''
+    `
+  },
+
   /**
      * Retrieves the latest resource information for a given LPA and dataset.
      *
@@ -364,6 +378,7 @@ export default {
      * @param {string} [database="digital-land"] - Database to query (defaults to "digital-land")
      * @returns {Promise<Object>} - Promise resolving to an object with formatted data
      */
+
   async getIssues ({ organisation, dataset, resource, issueType, issueField }, database = 'digital-land') {
     let sql = `
       SELECT i.field, i.line_number, entry_number, message, issue_type, value
@@ -392,6 +407,20 @@ export default {
     return result.formattedData
   },
 
+  issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery ({ resources, dataset, issueType, issueField }) {
+    return /* sql */ `
+      SELECT DISTINCT i.message, i.value, i.field, i.issue_type, i.entry_number, f.value as reference
+      FROM issue i
+      LEFT JOIN fact_resource fr ON i.entry_number = fr.entry_number AND i.resource = fr.resource
+      LEFT JOIN fact f ON fr.fact = f.fact
+      WHERE i.resource  in ('${resources.join("', '")}')
+      AND dataset  = '${dataset}'
+      AND issue_type  = '${issueType}'
+      AND i.field  = '${issueField}'
+      AND f.field = 'reference'
+    `
+  },
+
   /**
      *
      * @param {*} resourceId
@@ -462,5 +491,23 @@ export default {
     //   LIMIT ${pagination.limit}
     //   OFFSET ${pagination.offset}
     // `
+  },
+
+  fetchEntityFromEntryNumber ({ entryNumber, organisationEntity }) {
+    return /* sql */ `
+      select DISTINCT f.entity, fr.entry_number, fr.resource, e.* from fact f
+      LEFT JOIN fact_resource fr ON f.fact = fr.fact
+      LEFT JOIN entity e ON f.entity = e.entity
+      AND e.organisation_entity = ${organisationEntity}
+      AND entry_number = ${entryNumber}
+    `
+  },
+
+  fetchEntitiesFromReferencesAndOrganisationEntity ({ references, organisationEntity }) {
+    return /* sql */ `
+      select * from entity 
+      WHERE reference in ('${references.join("', '")}')
+      AND organisation_entity = ${organisationEntity}
+    `
   }
 }