diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index 1496e58c..800b3a1b 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -50,6 +50,11 @@ export const fetchLatestResource = fetchOne({ result: 'resource' }) +export const fetchActiveResourcesForOrganisationAndDataset = fetchMany({ + query: ({ params }) => performanceDbApi.activeResourcesForOrganisationAndDatasetQuery(params.lpa, params.dataset), + result: 'resources' +}) + export const takeResourceIdFromParams = (req) => { logger.debug('skipping resource fetch', { type: types.App, params: req.params }) req.resource = { resource: req.params.resourceId } @@ -158,8 +163,8 @@ export async function fetchIssues (req, res, next) { * @param {*} next */ export async function reformatIssuesToBeByEntryNumber (req, res, next) { - const { issues } = req - const issuesByEntryNumber = issues.reduce((acc, current) => { + const { issuesWithReferences } = req + const issuesByEntryNumber = issuesWithReferences.reduce((acc, current) => { acc[current.entry_number] = acc[current.entry_number] || [] acc[current.entry_number].push(current) return acc @@ -170,7 +175,7 @@ export async function reformatIssuesToBeByEntryNumber (req, res, next) { export function formatErrorSummaryParams (req, res, next) { const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params - const { issuesByEntryNumber, entityCount: entityCountRow, issues } = req + const { issuesByEntryNumber, entityCount: entityCountRow, issuesWithReferences } = req const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 } @@ -180,7 +185,7 @@ export function formatErrorSummaryParams (req, res, next) { let issueItems if (Object.keys(issuesByEntryNumber).length < entityCount) { - errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true) + errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true) issueItems = Object.keys(issuesByEntryNumber).map((entryNumber, i) => { return { html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`, @@ -189,7 +194,7 @@ export function formatErrorSummaryParams (req, res, next) { }) } else { issueItems = [{ - html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issues.length, entityCount, field: issueField }, true) + html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issuesWithReferences.length, entityCount, field: issueField }, true) }] } @@ -218,10 +223,13 @@ export const fetchEntitiesFromOrganisationAndEntryNumbers = fetchMany({ export const paginateEntitiesAndPullOutCount = (req, res, next) => { const { entities, pagination } = req + const { pageNumber } = req.params + + const paginationIndex = pageNumber - 1 req.entitiesWithIssuesCount = entities.length - req.entities = entities.slice(pagination.offset, pagination.offset + pagination.limit) + req.entities = entities.slice(pagination.offset * paginationIndex, pagination.offset * paginationIndex + pagination.limit) next() } @@ -278,10 +286,10 @@ export const nestEntityFields = (req, res, next) => { } export const addIssuesToEntities = (req, res, next) => { - const { entities, issues } = req + const { entities, issuesWithReferences } = req req.entitiesWithIssues = entities.map(entity => { - const entityIssues = issues.filter(issue => issue.entryNumber === entity.entryNumber) + const entityIssues = issuesWithReferences.filter(issue => issue.entryNumber === entity.entryNumber) entityIssues.forEach(issue => { entity[issue.field].value = issue.value @@ -293,3 +301,27 @@ export const addIssuesToEntities = (req, res, next) => { next() } + +export const hasEntities = (req, res, next) => req.entities !== undefined + +export const fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield = fetchMany({ + query: ({ req, params }) => performanceDbApi.issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery({ + resources: req.resources.map(resourceObj => resourceObj.resource), + dataset: params.dataset, + issueType: params.issue_type, + issueField: params.issue_field + }), + result: 'issuesWithReferences', + dataset: FetchOptions.fromParams +}) + +export const fetchEntitiesFromIssuesWithReferences = fetchMany({ + query: ({ req }) => performanceDbApi.fetchEntitiesFromReferencesAndOrganisationEntity({ + references: req.issuesWithReferences.map(issueWithReference => issueWithReference.reference), + organisationEntity: req.orgInfo.entity + }), + result: 'entities', + dataset: FetchOptions.fromParams +}) + +// export const getReferencesOfIssueEntities diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js index 9904f802..0ac54fd9 100644 --- a/src/middleware/issueDetails.middleware.js +++ b/src/middleware/issueDetails.middleware.js @@ -1,5 +1,5 @@ import performanceDbApi from '../services/performanceDbApi.js' -import { fetchDatasetInfo, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js' +import { fetchDatasetInfo, fetchEntitiesFromOrganisationAndEntryNumbers, fetchEntityCount, fetchIssueEntitiesCount, fetchIssues, fetchLatestResource, fetchOrgInfo, formatErrorSummaryParams, getEntryNumbersWithIssues, isResourceIdNotInParams, logPageError, reformatIssuesToBeByEntryNumber, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js' import { fetchIf, parallel, renderTemplate } from './middleware.builders.js' import * as v from 'valibot' import { pagination } from '../utils/pagination.js' @@ -245,6 +245,8 @@ export default [ fetchDatasetInfo, fetchIf(isResourceIdNotInParams, fetchLatestResource, takeResourceIdFromParams), fetchIssues, + getEntryNumbersWithIssues, + fetchEntitiesFromOrganisationAndEntryNumbers, reformatIssuesToBeByEntryNumber, getEntryNumberFromPageNumber, parallel([ diff --git a/src/middleware/issueTable.middleware.js b/src/middleware/issueTable.middleware.js index aedaddd2..b69ec516 100644 --- a/src/middleware/issueTable.middleware.js +++ b/src/middleware/issueTable.middleware.js @@ -3,15 +3,13 @@ import { addIssuesToEntities, extractJsonFieldFromEntities, fetchDatasetInfo, - fetchEntitiesFromOrganisationAndEntryNumbers, fetchEntityCount, - fetchIssues, fetchLatestResource, fetchOrgInfo, fetchSpecification, formatErrorSummaryParams, - getEntryNumbersWithIssues, getPaginationOptions, + hasEntities, isResourceIdNotInParams, logPageError, nestEntityFields, @@ -20,7 +18,10 @@ import { reformatIssuesToBeByEntryNumber, replaceUnderscoreWithHyphenForEntities, takeResourceIdFromParams, - validateQueryParams + validateQueryParams, + fetchActiveResourcesForOrganisationAndDataset, + fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield, + fetchEntitiesFromIssuesWithReferences } from './common.middleware.js' import { fetchIf, parallel, renderTemplate } from './middleware.builders.js' import * as v from 'valibot' @@ -159,6 +160,14 @@ export const getIssueTable = renderTemplate({ handlerName: 'getIssueTable' }) +// const getEntitiesWithIssuesMiddlewareChain = [ +// fetchResourcesFromOrganisationAndDataset, +// fetchIssuesFromResourcesDatasetIssuetypefield, +// // have a list of issues with resource, and entry number +// getReferencesOfIssueEntities, +// getEntitiesFromRefernces +// ] + export default [ validateIssueTableQueryParams, setDefaultQueryParams, @@ -170,14 +179,14 @@ export default [ fetchSpecification, pullOutDatasetSpecification, getPaginationOptions(paginationPageLength), - fetchIssues, - getEntryNumbersWithIssues, - fetchEntitiesFromOrganisationAndEntryNumbers, - paginateEntitiesAndPullOutCount, - extractJsonFieldFromEntities, - replaceUnderscoreWithHyphenForEntities, - nestEntityFields, - addIssuesToEntities, + fetchActiveResourcesForOrganisationAndDataset, + fetchIssuesWithReferencesFromResourcesDatasetIssuetypefield, + fetchEntitiesFromIssuesWithReferences, + fetchIf(hasEntities, paginateEntitiesAndPullOutCount), + fetchIf(hasEntities, extractJsonFieldFromEntities), + fetchIf(hasEntities, replaceUnderscoreWithHyphenForEntities), + fetchIf(hasEntities, nestEntityFields), + fetchIf(hasEntities, addIssuesToEntities), fetchEntityCount, reformatIssuesToBeByEntryNumber, formatErrorSummaryParams, diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index b165d24d..b68edba9 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -263,6 +263,20 @@ export default { AND rle.pipeline = '${dataset}'` }, + activeResourcesForOrganisationAndDatasetQuery: (lpa, dataset) => { + return /* sql */` + select + rhe.endpoint, rhe.endpoint_url, rhe.resource, rhe.status + from + reporting_historic_endpoints rhe + LEFT JOIN resource_organisation ro ON rhe.resource = ro.resource + LEFT JOIN organisation o ON REPLACE(ro.organisation, '-eng', '') = o.organisation + WHERE REPLACE(ro.organisation, '-eng', '') = '${lpa}' + AND pipeline = '${dataset}' + AND rhe.endpoint_end_date == '' + ` + }, + /** * Retrieves the latest resource information for a given LPA and dataset. * @@ -364,6 +378,7 @@ export default { * @param {string} [database="digital-land"] - Database to query (defaults to "digital-land") * @returns {Promise} - Promise resolving to an object with formatted data */ + async getIssues ({ organisation, dataset, resource, issueType, issueField }, database = 'digital-land') { let sql = ` SELECT i.field, i.line_number, entry_number, message, issue_type, value @@ -392,6 +407,20 @@ export default { return result.formattedData }, + issuesWithReferenceFromResourcesDatasetIssueTypeFieldQuery ({ resources, dataset, issueType, issueField }) { + return /* sql */ ` + SELECT DISTINCT i.message, i.value, i.field, i.issue_type, i.entry_number, f.value as reference + FROM issue i + LEFT JOIN fact_resource fr ON i.entry_number = fr.entry_number AND i.resource = fr.resource + LEFT JOIN fact f ON fr.fact = f.fact + WHERE i.resource in ('${resources.join("', '")}') + AND dataset = '${dataset}' + AND issue_type = '${issueType}' + AND i.field = '${issueField}' + AND f.field = 'reference' + ` + }, + /** * * @param {*} resourceId @@ -462,5 +491,23 @@ export default { // LIMIT ${pagination.limit} // OFFSET ${pagination.offset} // ` + }, + + fetchEntityFromEntryNumber ({ entryNumber, organisationEntity }) { + return /* sql */ ` + select DISTINCT f.entity, fr.entry_number, fr.resource, e.* from fact f + LEFT JOIN fact_resource fr ON f.fact = fr.fact + LEFT JOIN entity e ON f.entity = e.entity + AND e.organisation_entity = ${organisationEntity} + AND entry_number = ${entryNumber} + ` + }, + + fetchEntitiesFromReferencesAndOrganisationEntity ({ references, organisationEntity }) { + return /* sql */ ` + select * from entity + WHERE reference in ('${references.join("', '")}') + AND organisation_entity = ${organisationEntity} + ` } }