682 dataset task list fetch data from all active resources

src/middleware/common.middleware.js

@@ -1,6 +1,7 @@
 // @ts-check
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
+import { entryIssueGroups } from '../utils/utils.js'
 import performanceDbApi from '../services/performanceDbApi.js'
 import { fetchMany, fetchOne, FetchOneFallbackPolicy, FetchOptions, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
@@ -195,7 +196,7 @@ export const createPaginationTemplateParams = (req, res, next) => {
 
 export const fetchResources = fetchMany({
   query: ({ req }) => `
-    SELECT r.end_date, r.entry_date, r.mime_type, r.resource, r.start_date, rle.endpoint_url, rle.licence, rle.status, rle.latest_log_entry_date, rle.endpoint_entry_date from resource r
+    SELECT DISTINCT rd.dataset, r.end_date, r.entry_date, r.mime_type, r.resource, r.start_date, rle.endpoint_url, rle.licence, rle.status, rle.latest_log_entry_date, rle.endpoint_entry_date from resource r
     LEFT JOIN resource_organisation ro ON ro.resource = r.resource
     LEFT JOIN resource_dataset rd ON rd.resource = r.resource
     LEFT JOIN reporting_latest_endpoints rle ON r.resource = rle.resource
@@ -206,6 +207,31 @@ export const fetchResources = fetchMany({
   result: 'resources'
 })
 
+export const addEntityCountsToResources = async (req, res, next) => {
+  const { resources } = req
+
+  const promises = resources.map(resource => {
+    const query = `SELECT entry_count FROM dataset_resource WHERE resource = "${resource.resource}"`
+    return datasette.runQuery(query, resource.dataset)
+  })
+
+  try {
+    const datasetResources = await Promise.all(promises).catch(error => {
+      logger.error('Failed to fetch dataset resources', { type: types.DataFetch, errorMessage: error.message, errorStack: error.stack })
+      throw error
+    })
+
+    req.resources = resources.map((resource, i) => {
+      return { ...resource, entry_count: datasetResources[i]?.formattedData[0]?.entry_count }
+    })
+
+    next()
+  } catch (error) {
+    logger.error('Error in addEntityCountsToResources', { type: types.App, errorMessage: error.message, errorStack: error.stack })
+    next(error)
+  }
+}
+
 // Specification
 
 export const fetchSpecification = fetchOne({
@@ -383,67 +409,25 @@ export const filterOutEntitiesWithoutIssues = (req, res, next) => {
 
 const fetchEntityIssuesForFieldAndType = fetchMany({
   query: ({ req, params }) => {
-    const issueTypeFilter = params.issue_type ? `AND issue_type = '${params.issue_type}'` : ''
-    const issueFieldFilter = params.issue_field ? `AND field = '${params.issue_field}'` : ''
-
+    const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : ''
+    const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : ''
     return `
-      SELECT e.entity, i.* FROM entity e
-      INNER JOIN issue i ON e.entity = i.entity
-      WHERE e.organisation_entity = ${req.orgInfo.entity}
-      ${issueTypeFilter}
-      ${issueFieldFilter}`
+        select i.issue_type, field, entity, message, severity, value
+        from issue i
+        LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+        WHERE resource = '${req.resources[0].resource}'
+        ${issueTypeClause}
+        AND it.responsibility = 'external'
+        AND it.severity = 'error'
+        ${issueFieldClause}
+        AND i.dataset = '${req.params.dataset}'
+        AND entity != ''
+        `
+    // LIMIT ${req.dataRange.pageLength} OFFSET ${req.dataRange.offset}
   },
-  dataset: FetchOptions.fromParams,
   result: 'issues'
 })
 
-export const FilterOutIssuesToMostRecent = (req, res, next) => {
-  const { resources, issues } = req
-
-  const issuesWithResources = issues.filter(issue => {
-    if (!issue.resource || !resources.find(resource => resource.resource === issue.resource)) {
-      logger.warn(`Missing resource on issue: ${JSON.stringify(issue)}`)
-      return false
-    }
-    return true
-  })
-
-  const groupedIssues = issuesWithResources.reduce((acc, current) => {
-    current.start_date = new Date(resources.find(resource => resource.resource === current.resource)?.start_date)
-    const { entity, field } = current
-    if (!acc[entity]) {
-      acc[entity] = {}
-    }
-    if (!acc[entity][field]) {
-      acc[entity][field] = []
-    }
-    acc[entity][field].push(current)
-
-    return acc
-  }, {})
-
-  const recentIssues = Object.fromEntries(Object.entries(groupedIssues).map(([entityName, issuesByEntity]) =>
-    [
-      entityName,
-      Object.fromEntries(Object.entries(issuesByEntity).map(([field, issues]) => [
-        field,
-        issues.sort((a, b) => b.start_date.getTime() - a.start_date.getTime())[0]
-      ]))
-    ]
-  ))
-
-  const issuesFlattened = []
-
-  Object.values(recentIssues).forEach(issueByEntry => {
-    Object.values(issueByEntry).forEach(issueByField => {
-      issuesFlattened.push(issueByField)
-    })
-  })
-
-  req.issues = issuesFlattened
-  next()
-}
-
 export const removeIssuesThatHaveBeenFixed = async (req, res, next) => {
   const { issues, resources } = req
 
@@ -518,18 +502,80 @@ export const addFieldMappingsToIssue = (req, res, next) => {
   next()
 }
 
-export const addReferencesToIssues = (req, res, next) => {
-  const { issues, entities } = req
+// We can only get the issues without entity from the latest resource as we have no way of knowing if those in previous resources have been fixed?
+export const fetchEntryIssues = fetchMany({
+  query: ({ req, params }) => {
+    const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : ''
+    const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : ''
+    return `
+      select i.issue_type, field, entity, message, severity, value, line_number
+      from issue i
+      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+      WHERE resource = '${req.resources[0].resource}'
+      ${issueTypeClause}
+      AND it.responsibility = 'external'
+      AND it.severity = 'error'
+      AND i.dataset = '${req.params.dataset}'
+      ${issueFieldClause}
+      AND (entity = '' OR entity is NULL OR i.issue_type in ('${entryIssueGroups.map(issue => issue.type).join("', '")}'))
+      LIMIT ${req.dataRange.pageLength} OFFSET ${req.dataRange.offset}
+    `
+  },
+  result: 'entryIssues'
+})
 
-  req.issues = issues.map(issue => {
-    const reference = entities.find(entity => entity.entity === issue.entity)?.reference
+export const fetchEntityIssueCounts = fetchMany({
+  query: ({ req }) => {
+    const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : ''
+    return `
+      select dataset, field, i.issue_type, COUNT(resource+line_number) as count
+      from issue i
+      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+      WHERE resource in ('${req.resources.map(resource => resource.resource).join("', '")}')
+      AND (entity != '' AND entity IS NOT NULL)
+      AND it.responsibility = 'external'
+      AND it.severity = 'error'
+      ${datasetClause}
+      GROUP BY field, i.issue_type, dataset
+    `
+  },
+  result: 'entityIssueCounts'
+})
 
-    return { ...issue, reference }
+export const getMostRecentResources = (resources) => {
+  const mostRecentResourcesMap = {}
+  resources.forEach(resource => {
+    const currentRecent = mostRecentResourcesMap[resource.dataset]
+    if (!currentRecent || new Date(currentRecent.start_date).getTime() < new Date(resource.start_date).getTime()) {
+      mostRecentResourcesMap[resource.dataset] = resource
+    }
   })
-
-  next()
+  return Object.values(mostRecentResourcesMap)
 }
 
+export const fetchEntryIssueCounts = fetchMany({
+  query: ({ req }) => {
+    const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : ''
+
+    const mostRecentResources = getMostRecentResources(req.resources)
+
+    const resourceIds = Object.values(mostRecentResources).map(resource => resource.resource)
+
+    return `
+      select dataset, field, i.issue_type, COUNT(resource + line_number) as count
+      from issue i
+      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
+      WHERE resource in ('${resourceIds.join("', '")}')
+      AND (entity = '' OR entity is NULL)
+      AND it.responsibility = 'external'
+      AND it.severity = 'error'
+      ${datasetClause}
+      GROUP BY field, i.issue_type, dataset
+    `
+  },
+  result: 'entryIssueCounts'
+})
+
 /**
  * This middleware chain is responsible for retrieving all entities for the given organisation, their latest issues,
  * filtering out issues that have been fixed, and constructing the table params.
@@ -544,11 +590,11 @@ export const addReferencesToIssues = (req, res, next) => {
  */
 export const processRelevantIssuesMiddlewares = [
   fetchEntityIssuesForFieldAndType,
-  FilterOutIssuesToMostRecent,
-  removeIssuesThatHaveBeenFixed,
+  // arguably removeIssuesThatHaveBeenFixed should be s step however we have only currently found one organisation,
+  // however this step is very time consuming, so in order to progress im commenting it out for now
+  // removeIssuesThatHaveBeenFixed,
   fetchFieldMappings,
-  addFieldMappingsToIssue,
-  addReferencesToIssues
+  addFieldMappingsToIssue
 ]
 
 // Other
@@ -608,14 +654,16 @@ export const getSetDataRange = (pageLength) => (req, res, next) => {
 
 export function getErrorSummaryItems (req, res, next) {
   const { issue_type: issueType, issue_field: issueField } = req.params
-  const { issues, issueCount, entities, resources } = req
+  const { entryIssues, issues: entityIssues, issueCount, entities, resources } = req
+
+  const issues = entityIssues || entryIssues
 
   const totalRecordCount = entities ? entities.length : resources[0].entry_count
   const totalIssues = issueCount?.count || issues.length
 
   const errorHeading = ''
   const issueItems = [{
-    html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: totalIssues, entityCount: totalRecordCount, field: issueField }, true)
+    html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: totalIssues, rowCount: totalRecordCount, field: issueField }, true)
   }]
 
   req.errorSummary = {