diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index aeab97eb..ebf17332 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -51,13 +51,19 @@ export const fetchLatestResource = fetchOne({ fallbackPolicy: FetchOneFallbackPolicy.continue }) +export const fetchResources = fetchMany({ + query: ({ params }) => `select * from endpoint_dataset_resource_summary where REPLACE(organisation, '-eng', '') = "${params.lpa}" AND dataset = "${params.dataset}"`, + dataset: FetchOptions.performanceDb, + result: 'resources' +}) + export const takeResourceIdFromParams = (req) => { logger.debug('skipping resource fetch', { type: types.App, params: req.params }) req.resource = { resource: req.params.resourceId } } export const fetchEntityCount = fetchOne({ - query: ({ req }) => performanceDbApi.entityCountQuery(req.resource.resource), + query: ({ req }) => performanceDbApi.entityCountQuery(req.orgInfo.entity), result: 'entityCount', dataset: FetchOptions.fromParams, fallbackPolicy: FetchOneFallbackPolicy.continue diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index cbcf67d9..5fc0be2f 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -129,11 +129,8 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { }).map((source, index) => { let error - if (parseInt(source.status) < 200 || parseInt(source.status) >= 300) { - error = { - code: parseInt(source.status), - exception: source.exception - } + if (parseInt(source.status) < 200 || parseInt(source.status) >= 300 || source.status === '' || source.exception) { + error = 'There was a ' + (source.exception || source.status) + ' error accessing the endpoint' } return { diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index f77db39e..f16c2d63 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -1,5 +1,5 @@ -import { fetchDatasetInfo, isResourceAccessible, isResourceNotAccessible, fetchLatestResource, fetchEntityCount, logPageError, fetchLpaDatasetIssues, validateQueryParams, getDatasetTaskListError } from './common.middleware.js' -import { fetchOne, fetchIf, onlyIf, renderTemplate } from './middleware.builders.js' +import { fetchDatasetInfo, fetchEntityCount, logPageError, validateQueryParams, fetchResources } from './common.middleware.js' +import { fetchOne, renderTemplate, fetchMany } from './middleware.builders.js' import performanceDbApi from '../services/performanceDbApi.js' import { statusToTagClass } from '../filters/filters.js' import * as v from 'valibot' @@ -14,7 +14,7 @@ export const fetchResourceStatus = fetchOne({ const fetchOrgInfoWithStatGeo = fetchOne({ query: ({ params }) => { - return /* sql */ `SELECT name, organisation, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'` + return /* sql */ `SELECT name, organisation, statistical_geography, entity FROM organisation WHERE organisation = '${params.lpa}'` }, result: 'orgInfo' }) @@ -43,19 +43,19 @@ function getStatusTag (status) { * @returns { { templateParams: object }} */ export const prepareDatasetTaskListTemplateParams = (req, res, next) => { - const { issues, entityCount: entityCountRow, params, dataset, orgInfo: organisation } = req + const { entityCount: entityCountRow, params, dataset, orgInfo: organisation, tasks } = req const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 } const { lpa, dataset: datasetId } = params - console.assert(req.resourceStatus.resource === req.resource.resource, 'mismatch between resourceStatus and resource data') + console.assert(typeof entityCount === 'number', 'entityCount should be a number') - const taskList = issues.map((issue) => { + const taskList = tasks.map((task) => { return { title: { - text: performanceDbApi.getTaskMessage({ ...issue, entityCount, field: issue.field }) + text: performanceDbApi.getTaskMessage({ ...task, field: task.field }) // using the entity count here doesn't make sense, should be using the entry count for each resource }, - href: `/organisations/${lpa}/${datasetId}/${issue.issue_type}/${issue.field}`, - status: getStatusTag(issue.status) + href: `/organisations/${lpa}/${datasetId}/${task.issue_type}/${task.field}`, + status: getStatusTag(task.status) } }) @@ -113,17 +113,46 @@ const validateParams = validateQueryParams({ }) }) +export const fetchLpaDatasetTasks = fetchMany({ + query: ({ req, params }) => ` + SELECT + i.field, + i.issue_type, + i.line_number, + i.value, + i.message, + CASE + WHEN COUNT( + CASE + WHEN it.severity == 'error' THEN 1 + ELSE null + END + ) > 0 THEN 'Needs fixing' + ELSE 'Live' + END AS status, + COUNT(i.issue_type) as num_issues + FROM + issue i + LEFT JOIN + issue_type it ON i.issue_type = it.issue_type + WHERE + i.resource in ('${req.resources.map(resource => resource.resource).join("', '")}') + AND i.dataset = '${params.dataset}' + AND (it.severity == 'error') + GROUP BY i.issue_type, i.field + ORDER BY it.severity`, + result: 'tasks' +}) + export default [ validateParams, fetchResourceStatus, fetchOrgInfoWithStatGeo, fetchDatasetInfo, - fetchIf(isResourceAccessible, fetchLatestResource), - fetchIf(isResourceAccessible, fetchLpaDatasetIssues), - fetchIf(isResourceAccessible, fetchEntityCount), - onlyIf(isResourceAccessible, prepareDatasetTaskListTemplateParams), - onlyIf(isResourceAccessible, getDatasetTaskList), - onlyIf(isResourceNotAccessible, prepareDatasetTaskListErrorTemplateParams), - onlyIf(isResourceNotAccessible, getDatasetTaskListError), + fetchResources, + fetchLpaDatasetTasks, + fetchEntityCount, + prepareDatasetTaskListTemplateParams, + getDatasetTaskList, logPageError ] diff --git a/src/middleware/issueDetails.middleware.js b/src/middleware/issueDetails.middleware.js index a537a3c9..f90290b8 100644 --- a/src/middleware/issueDetails.middleware.js +++ b/src/middleware/issueDetails.middleware.js @@ -1,8 +1,6 @@ import performanceDbApi from '../services/performanceDbApi.js' -import logger from '../utils/logger.js' -import { types } from '../utils/logging.js' -import { fetchDatasetInfo, fetchEntityCount, fetchLatestResource, fetchOrgInfo, isResourceIdInParams, logPageError, takeResourceIdFromParams, validateQueryParams } from './common.middleware.js' -import { fetchIf, renderTemplate } from './middleware.builders.js' +import { fetchDatasetInfo, fetchEntityCount, fetchOrgInfo, fetchResources, logPageError, validateQueryParams } from './common.middleware.js' +import { renderTemplate } from './middleware.builders.js' import * as v from 'valibot' import { pagination } from '../utils/pagination.js' @@ -31,14 +29,10 @@ const validateIssueDetailsQueryParams = validateQueryParams({ */ async function fetchIssues (req, res, next) { const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params - const { resource: resourceId } = req.resource - if (!resourceId) { - logger.debug('fetchIssues(): missing resourceId', { type: types.App, params: req.params, resource: req.resource }) - throw Error('fetchIssues: missing resourceId') - } + const { resources } = req try { - const issues = await performanceDbApi.getIssues({ resource: resourceId, issueType, issueField }, datasetId) + const issues = await performanceDbApi.getIssues({ resources: resources.map(resource => resource.resource), issueType, issueField }, datasetId) req.issues = issues next() } catch (error) { @@ -56,11 +50,11 @@ async function fetchIssues (req, res, next) { * @param {*} res * @param {*} next */ -async function reformatIssuesToBeByEntryNumber (req, res, next) { +async function reformatIssuesToBeByResourceEntryNumber (req, res, next) { const { issues } = req const issuesByEntryNumber = issues.reduce((acc, current) => { - acc[current.entry_number] = acc[current.entry_number] || [] - acc[current.entry_number].push(current) + acc[current.resource + current.entry_number] = acc[current.resource + current.entry_number] || [] + acc[current.resource + current.entry_number].push(current) return acc }, {}) req.issuesByEntryNumber = issuesByEntryNumber @@ -86,10 +80,13 @@ async function fetchEntry (req, res, next) { // look at issue Entries and get the index of that entry - 1 - const entityNum = Object.values(issuesByEntryNumber)[pageNum - 1][0].entry_number + const issue = Object.values(issuesByEntryNumber)[pageNum - 1][0] + + const entityNum = issue.entry_number + const resource = issue.resource req.entryData = await performanceDbApi.getEntry( - req.resource.resource, + resource, entityNum, datasetId ) @@ -109,9 +106,8 @@ async function fetchEntry (req, res, next) { */ async function fetchIssueEntitiesCount (req, res, next) { const { dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params - const { resource: resourceId } = req.resource - console.assert(resourceId, 'missng resource id') - const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resource: resourceId, issueType, issueField }, datasetId) + const { resources } = req + const issueEntitiesCount = await performanceDbApi.getEntitiesWithIssuesCount({ resources: resources.map(resource => resource.resource), issueType, issueField }, datasetId) req.issueEntitiesCount = parseInt(issueEntitiesCount) next() } @@ -182,29 +178,19 @@ const processEntryRow = (issueType, issuesByEntryNumber, row) => { * Middleware. Updates req with `templateParams` */ export function prepareIssueDetailsTemplateParams (req, res, next) { - const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber, entityCount: entityCountRow } = req + const { entryData, pageNumber, issueEntitiesCount, issuesByEntryNumber, entryNumber } = req const { lpa, dataset: datasetId, issue_type: issueType, issue_field: issueField } = req.params - const { entity_count: entityCount } = entityCountRow ?? { entity_count: 0 } - - let errorHeading - let issueItems const BaseSubpath = `/organisations/${lpa}/${datasetId}/${issueType}/${issueField}/` - if (Object.keys(issuesByEntryNumber).length < entityCount) { - errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true) - issueItems = Object.entries(issuesByEntryNumber).map(([entryNumber, issues], i) => { - const pageNum = i + 1 - return { - html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${entryNumber}`, - href: `${BaseSubpath}${pageNum}` - } - }) - } else { - issueItems = [{ - html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, entityCount, field: issueField }, true) - }] - } + const errorHeading = performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: issueEntitiesCount, field: issueField }, true) + const issueItems = Object.entries(issuesByEntryNumber).map(([entryNumber, issues], i) => { + const pageNum = i + 1 + return { + html: performanceDbApi.getTaskMessage({ issue_type: issueType, num_issues: 1, field: issueField }) + ` in record ${issues[0].entry_number}`, + href: `${BaseSubpath}${pageNum}` + } + }) const fields = entryData.map((row) => processEntryRow(issueType, issuesByEntryNumber, row)) const entityIssues = Object.values(issuesByEntryNumber)[pageNumber - 1] || [] @@ -288,9 +274,9 @@ export default [ validateIssueDetailsQueryParams, fetchOrgInfo, fetchDatasetInfo, - fetchIf(isResourceIdInParams, fetchLatestResource, takeResourceIdFromParams), + fetchResources, fetchIssues, - reformatIssuesToBeByEntryNumber, + reformatIssuesToBeByResourceEntryNumber, fetchEntry, fetchEntityCount, fetchIssueEntitiesCount, diff --git a/src/middleware/overview.middleware.js b/src/middleware/overview.middleware.js index 55f738b8..197978d1 100644 --- a/src/middleware/overview.middleware.js +++ b/src/middleware/overview.middleware.js @@ -1,8 +1,6 @@ -import performanceDbApi, { lpaOverviewQuery } from '../services/performanceDbApi.js' import { fetchOrgInfo, logPageError } from './common.middleware.js' import { fetchMany, FetchOptions, renderTemplate } from './middleware.builders.js' import { dataSubjects } from '../utils/utils.js' -import config from '../../config/index.js' import _ from 'lodash' // get a list of available datasets @@ -12,43 +10,6 @@ const availableDatasets = Object.values(dataSubjects).flatMap((dataSubject) => .map((dataset) => dataset.value) ) -/** - * Middleware. Updates req with 'lpaOverview' - * - * Relies on {@link config}. - * - * @param {{ params: { lpa: string }, entityCounts: { dataset: string, resource: string, entityCount?: number }[]}} req - */ -const fetchLpaOverview = fetchMany({ - query: ({ req, params }) => { - return lpaOverviewQuery(params.lpa, { datasetsFilter: config.datasetsFilter, entityCounts: req.entityCounts }) - }, - dataset: FetchOptions.performanceDb, - result: 'lpaOverview' -}) - -const fetchLatestResources = fetchMany({ - query: ({ params }) => { - return performanceDbApi.latestResourcesQuery(params.lpa, { datasetsFilter: config.datasetsFilter }) - }, - result: 'resourceLookup', - dataset: FetchOptions.performanceDb -}) - -/** - * Updates req with `entityCounts` (of shape `{ resource, dataset}|{ resource, dataset, entityCount }`) - * - * @param {{ resourceLookup: {resource: string, dataset: string}[] }} req - * @param {*} res - * @param {*} next - */ -const fetchEntityCounts = async (req, res, next) => { - const { resourceLookup } = req - - req.entityCounts = await performanceDbApi.getEntityCounts(resourceLookup) - next() -} - /** * For the purpose of displaying single status label on (possibly) many issues, * we want issues with 'worse' status to be weighted higher. @@ -101,40 +62,61 @@ export function aggregateOverviewData (lpaOverview) { * @returns */ const orgStatsReducer = (accumulator, dataset) => { - if (dataset.endpoint) accumulator[0]++ - if (dataset.status === 'Needs fixing') accumulator[1]++ + if (dataset.active_endpoint_count > 0) accumulator[0]++ if (dataset.status === 'Error') accumulator[2]++ + if (dataset.error_endpoint_count > 0) accumulator[2]++ return accumulator } export function prepareOverviewTemplateParams (req, res, next) { - const { lpaOverview, orgInfo: organisation } = req - const datasets = aggregateOverviewData(lpaOverview) - // add in any of the missing key 8 datasets - const keys = new Set(datasets.map(d => d.slug)) + const { provisionSummary, orgInfo: organisation } = req + + // filter down to only the ones we want + const datasets = provisionSummary.filter(dataset => availableDatasets.includes(dataset.dataset)) + + // add in any datasets that they the performance db doesn't have + const keys = new Set(datasets.map(d => d.dataset)) availableDatasets.forEach((dataset) => { if (!keys.has(dataset)) { - const row = { - slug: dataset, - endpoint: null, - status: 'Not submitted', - issue_count: 0, - entity_count: undefined - } - datasets.push(row) + datasets.push({ + dataset, + active_endpoint_count: 0, + error_endpoint_count: 0, + count_issue_error_internal: 0, + count_issue_error_external: 0, + count_issue_warning_internal: 0, + count_issue_warning_external: 0, + count_issue_notice_internal: 0, + count_issue_notice_external: 0 + }) } }) // re-sort the datasets to be in alphabetical order - datasets.sort((a, b) => a.slug.localeCompare(b.slug)) + datasets.sort((a, b) => a.dataset.localeCompare(b.dataset)) + + // add status's to the dataset + const datasetsWithStatus = datasets.map(dataset => { + const datasetWithStatus = { ...dataset } + + if (dataset.error_endpoint_count > 0) { + datasetWithStatus.status = 'Error' + } else if (dataset.count_issue_error_external > 0) { + datasetWithStatus.status = 'Needs fixing' + } else if (dataset.active_endpoint_count > 0) { + datasetWithStatus.status = 'Live' + } else { + datasetWithStatus.status = 'Not submitted' + } + return datasetWithStatus + }) const totalDatasets = datasets.length - const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = - datasets.reduce(orgStatsReducer, [0, 0, 0]) + const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce(orgStatsReducer, [0, 0, 0]) req.templateParams = { organisation, - datasets, + datasets: datasetsWithStatus, totalDatasets, datasetsWithEndpoints, datasetsWithIssues, @@ -153,11 +135,20 @@ export const getOverview = renderTemplate({ handlerName: 'getOverview' }) +export const fetchProvisionSummary = fetchMany({ + query: ({ req, params }) => `select * from provision_summary where REPLACE(organisation, '-eng', '') = "${params.lpa}"`, + dataset: FetchOptions.performanceDb, + result: 'provisionSummary' +}) + +/* + Notes on how this middleware needs to change: + +*/ + export default [ fetchOrgInfo, - fetchLatestResources, - fetchEntityCounts, - fetchLpaOverview, + fetchProvisionSummary, prepareOverviewTemplateParams, getOverview, logPageError diff --git a/src/routes/schemas.js b/src/routes/schemas.js index 7434f22d..e9af8680 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -44,16 +44,18 @@ const DatasetNameField = v.strictObject({ name: NonEmptyString, dataset: NonEmpt export const OrgOverviewPage = v.strictObject({ organisation: OrgField, datasets: v.array(v.strictObject({ - endpoint: v.optional(v.url()), - status: v.enum(datasetStatusEnum), - slug: NonEmptyString, - issue_count: v.optional(v.number()), - error: v.optional(v.nullable(NonEmptyString)), - http_error: v.optional(NonEmptyString), - issue: v.optional(NonEmptyString), - entity_count: v.optional(v.number()), - // synthetic entry, represents a user friendly count (e.g. count missing value in a column as 1 issue) - numIssues: v.optional(v.number()) + organisation: v.optional(v.string()), + organisation_name: v.optional(v.string()), + dataset: v.string(), + status: v.string(), + active_endpoint_count: v.number(), + error_endpoint_count: v.number(), + count_issue_error_internal: v.number(), + count_issue_error_external: v.number(), + count_issue_warning_internal: v.number(), + count_issue_warning_external: v.number(), + count_issue_notice_internal: v.number(), + count_issue_notice_external: v.number() })), totalDatasets: v.integer(), datasetsWithEndpoints: v.integer(), @@ -85,10 +87,7 @@ export const OrgDatasetOverview = v.strictObject({ endpoint: v.string(), lastAccessed: v.string(), lastUpdated: v.string(), - error: v.optional(v.strictObject({ - code: v.integer(), - exception: v.string() - })) + error: v.optional(v.string()) })) }) }) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 10d29d19..837524ce 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -294,11 +294,21 @@ export default { return /* sql */ ` select rle.pipeline as dataset, - rle.resource as resource + endpoint, + endpoint_url, + rle.resource as resource, + status, + exception, + latest_log_entry_date, + endpoint_entry_date, + resource_start_date from reporting_latest_endpoints rle where REPLACE(organisation, '-eng', '') = '${lpa}' - ${datasetClause}` + ${datasetClause} + AND (endpoint_end_date is null OR endpoint_end_date == '' OR DATE(endpoint_end_date) > DATE('now')) + AND (resource_end_date is null OR resource_end_date == '' OR DATE(resource_end_date) > DATE('now')) + AND (resource_start_date is not null AND resource_start_date != '' AND DATE(resource_start_date) < DATE('now'))` }, /** @@ -337,14 +347,14 @@ export default { * @returns {Promise} Count of entities with issues */ async getEntitiesWithIssuesCount ({ - resource, + resources, issueType, issueField }, database = 'digital-land') { const sql = ` - SELECT count(DISTINCT entry_number) as count + SELECT count(entry_number) as count FROM issue - WHERE resource = '${resource}' + WHERE resource in ('${resources.join("', '")}') AND issue_type = '${issueType}' AND field = '${issueField}' ` @@ -365,14 +375,14 @@ export default { * @returns {Promise} - Promise resolving to an object with formatted data */ async getIssues ({ - resource, + resources, issueType, issueField }, database = 'digital-land') { const sql = ` - SELECT i.field, i.line_number, entry_number, message, issue_type, value + SELECT i.field, i.line_number, entry_number, message, issue_type, value, resource FROM issue i - WHERE resource = '${resource}' + WHERE resource in ('${resources.join("', '")}') AND issue_type = '${issueType}' AND field = '${issueField}' ` diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html index 5f822833..5024ee19 100644 --- a/src/views/organisations/dataset-overview.html +++ b/src/views/organisations/dataset-overview.html @@ -113,7 +113,7 @@ classes: 'app-inset-text---error' }, value: { - html: (endpoint.lastAccessed | govukDateTime) + '

There was a '+endpoint.error.code+' error accessing the data URL

' + html: (endpoint.lastAccessed | govukDateTime) + '

'+endpoint.error+'

' } } %} {% else %} diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index 01c7b92a..32bb91bc 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -71,13 +71,13 @@

Datasets

{% if dataset.status == 'Not submitted' %} - {{dataset.slug | datasetSlugToReadableName}} + href="/organisations/{{ organisation.organisation }}/{{dataset.dataset}}/get-started"> + {{dataset.dataset | datasetSlugToReadableName}} {% else %} - {{dataset.slug | datasetSlugToReadableName}} + href="/organisations/{{ organisation.organisation }}/{{dataset.dataset}}/overview"> + {{dataset.dataset | datasetSlugToReadableName}} {% endif %}

@@ -85,9 +85,9 @@

{% if dataset.status == 'Not submitted' %}

Data URL not submitted

{% elif dataset.status == 'Error' %} -

{{dataset.error}}

+

{{dataset.error_endpoint_count}} out of a supplied {{dataset.active_endpoint_count + dataset.error_endpoint_count}} endpoints could not be indexed

{% elif dataset.status == 'Needs fixing' %} -

There {{ "is" | pluralise(dataset.issue_count) }} {{ dataset.issue_count }} {{ "issue" | pluralise(dataset.issue_count) }} in this dataset

+

There {{ "is" | pluralise(dataset.count_issue_error_external) }} {{ dataset.count_issue_error_external }} {{ "issue" | pluralise(dataset.count_issue_error_external) }} in this dataset

{% else %}

Data URL submitted

{% endif %}