get slug mapping from datasette on startup

index.js

@@ -20,7 +20,7 @@ const app = express()
 
 setupMiddlewares(app)
 setupSession(app)
-setupNunjucks({ app, dataSubjects })
+await setupNunjucks({ app, dataSubjects })
 setupRoutes(app)
 setupSentry(app)
 setupErrorHandlers(app)

src/filters/filters.js

@@ -5,11 +5,14 @@ import toErrorList from './toErrorList.js'
 import prettifyColumnName from './prettifyColumnName.js'
 import getFullServiceName from './getFullServiceName.js'
 import { makeDatasetSlugToReadableNameFilter, createDatasetMapping } from './makeDatasetSlugToReadableNameFilter.js'
+import datasette from '../services/datasette.js'
 
 const { govukMarkdown } = xGovFilters
 
-const addFilters = (nunjucksEnv, { dataSubjects }) => {
-  const datasetNameMapping = createDatasetMapping(dataSubjects)
+const addFilters = async (nunjucksEnv) => {
+
+  const datasetSlugNameTable = await datasette.runQuery('select dataset, name from dataset')
+  const datasetNameMapping = createDatasetMapping(datasetSlugNameTable.rows)
   const datasetSlugToReadableName = makeDatasetSlugToReadableNameFilter(datasetNameMapping)
   nunjucksEnv.addFilter('datasetSlugToReadableName', datasetSlugToReadableName)
 

src/filters/makeDatasetSlugToReadableNameFilter.js

@@ -28,16 +28,17 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => {
 }
 
 /**
- *
- * @param {*} dataSubjects
- * @returns {Map<string,string>}
+ * Creates a mapping of dataset slugs to their corresponding readable names.
+ * 
+ * @param {Array<[string, string]>} dataSubjects - An array of tuples, where each tuple contains a dataset slug and its corresponding readable name.
+ * @returns {Map<string, string>} - A Map object where each key is a dataset slug and its value is the corresponding readable name.
  */
 export const createDatasetMapping = (dataSubjects) => {
-  const mapping = new Map()
-  for (const data of Object.values(dataSubjects)) {
-    for (const dataset of data.dataSets) {
-      mapping.set(dataset.value, dataset.text)
-    }
-  }
-  return mapping
+  const datasetMapping = new Map();
+  dataSubjects.forEach(([slug, name]) => {
+    datasetMapping.set(slug, name);
+  });
+  return datasetMapping;
 }
+
+

src/serverSetup/nunjucks.js

@@ -2,7 +2,7 @@ import nunjucks from 'nunjucks'
 import config from '../../config/index.js'
 import addFilters from '../filters/filters.js'
 
-export function setupNunjucks ({ app, dataSubjects }) {
+export async function setupNunjucks ({ app, dataSubjects }) {
   if (app) {
     app.set('view engine', 'html')
   }
@@ -33,7 +33,7 @@ export function setupNunjucks ({ app, dataSubjects }) {
   Object.keys(globalValues).forEach((key) => {
     nunjucksEnv.addGlobal(key, globalValues[key])
   })
-  addFilters(nunjucksEnv, { dataSubjects })
+  await addFilters(nunjucksEnv, { dataSubjects })
 
   return nunjucks
 }

src/services/datasette.js

@@ -5,15 +5,44 @@ const datasetteUrl = 'https://datasette.planning.data.gov.uk'
 const database = 'digital-land'
 
 export default {
+  /**
+ * Executes a SQL query on the Datasette instance and returns the results.
+ *
+ * @param {string} query - The SQL query to execute.
+ * @returns {Promise<{data: object, formattedData: object}>} - A promise that resolves to an object with the following properties:
+ *   - `data`: The raw data returned by Datasette.
+ *   - `formattedData`: The formatted data, with columns and rows parsed into a usable format.
+ * @throws {Error} If the query fails or there is an error communicating with Datasette.
+ */
   runQuery: async (query) => {
     const encodedQuery = encodeURIComponent(query)
     const url = `${datasetteUrl}/${database}.json?sql=${encodedQuery}`
     try {
       const response = await axios.get(url)
-      return response.data
+      return {
+        ...response.data,
+        formattedData: formatData(response.data.columns, response.data.rows)
+      }
     } catch (error) {
       logger.error(error)
       throw error
     }
   }
 }
+
+/**
+ * Formats an array of rows into an easier to access format, where each row is an object with column names as keys.
+ *
+ * @param {string[]} columns - An array of column names
+ * @param {any[][]} rows - A 2D array of row data, where each inner array represents a row
+ * @returns {object[]} - An array of objects, where each object represents a row with column names as keys
+ */
+function formatData (columns, rows){
+  // convert the rows into an easier to access format
+  return rows.map((row) => {
+    return row.reduce((acc, val, index) => {
+      acc[columns[index]] = val
+      return acc
+    }, {})
+  })
+}

src/services/performanceDbApi.js

@@ -86,16 +86,7 @@ ORDER BY
 
     const result = await datasette.runQuery(query)
 
-    // convert the rows into an easier to access format
-    const columns = result.columns
-    const rows = result.rows.map((row) => {
-      return row.reduce((acc, val, index) => {
-        acc[columns[index]] = val
-        return acc
-      }, {})
-    })
-
-    const datasets = rows.reduce((accumulator, row) => {
+    const datasets = result.formattedData.reduce((accumulator, row) => {
       let error
       if (row.http_status !== '200' || row.exception !== '') {
         error = row.exception !== '' ? row.exception : `endpoint returned with a status of ${row.http_status}`
@@ -115,7 +106,7 @@ ORDER BY
     }, {})
 
     return {
-      name: result.rows[0][1],
+      name: result.formattedData[0].name,
       datasets
     }
   }