diff --git a/CHANGELOG.md b/CHANGELOG.md index f831ae548..27f168506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ You can also check the [release page](https://github.com/visualize-admin/visuali - Features - Localized cube landing pages are now supported (dcat:landingPage) 🌎 + - Temporal dimension filters can now be pinned to dynamically use the most recent value (so that published charts automatically switch to it when the cube is updated) 📅 - Temporal dimensions can now be used as segmentation fields (excluding area and line charts) - Fixes - Copying a link to a new visualization from a dataset preview now correctly includes a data source diff --git a/app/components/chart-filters-list.tsx b/app/components/chart-filters-list.tsx index 8c19bfedb..e38262c13 100644 --- a/app/components/chart-filters-list.tsx +++ b/app/components/chart-filters-list.tsx @@ -9,6 +9,7 @@ import { FilterValue, getAnimationField, } from "@/configurator"; +import { isDynamicMaxValue } from "@/configurator/components/field"; import { Dimension, Measure, @@ -74,12 +75,15 @@ export const ChartFiltersList = (props: ChartFiltersListProps) => { return []; } + const filterValue = isDynamicMaxValue(f.value) + ? dimension.values[dimension.values.length - 1].value + : f.value; const value = isTemporalDimension(dimension) ? { - value: f.value, - label: timeFormatUnit(`${f.value}`, dimension.timeUnit), + value: filterValue, + label: timeFormatUnit(`${filterValue}`, dimension.timeUnit), } - : dimension.values.find((d) => d.value === f.value); + : dimension.values.find((d) => d.value === filterValue); return [{ dimension, value }]; }); diff --git a/app/configurator/components/chart-configurator.tsx b/app/configurator/components/chart-configurator.tsx index 41670699e..23d4812ac 100644 --- a/app/configurator/components/chart-configurator.tsx +++ b/app/configurator/components/chart-configurator.tsx @@ -56,10 +56,11 @@ import { ChartTypeSelector } from "@/configurator/components/chart-type-selector import { ControlTabField, DataFilterSelect, - DataFilterSelectDay, - DataFilterSelectTime, + DataFilterTemporal, + isDynamicMaxValue, OnOffControlTabField, } from "@/configurator/components/field"; +import { canRenderDatePickerField } from "@/configurator/components/field-date-picker"; import { getFiltersByMappingStatus, isConfiguring, @@ -97,7 +98,6 @@ type DataFilterSelectGenericProps = { const DataFilterSelectGeneric = (props: DataFilterSelectGenericProps) => { const { dimension, index, disabled, onRemove } = props; - const values = dimension.values; const controls = dimension.isKeyDimension ? null : ( { isOptional: !dimension.isKeyDimension, }; - if (isTemporalDimension(dimension)) { - if (dimension.timeUnit === "Day") { - return ; - } else if (dimension.timeUnit === "Month") { - return ; - } else { - const from = `${values[0].value}`; - const to = `${values[values.length - 1]?.value || from}`; - - return ( - - ); - } + if ( + isTemporalDimension(dimension) && + canRenderDatePickerField(dimension.timeUnit) + ) { + return ( + + ); } else { return ( @@ -231,6 +223,21 @@ const useEnsurePossibleFilters = ({ const oldFilters = getChartConfigFilters(chartConfig.cubes, cube.iri); + // Replace resolved values with potential dynamic max values to not + // override the dynamic max value with the resolved value + for (const [key, value] of Object.entries(oldFilters)) { + if ( + value.type === "single" && + isDynamicMaxValue(value.value) && + filters[key] + ) { + filters[key] = { + type: "single", + value: `${value.value}`, + }; + } + } + if (!isEqual(filters, oldFilters) && !isEmpty(filters)) { dispatch({ type: "CHART_CONFIG_FILTERS_UPDATE", diff --git a/app/configurator/components/field-date-picker.tsx b/app/configurator/components/field-date-picker.tsx index ce909583c..96938ef17 100644 --- a/app/configurator/components/field-date-picker.tsx +++ b/app/configurator/components/field-date-picker.tsx @@ -107,7 +107,7 @@ export const DatePickerField = (props: DatePickerFieldProps) => { ); }; -type DatePickerTimeUnit = +export type DatePickerTimeUnit = | TimeUnit.Day | TimeUnit.Week | TimeUnit.Month diff --git a/app/configurator/components/field.tsx b/app/configurator/components/field.tsx index 39c68022b..f4fbbaca8 100644 --- a/app/configurator/components/field.tsx +++ b/app/configurator/components/field.tsx @@ -1,12 +1,14 @@ -import { t } from "@lingui/macro"; +import { Trans, t } from "@lingui/macro"; import { CircularProgress, FormControlLabel, + FormGroup, + Switch as MUISwitch, Theme, Typography, } from "@mui/material"; import { makeStyles } from "@mui/styles"; -import { TimeLocaleObject, extent, timeFormat, timeParse } from "d3"; +import { TimeLocaleObject } from "d3"; import get from "lodash/get"; import orderBy from "lodash/orderBy"; import React, { @@ -28,6 +30,7 @@ import { Slider, Switch, } from "@/components/form"; +import { OpenMetadataPanelWrapper } from "@/components/metadata-panel"; import SelectTree from "@/components/select-tree"; import useDisclosure from "@/components/use-disclosure"; import { @@ -42,7 +45,10 @@ import { ControlTab, OnOffControlTab, } from "@/configurator/components/chart-controls/control-tab"; -import { DatePickerField } from "@/configurator/components/field-date-picker"; +import { + DatePickerField, + DatePickerTimeUnit, +} from "@/configurator/components/field-date-picker"; import { getTimeIntervalFormattedSelectOptions, getTimeIntervalWithProps, @@ -70,9 +76,9 @@ import { Component, Dimension, HierarchyValue, + ObservationValue, TemporalDimension, } from "@/domain/data"; -import { truthy } from "@/domain/types"; import { useTimeFormatLocale } from "@/formatters"; import { TimeUnit } from "@/graphql/query-hooks"; import { useLocale } from "@/locales/use-locale"; @@ -252,94 +258,132 @@ export const DataFilterSelect = ({ ); }; -const formatDate = timeFormat("%Y-%m-%d"); -const parseDate = timeParse("%Y-%m-%d"); +/** We can pin some filters' values to max value dynamically, so that when a new + * value is added to the dataset, it will be automatically used as default filter + * value for published charts. + */ +const VISUALIZE_MAX_VALUE = "VISUALIZE_MAX_VALUE"; + +/** Checks if a given filter value is supposed to be dynamiaclly pinned to max + * value. + */ +export const isDynamicMaxValue = ( + value: ObservationValue +): value is "VISUALIZE_MAX_VALUE" => { + return value === VISUALIZE_MAX_VALUE; +}; -export const DataFilterSelectDay = ({ - dimension, - label, - disabled, - isOptional, - controls, -}: { +type DataFilterTemporalProps = { dimension: TemporalDimension; - label: React.ReactNode; + timeUnit: DatePickerTimeUnit; disabled?: boolean; isOptional?: boolean; controls?: React.ReactNode; -}) => { +}; + +export const DataFilterTemporal = (props: DataFilterTemporalProps) => { + const { dimension, timeUnit, disabled, isOptional, controls } = props; + const { label: _label, values, timeFormat } = dimension; + const formatLocale = useTimeFormatLocale(); + const formatDate = formatLocale.format(timeFormat); + const parseDate = formatLocale.parse(timeFormat); const fieldProps = useSingleFilterSelect({ cubeIri: dimension.cubeIri, dimensionIri: dimension.iri, }); - const noneLabel = t({ - id: "controls.dimensionvalue.none", - message: `No Filter`, - }); - const optionalLabel = t({ - id: "controls.select.optional", - message: `optional`, - }); - const allOptions = useMemo(() => { - return isOptional - ? [ - { - value: FIELD_VALUE_NONE, - label: noneLabel, - isNoneValue: true, - }, - ...dimension.values, - ] - : dimension.values; - }, [isOptional, dimension.values, noneLabel]); - - const allOptionsSet = useMemo(() => { - return new Set( - allOptions - .filter((x) => x.value !== FIELD_VALUE_NONE) - .map((x) => { - try { - return x.value as string; - } catch (e) { - console.warn(`Bad value ${x.value}`); - return; - } - }) - .filter(truthy) - ); - }, [allOptions]); - - const isDisabled = useCallback( + const usesMostRecentDate = isDynamicMaxValue(fieldProps.value); + const label = isOptional ? ( + <> + {_label}{" "} + + (optional) + + + ) : ( + _label + ); + const { minDate, maxDate, optionValues } = React.useMemo(() => { + if (values.length) { + const options = values.map((d) => { + return { + label: `${d.value}`, + value: `${d.value}`, + }; + }); + + return { + minDate: parseDate(`${values[0].value}`) as Date, + maxDate: parseDate(`${values[values.length - 1].value}`) as Date, + optionValues: options.map((d) => d.value), + }; + } else { + const date = new Date(); + return { + minDate: date, + maxDate: date, + optionValues: [], + }; + } + }, [values, parseDate]); + const isDateDisabled = React.useCallback( (date: Date) => { - return !allOptionsSet.has(formatDate(date)); + return !optionValues.includes(formatDate(date)); }, - [allOptionsSet] + [optionValues, formatDate] ); - const dateValue = useMemo(() => { - const parsed = fieldProps.value ? parseDate(fieldProps.value) : undefined; - return parsed || new Date(); - }, [fieldProps.value]); - - const [minDate, maxDate] = useMemo(() => { - const [min, max] = extent(Array.from(allOptionsSet)); - if (!min || !max) { - return []; - } - return [new Date(min), new Date(max)] as const; - }, [allOptionsSet]); - return ( + + {label} + + } + /> + {/* FIXME: adapt to design */} + + + fieldProps.onChange({ + target: { + value: usesMostRecentDate + ? formatDate(maxDate) + : VISUALIZE_MAX_VALUE, + }, + }) + } + /> + } + // FIXME: adapt to design, translate + label={Use most recent} + /> + + + } + value={ + usesMostRecentDate ? maxDate : (parseDate(fieldProps.value) as Date) + } onChange={fieldProps.onChange} - name={dimension.iri} - value={dateValue} - isDateDisabled={isDisabled} + isDateDisabled={isDateDisabled} + timeUnit={timeUnit} + dateFormat={formatDate} minDate={minDate} maxDate={maxDate} + disabled={disabled || usesMostRecentDate} + controls={controls} /> ); }; diff --git a/app/rdf/queries.ts b/app/rdf/queries.ts index 1f935c8f0..14f8926af 100644 --- a/app/rdf/queries.ts +++ b/app/rdf/queries.ts @@ -1,3 +1,4 @@ +import { SELECT } from "@tpluscode/sparql-builder"; import { descending, group, index } from "d3"; import { Maybe } from "graphql-tools"; import keyBy from "lodash/keyBy"; @@ -7,6 +8,7 @@ import { Literal, NamedNode } from "rdf-js"; import { ParsingClient } from "sparql-http-client/ParsingClient"; import { LRUCache } from "typescript-lru-cache"; +import { isDynamicMaxValue } from "@/configurator/components/field"; import { PromiseValue, truthy } from "@/domain/types"; import { DataCubeComponentFilter } from "@/graphql/resolver-types"; import { createSource, pragmas } from "@/rdf/create-source"; @@ -550,14 +552,12 @@ export const getCubeObservations = async ({ cache: LRUCache | undefined; }): Promise => { const cubeView = View.fromCube(cube, false); - const allResolvedDimensions = await getCubeDimensions({ cube, locale, sparqlClient, cache, }); - const resolvedDimensions = allResolvedDimensions.filter((d) => { if (componentIris) { return ( @@ -569,7 +569,6 @@ export const getCubeObservations = async ({ return true; }); const resolvedDimensionsByIri = keyBy(resolvedDimensions, (d) => d.data.iri); - componentIris = resolvedDimensions.map((d) => d.data.iri); const serverFilters: Record = {}; @@ -592,7 +591,13 @@ export const getCubeObservations = async ({ } const observationFilters = filters - ? buildFilters({ cube, view: cubeView, filters: dbFilters, locale }) + ? await buildFilters({ + cube, + view: cubeView, + filters: dbFilters, + locale, + sparqlClient, + }) : []; const observationDimensions = buildDimensions({ @@ -784,103 +789,130 @@ export const hasHierarchy = (dim: CubeDimension) => { ); }; -const buildFilters = ({ +const buildFilters = async ({ cube, view, filters, locale, + sparqlClient, }: { cube: ExtendedCube; view: View; filters: Filters; locale: string; -}): Filter[] => { + sparqlClient: ParsingClient; +}): Promise => { const lookupSource = LookupSource.fromSource(cube.source); lookupSource.queryPrefix = pragmas; - return Object.entries(filters).flatMap(([iri, filter]) => { - const cubeDimension = cube.dimensions.find((d) => d.path?.value === iri); + return await Promise.all( + Object.entries(filters).flatMap(async ([iri, filter]) => { + const cubeDimension = cube.dimensions.find((d) => d.path?.value === iri); - if (!cubeDimension) { - console.warn(`WARNING: No cube dimension ${iri}`); - return []; - } + if (!cubeDimension) { + console.warn(`WARNING: No cube dimension ${iri}`); + return []; + } - const dimension = view.dimension({ cubeDimension: iri }); + const dimension = view.dimension({ cubeDimension: iri }); - if (!dimension) { - console.warn(`WARNING: No dimension ${iri}`); - return []; - } + if (!dimension) { + console.warn(`WARNING: No dimension ${iri}`); + return []; + } - // FIXME: Adding this dimension will make the query return nothing for dimensions that don't have it (no way to make it optional) - - /** - * When dealing with a versioned dimension, the value provided from the config is unversioned - * The relationship is expressed with schema:sameAs, so we need to look up the *versioned* value to apply the filter - * If the dimension is not versioned (e.g. if its values are Literals), it can be used directly to filter - */ - const filterDimension = dimensionIsVersioned(cubeDimension) - ? view.createDimension({ - source: lookupSource, - path: ns.schema.sameAs, - join: dimension, - as: labelDimensionIri(`${iri}/__sameAs__`), // Just a made up dimension name that is used in the generated query but nowhere else - }) - : dimension; - - const parsedCubeDimension = parseCubeDimension({ - dim: cubeDimension, - cube, - locale, - }); + // FIXME: Adding this dimension will make the query return nothing for dimensions that don't have it (no way to make it optional) + + /** + * When dealing with a versioned dimension, the value provided from the config is unversioned + * The relationship is expressed with schema:sameAs, so we need to look up the *versioned* value to apply the filter + * If the dimension is not versioned (e.g. if its values are Literals), it can be used directly to filter + */ + const filterDimension = dimensionIsVersioned(cubeDimension) + ? view.createDimension({ + source: lookupSource, + path: ns.schema.sameAs, + join: dimension, + as: labelDimensionIri(`${iri}/__sameAs__`), // Just a made up dimension name that is used in the generated query but nowhere else + }) + : dimension; + + const parsedCubeDimension = parseCubeDimension({ + dim: cubeDimension, + cube, + locale, + }); - const { dataType } = parsedCubeDimension.data; + const { dataType } = parsedCubeDimension.data; - if (ns.rdf.langString.value === dataType) { - throw new Error( - `Dimension <${iri}> has dataType 'langString', which is not supported by Visualize. In order to fix it, change the dataType to 'string' in the cube definition.` - ); - } + if (ns.rdf.langString.value === dataType) { + throw new Error( + `Dimension <${iri}> has dataType 'langString', which is not supported by Visualize. In order to fix it, change the dataType to 'string' in the cube definition.` + ); + } - const dimensionHasHierarchy = hasHierarchy(cubeDimension); - const toRDFValue = (d: string): NamedNode | Literal => { - return dataType && !dimensionHasHierarchy - ? parsedCubeDimension.data.hasUndefinedValues && - d === DIMENSION_VALUE_UNDEFINED - ? rdf.literal("", ns.cube.Undefined) - : rdf.literal(d, dataType) - : rdf.namedNode(d); - }; - - return filter.type === "single" - ? [ - filterDimension.filter.eq( - toRDFValue( - typeof filter.value === "number" - ? filter.value.toString() - : filter.value - ) - ), - ] - : filter.type === "multi" - ? // If values is an empty object, we filter by something that doesn't exist - [ - filterDimension.filter.in( - Object.keys(filter.values).length > 0 - ? Object.entries(filter.values).flatMap(([iri, selected]) => - selected ? [toRDFValue(iri)] : [] - ) - : [rdf.namedNode("EMPTY_VALUE")] - ), - ] - : filter.type === "range" - ? [ - filterDimension.filter.gte(toRDFValue(filter.from)), - filterDimension.filter.lte(toRDFValue(filter.to)), - ] - : []; - }); + const dimensionHasHierarchy = hasHierarchy(cubeDimension); + const toRDFValue = (d: string): NamedNode | Literal => { + return dataType && !dimensionHasHierarchy + ? parsedCubeDimension.data.hasUndefinedValues && + d === DIMENSION_VALUE_UNDEFINED + ? rdf.literal("", ns.cube.Undefined) + : rdf.literal(d, dataType) + : rdf.namedNode(d); + }; + + switch (filter.type) { + case "single": { + if (isDynamicMaxValue(filter.value)) { + if (cubeDimension.maxInclusive) { + return [ + filterDimension.filter.eq( + toRDFValue(cubeDimension.maxInclusive.value) + ), + ]; + } + + // Ideally we would query the max value directly in the observations + // query, but it doesn't seem to be supported by the cube-view-query + const maxValueQuery = SELECT`(MAX(?value) as ?value)` + .WHERE` <${cube.term?.value}> ?observationSet . + ?observationSet ?source . + ?source <${cubeDimension.path?.value}> ?value .`; + const maxValueRaw = await maxValueQuery.execute( + sparqlClient.query, + { operation: "postUrlencoded" } + ); + const maxValue = maxValueRaw?.[0]?.value?.value as string; + + return [filterDimension.filter.eq(toRDFValue(maxValue))]; + } + + return [filterDimension.filter.eq(toRDFValue(`${filter.value}`))]; + } + case "multi": { + // If values is an empty object, we filter by something that doesn't exist + return [ + filterDimension.filter.in( + Object.keys(filter.values).length > 0 + ? Object.entries(filter.values).flatMap(([iri, selected]) => + selected ? [toRDFValue(iri)] : [] + ) + : [rdf.namedNode("EMPTY_VALUE")] + ), + ]; + } + case "range": { + return [ + filterDimension.filter.gte(toRDFValue(filter.from)), + filterDimension.filter.lte(toRDFValue(filter.to)), + ]; + } + default: + const _exhaustiveCheck: never = filter; + return _exhaustiveCheck; + } + }) + ).then((d) => d.flat()); }; type ObservationRaw = Record;