✨ [#4993] Implement fetching select(boxes) options from Referentielijsten

docker/docker-compose.referentielijsten.yml

@@ -0,0 +1,57 @@
+version: '3.8'
+
+name: referentielijsten
+
+services:
+  referentielijsten-redis:
+    image: redis:7
+    command: ["redis-server", "--appendonly", "yes"]
+    networks:
+      - open-forms-dev
+
+  referentielijsten-db:
+    image: postgres:${PG_VERSION:-14}
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+    volumes:
+      - ./referentielijsten/docker-init-referentielijsten-db/:/docker-entrypoint-initdb.d
+      - referentielijsten-db:/var/lib/postgresql/data
+    networks:
+      - open-forms-dev
+
+  referentielijsten-web.local:
+    image: maykinmedia/referentielijsten-api:${REFERENTIELIJSTEN_VERSION:-0.2.0}
+    environment: &referentielijsten_web_env
+      - DJANGO_SETTINGS_MODULE=referentielijsten.conf.docker
+      - SECRET_KEY=${SECRET_KEY:-7&3f^bo1(-5($bre4iv-!nt%1xr!b54b&y7+97j5f&ndm_e=lz}
+      - ALLOWED_HOSTS=referentielijsten-web.local,localhost
+      - DB_NAME=referentielijsten
+      - DB_USER=referentielijsten
+      - DB_HOST=referentielijsten-db
+      - DISABLE_2FA=true
+      - IS_HTTPS=no
+      - CACHE_DEFAULT=referentielijsten-redis:6379/0
+      - CACHE_AXES=referentielijsten-redis:6379/0
+      - SUBPATH=${SUBPATH:-/}
+      - DJANGO_SUPERUSER_PASSWORD=admin
+    ports:
+      - 8004:8000
+    volumes: &referentielijsten_web_volumes
+      # mount fixtures dir to automatically populate the DB
+      - ./referentielijsten/fixtures/:/app/fixtures
+      - media:/app/media  # Shared media volume to get access to saved OAS files
+      - private-media:/app/private-media
+    depends_on:
+      - referentielijsten-db
+      - referentielijsten-redis
+    networks:
+      - open-forms-dev
+
+volumes:
+  referentielijsten-db:
+  media:
+  private-media:
+
+networks:
+  open-forms-dev:
+    name: open-forms-dev

docker/referentielijsten/README.md

@@ -0,0 +1,41 @@
+# Referentielijsten API
+
+The `docker-compose.referentielijsten.yml` compose file is available to run an instance of Referentielijsten API.
+
+## docker compose
+
+Start an instance in your local environment from the parent directory:
+
+```bash
+docker compose -f docker-compose.referentielijsten.yml up -d
+```
+
+This brings up the admin at http://localhost:8004/admin/. You can log in with the `admin` / `admin`
+credentials.
+
+## Load fixtures
+
+The fixtures in `referentielijsten/fixtures` are automatically loaded when the Referentielijsten container starts.
+
+## Dump fixtures
+
+Whenever you make changes in the admin for the tests, you need to dump the fixtures again so that
+bringing up the containers the next time (or in other developers' environments) will still have the
+same data.
+
+Dump the fixtures with (in the `docker` directory):
+
+```bash
+docker compose -f docker-compose.referentielijsten.yml run referentielijsten-web.local \
+    python src/manage.py dumpdata \
+        --indent=4 \
+        --output /app/fixtures/referentielijsten_fixtures.json \
+        accounts \
+        api
+```
+
+Depending on your OS, you may need to grant extra write permissions:
+
+```bash
+chmod o+rwx ./referentielijsten/fixtures
+```

docker/referentielijsten/docker-init-referentielijsten-db/0001-docker-init-referentielijsten-db.sql

@@ -0,0 +1,5 @@
+CREATE USER referentielijsten;
+CREATE DATABASE referentielijsten;
+GRANT ALL PRIVILEGES ON DATABASE referentielijsten TO referentielijsten;
+-- On Postgres 15+, connect to the database and grant schema permissions.
+-- GRANT USAGE, CREATE ON SCHEMA public TO referentielijsten;

docker/referentielijsten/fixtures/referentielijsten_fixtures.json

@@ -0,0 +1,57 @@
+[
+{
+    "model": "accounts.user",
+    "pk": 1,
+    "fields": {
+        "password": "pbkdf2_sha256$600000$DCgCQRA6R57PicaatiWGHU$S65r6Yedgkfv/exr8gFZogpyuCgortmrloq7+LBdnyY=",
+        "last_login": "2025-01-07T14:17:06.242Z",
+        "is_superuser": true,
+        "username": "admin",
+        "first_name": "",
+        "last_name": "",
+        "email": "",
+        "is_staff": true,
+        "is_active": true,
+        "date_joined": "2025-01-07T14:10:46.508Z",
+        "groups": [],
+        "user_permissions": []
+    }
+},
+{
+    "model": "api.tabel",
+    "pk": 1,
+    "fields": {
+        "code": "tabel1",
+        "naam": "Tabel1",
+        "einddatum_geldigheid": null,
+        "beheerder_naam": "John Doe",
+        "beheerder_email": "[email protected]",
+        "beheerder_afdeling": "",
+        "beheerder_organisatie": ""
+    }
+},
+{
+    "model": "api.item",
+    "pk": 1,
+    "fields": {
+        "tabel": 1,
+        "code": "option1",
+        "naam": "Option 1",
+        "begindatum_geldigheid": "2025-01-07T14:17:53Z",
+        "einddatum_geldigheid": null,
+        "aanvullende_gegevens": null
+    }
+},
+{
+    "model": "api.item",
+    "pk": 2,
+    "fields": {
+        "tabel": 1,
+        "code": "option2",
+        "naam": "Option 2",
+        "begindatum_geldigheid": "2025-01-07T14:17:59Z",
+        "einddatum_geldigheid": null,
+        "aanvullende_gegevens": null
+    }
+}
+]

pyright.pyproject.toml

@@ -32,6 +32,7 @@ include = [
     # Interaction with the outside world
     "src/openforms/contrib/zgw/service.py",
     "src/openforms/contrib/objects_api/",
+    "src/openforms/contrib/referentielijsten/",
     # Emails
     "src/openforms/emails/templatetags/cosign_information.py",
     # Logging

src/openforms/contrib/referentielijsten/client.py

@@ -0,0 +1,35 @@
+from functools import partial
+from typing import Any, TypedDict
+
+from django.core.cache import cache
+
+from ape_pie import APIClient
+from zgw_consumers.service import pagination_helper
+
+REFERENTIELIJSTEN_LOOKUP_CACHE_TIMEOUT = 5 * 60
+
+
+class TabelItem(TypedDict):
+    code: str
+    naam: str
+    begindatumGeldigheid: str  # ISO 8601 datetime string
+    einddatumGeldigheid: str | None  # ISO 8601 datetime string
+    aanvullendeGegevens: Any
+
+
+class ReferentielijstenClient(APIClient):
+    def get_items_for_tabel(self, code: str) -> list[TabelItem]:
+        response = self.get("items", params={"tabel__code": code}, timeout=10)
+        response.raise_for_status()
+        data = response.json()
+        all_data = list(pagination_helper(self, data))
+        return all_data
+
+    def get_items_for_tabel_cached(self, code: str) -> list[TabelItem]:
+        result = cache.get_or_set(
+            key=f"referentielijsten|get_items_for_tabel|code:{code}",
+            default=partial(self.get_items_for_tabel, code),
+            timeout=REFERENTIELIJSTEN_LOOKUP_CACHE_TIMEOUT,
+        )
+        assert result is not None
+        return result

src/openforms/formio/dynamic_config/dynamic_options.py

@@ -6,18 +6,20 @@
 from glom import assign, glom
 from json_logic import jsonLogic
 
+from openforms.api.exceptions import ServiceUnavailable
 from openforms.logging import logevent
 from openforms.submissions.models import Submission
 from openforms.typing import DataMapping, JSONValue
 
 from ..typing import Component
+from .referentielijsten import fetch_options_from_referentielijsten
 
 
-def normalise_option(option: JSONValue) -> JSONValue:
+def normalise_option(option: JSONValue) -> tuple[JSONValue, JSONValue]:
     if not isinstance(option, list):
-        return [option, option]
+        return (option, option)
 
-    return option[:2]
+    return (option[0], option[1])
 
 
 def is_or_contains_none(option: JSONValue) -> bool:
@@ -26,29 +28,23 @@ def is_or_contains_none(option: JSONValue) -> bool:
     return option is None
 
 
-def escape_option(option: JSONValue) -> list[str]:
-    return [escape(item) for item in option]
+def escape_option(option: tuple[JSONValue, JSONValue]) -> tuple[str, str]:
+    return (escape(str(option[0])), escape(str(option[1])))
 
 
 def deduplicate_options(
-    options: JSONValue,
-) -> JSONValue:
+    options: list[tuple[str, str]],
+) -> list[tuple[str, str]]:
     new_options = []
     for option in options:
         if option not in new_options:
             new_options.append(option)
     return new_options
 
 
-def add_options_to_config(
-    component: Component,
-    data: DataMapping,
-    submission: Submission,
-    options_path: str = "values",
-) -> None:
-    if glom(component, "openForms.dataSrc", default=None) != "variable":
-        return
-
+def get_options_from_variable(
+    component: Component, data: DataMapping, submission: Submission
+) -> list[tuple[str, str]] | None:
     items_expression = glom(component, "openForms.itemsExpression")
     items_array = jsonLogic(items_expression, data)
     if not items_array:
@@ -80,7 +76,10 @@ def add_options_to_config(
             % {"items_expression": json.dumps(items_expression)},
         )
 
-    normalised_options = [normalise_option(option) for option in not_none_options]
+    normalised_options: list[tuple[JSONValue, JSONValue]] = [
+        normalise_option(option) for option in not_none_options
+    ]
+
     if any(
         isinstance(item_key, (dict, list)) or isinstance(item_label, (dict, list))
         for item_key, item_label in normalised_options
@@ -97,12 +96,37 @@ def add_options_to_config(
 
     escaped_options = [escape_option(option) for option in normalised_options]
     deduplicated_options = deduplicate_options(escaped_options)
+
+    return deduplicated_options
+
+
+def add_options_to_config(
+    component: Component,
+    data: DataMapping,
+    submission: Submission,
+    options_path: str = "values",
+) -> None:
+    data_src = glom(component, "openForms.dataSrc", default=None)
+    match data_src:
+        case "referentielijsten":
+            items_array = fetch_options_from_referentielijsten(component, submission)
+            if not items_array:
+                raise ServiceUnavailable(
+                    "Could not retrieve options from Referentielijsten API.",
+                )
+        case "variable":
+            items_array = get_options_from_variable(component, data, submission)
+            if items_array is None:
+                return
+        case _:
+            return
+
     assign(
         component,
         options_path,
         [
             {"label": escaped_label, "value": escaped_key}
-            for escaped_key, escaped_label in deduplicated_options
+            for escaped_key, escaped_label in items_array
         ],
         missing=dict,
     )

src/openforms/formio/dynamic_config/referentielijsten.py

@@ -0,0 +1,71 @@
+from django.utils.translation import gettext as _
+
+from glom import glom
+from requests.exceptions import RequestException
+from zgw_consumers.client import build_client
+from zgw_consumers.models import Service
+
+from openforms.contrib.referentielijsten.client import ReferentielijstenClient
+from openforms.logging import logevent
+from openforms.submissions.models import Submission
+
+from ..typing import Component
+
+
+def fetch_options_from_referentielijsten(
+    component: Component, submission: Submission
+) -> list[tuple[str, str]] | None:
+    service_slug = glom(component, "openForms.service", default=None)
+    code = glom(component, "openForms.code", default=None)
+    if not service_slug:
+        logevent.form_configuration_error(
+            submission.form,
+            component,
+            _(
+                "Cannot fetch from Referentielijsten API, because no `service` is configured."
+            ),
+        )
+        return
+
+    if not code:
+        logevent.form_configuration_error(
+            submission.form,
+            component,
+            _(
+                "Cannot fetch from Referentielijsten API, because no `code` is configured."
+            ),
+        )
+        return
+
+    try:
+        service = Service.objects.get(slug=service_slug)
+    except Service.DoesNotExist:
+        logevent.form_configuration_error(
+            submission.form,
+            component,
+            _(
+                "Cannot fetch from Referentielijsten API, service with {service_slug} does not exist."
+            ).format(service_slug=service_slug),
+        )
+        return
+
+    try:
+        with build_client(service, client_factory=ReferentielijstenClient) as client:
+            result = client.get_items_for_tabel_cached(code)
+    except RequestException as e:
+        logevent.referentielijsten_failure_response(
+            submission.form,
+            component,
+            _(
+                "Exception occurred while fetching from Referentielijsten API: {exception}."
+            ).format(exception=e),
+        )
+        return
+    else:
+        if not result:
+            logevent.referentielijsten_failure_response(
+                submission.form,
+                component,
+                _("No results found from Referentielijsten API."),
+            )
+        return [[item["code"], item["naam"]] for item in result]