Add fastapi app

arpav_ppcv/config.py

@@ -0,0 +1,150 @@
+import logging
+import re
+from pathlib import Path
+
+import pydantic
+from pydantic_settings import (
+    BaseSettings,
+    SettingsConfigDict
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ContactSettings(pydantic.BaseModel):
+    name: str = "[email protected]"
+    url: str = "http://geobeyond.it"  # noqa
+    email: str = "[email protected]"
+
+
+class ThreddsDatasetSettings(pydantic.BaseModel):
+    thredds_url_pattern: str
+    unit: str | None = None
+    palette: str
+    range: list[float]
+    allowed_values: dict[str, list[str]] | None = None
+
+    @pydantic.model_validator(mode="after")
+    def strip_slashes_from_urls(self):
+        self.thredds_url_pattern = self.thredds_url_pattern.strip("/")
+        return self
+
+    @pydantic.computed_field()
+    @property
+    def dataset_id_pattern(self) -> str:
+        id_parts = ["{identifier}"]
+        for match_obj in re.finditer(r"(\{\w+\})", self.thredds_url_pattern):
+            id_parts.append(match_obj.group(1))
+        return "-".join(id_parts)
+
+    def get_dynamic_id_parameters(self, dataset_id: str) -> dict[str, str]:
+        pattern_parts = re.finditer(
+            r"\{(\w+)\}",
+            self.dataset_id_pattern.partition("-")[-1])
+        id_parts = dataset_id.split("-")[1:]
+        result = {}
+        for index, pattern_match_obj in enumerate(pattern_parts):
+            id_part = id_parts[index]
+            name = pattern_match_obj.group(1)
+            result[name] = id_part
+        return result
+
+    def validate_dataset_id(self, dataset_id: str) -> dict[str, str]:
+        id_parameters = self.get_dynamic_id_parameters(dataset_id)
+        logger.debug(f"{id_parameters=}")
+        allowed = self.allowed_values or {}
+        for name, value in id_parameters.items():
+            if value not in allowed.get(name, []):
+                raise ValueError(
+                    f"Invalid dataset identifier: {name!r} cannot take the "
+                    f"value {value!r}"
+                )
+        return id_parameters
+
+
+class ThreddsServerSettings(pydantic.BaseModel):
+    base_url: str = "http://localhost:8080/thredds"
+    wms_service_url_fragment: str = "wms"
+    netcdf_subset_service_url_fragment: str = "ncss/grid"  # noqa
+    datasets: dict[str, ThreddsDatasetSettings] = pydantic.Field(
+        default_factory=dict)
+    uncertainty_visualization_scale_range: tuple[float, float] = pydantic.Field(
+        default=(0, 9))
+
+    @pydantic.model_validator(mode="after")
+    def strip_slashes_from_urls(self):
+        self.base_url = self.base_url.strip("/")
+        self.wms_service_url_fragment = (
+            self.wms_service_url_fragment.strip("/"))
+        self.netcdf_subset_service_url_fragment = (
+            self.netcdf_subset_service_url_fragment.strip("/"))
+        return self
+
+    @pydantic.model_validator(mode="after")
+    def validate_dataset_config_ids(self):
+        illegal_strings = (
+            "-",
+        )
+        for ds_conf_id in self.datasets.keys():
+            for patt in illegal_strings:
+                if patt in ds_conf_id:
+                    raise ValueError(
+                        f"Invalid dataset identifier: {ds_conf_id!r} - these patterns "
+                        f"are not allowed to be part of a dataset "
+                        f"identifier: {', '.join(repr(p) for p in illegal_strings)}"
+                    )
+        return self
+
+
+class DjangoEmailSettings(pydantic.BaseModel):
+    host: str = "localhost"
+    host_user: str = "user"
+    host_password: str = "password"
+    port: int = 587
+
+
+class DjangoThreddsSettings(pydantic.BaseModel):
+    host: str = "localhost"
+    auth_url: str = (
+        "https://thredds.arpa.veneto.it/thredds/restrictedAccess/dati_accordo")
+    port: int = 8080
+    user: str = 'admin'
+    password: str = 'admin'
+    proxy: str = 'http://proxy:8089/thredds/'
+
+
+class DjangoAppSettings(pydantic.BaseModel):
+    settings_module: str = "djangoapp.settings"
+    secret_key: str = "changeme"
+    mount_prefix: str = "/legacy"
+    static_root: Path = Path.home() / "django_static"
+    static_mount_prefix: str = "/static/legacy"
+    db_engine: str = "django.contrib.gis.db.backends.postgis"
+    db_dsn: pydantic.PostgresDsn = pydantic.PostgresDsn(
+        "postgresql://django_user:django_password@localhost:5432/django_db")
+    email: DjangoEmailSettings = DjangoEmailSettings()
+    redis_dsn: pydantic.RedisDsn = pydantic.RedisDsn("redis://localhost:6379")
+    thredds: DjangoThreddsSettings = DjangoThreddsSettings()
+
+
+
+class ArpavPpcvSettings(BaseSettings):  # noqa
+    model_config = SettingsConfigDict(
+        env_prefix="ARPAV_PPCV__",  # noqa
+        env_nested_delimiter="__",
+    )
+
+    debug: bool = False
+    bind_host: str = "127.0.0.1"
+    bind_port: int = 5001
+    public_url: str = "http://localhost:5001"
+    contact: ContactSettings = ContactSettings()
+    thredds_server: ThreddsServerSettings = ThreddsServerSettings()
+    v1_mount_prefix: str = "/v1"
+    v2_mount_prefix: str = "/v2"
+    django_app: DjangoAppSettings = DjangoAppSettings()
+    uvicorn_log_config_file: Path | None = None
+
+
+def get_settings() -> ArpavPpcvSettings:
+    return ArpavPpcvSettings()

arpav_ppcv/main.py

@@ -1,80 +1,110 @@
 """Command-line interface for the project."""
 
-import enum
 import logging
+import os
+import sys
 from typing import (
     Annotated,
     Optional
 )
 from pathlib import Path
 
 import anyio
+import django
 import httpx
 import typer
+from django.conf import settings as django_settings
+from django.core import management
+from django.utils.module_loading import import_string
 
 from .thredds import crawler
+from .webapp.legacy.django_settings import get_custom_django_settings
+from . import config
 
+app = typer.Typer()
+dev_app = typer.Typer()
+app.add_typer(dev_app, name="dev")
 
-class KnownCatalogIdentifier(enum.Enum):
-    THIRTY_YEAR_ANOMALY_5_MODEL_AVERAGE = "30y-anomaly-ensemble"
-    THIRTY_YEAR_ANOMALY_TEMPERATURE_PRECIPITATION = "30y-anomaly-tas-pr"
-    THIRTY_YEAR_ANOMALY_CLIMATIC_INDICES = "30y-anomaly-climate-idx"
-    YEARLY_ANOMALY_5_MODEL_AVERAGE_TAS_PR = "anomaly-ensemble-tas-pr"
-    YEARLY_ABSOLUTE_5_MODEL_AVERAGE = "yearly-ensemble-absolute"
-    YEARLY_ANOMALY_EC_EARTH_CCLM4_8_17 = "anomaly-ec-earth-cclm4-8-17"
-    YEARLY_ABSOLUTE_EC_EARTH_CCLM4_8_17 = "yearly-ec-earth-cclm4-8-17-absolute"
-    YEARLY_ANOMALY_EC_EARTH_RACM022E = "anomaly-ec-earth-racm022e"  # noqa
-    YEARLY_ABSOLUTE_EC_EARTH_RACM022E = "yearly-ec-earth-racm022e-absolute"  # noqa
-    YEARLY_ANOMALY_EC_EARTH_RCA4 = "anomaly-ec-earth-rca4"
-    YEARLY_ABSOLUTE_EC_EARTH_RCA4 = "yearly-ec-earth-rca4-absolute"
-    YEARLY_ANOMALY_HADGEM2_ES_RACMO22E = "anomaly-hadgem2-es-racmo22e"  # noqa
-    YEARLY_ABSOLUTE_HADGEM2_ES_RACMO22E = "yearly-hadgem2-es-racmo22e-absolute"  # noqa
-    YEARLY_ANOMALY_MPI_ESM_LR_REMO2009 = "anomaly-mpi-esm-lr-remo2009"
-    YEARLY_ABSOLUTE_MPI_ESM_LR_REMO2009 = "yearly-mpi-esm-lr-remo2009-absolute"
 
+@app.callback()
+def base_callback(ctx: typer.Context) -> None:
+    ctx_obj = ctx.ensure_object(dict)
+    settings = config.get_settings()
+    ctx_obj.update(
+        {
+            "settings": settings,
+        }
+    )
+    logging.basicConfig(level=logging.DEBUG if settings.debug else logging.INFO)
 
-def _get_catalog_url(catalog_identifier: KnownCatalogIdentifier) -> str:
-    return {
-        KnownCatalogIdentifier.THIRTY_YEAR_ANOMALY_5_MODEL_AVERAGE: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/ensembletwbc/clipped"),
-        KnownCatalogIdentifier.THIRTY_YEAR_ANOMALY_TEMPERATURE_PRECIPITATION: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/taspr5rcm/clipped"),
-        KnownCatalogIdentifier.THIRTY_YEAR_ANOMALY_CLIMATIC_INDICES: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/indici5rcm/clipped"),
-        KnownCatalogIdentifier.YEARLY_ANOMALY_5_MODEL_AVERAGE_TAS_PR: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/ens5ym/clipped"),
-        KnownCatalogIdentifier.YEARLY_ABSOLUTE_5_MODEL_AVERAGE: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/ensymbc/clipped"),
-        KnownCatalogIdentifier.YEARLY_ANOMALY_EC_EARTH_CCLM4_8_17: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/EC-EARTH_CCLM4-8-17ym/clipped"),
-        KnownCatalogIdentifier.YEARLY_ABSOLUTE_EC_EARTH_CCLM4_8_17: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/EC-EARTH_CCLM4-8-17ymbc/clipped"),
-        KnownCatalogIdentifier.YEARLY_ANOMALY_EC_EARTH_RACM022E: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/EC-EARTH_RACMO22Eym/clipped"),
-        KnownCatalogIdentifier.YEARLY_ABSOLUTE_EC_EARTH_RACM022E: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/EC-EARTH_RACMO22Eymbc/clipped"),
-        KnownCatalogIdentifier.YEARLY_ANOMALY_EC_EARTH_RCA4: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/EC-EARTH_RCA4ym/clipped"),
-        KnownCatalogIdentifier.YEARLY_ABSOLUTE_EC_EARTH_RCA4: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/EC-EARTH_RCA4ymbc/clipped"),
-        KnownCatalogIdentifier.YEARLY_ANOMALY_HADGEM2_ES_RACMO22E: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/HadGEM2-ES_RACMO22Eym/clipped"),
-        KnownCatalogIdentifier.YEARLY_ABSOLUTE_HADGEM2_ES_RACMO22E: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/HadGEM2-ES_RACMO22Eymbc/clipped"),
-        KnownCatalogIdentifier.YEARLY_ANOMALY_MPI_ESM_LR_REMO2009: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/MPI-ESM-LR_REMO2009ym/clipped"),
-        KnownCatalogIdentifier.YEARLY_ABSOLUTE_MPI_ESM_LR_REMO2009: (
-            "https://thredds.arpa.veneto.it/thredds/catalog/MPI-ESM-LR_REMO2009ymbc/clipped"),
-    }[catalog_identifier]
 
+@app.command()
+def run_server(ctx: typer.Context):
+    """Run the uvicorn server.
 
-app = typer.Typer()
+    Example (dev) invocation:
 
+    ```
+    bash -c 'set -o allexport; source sample_env.env; set +o allexport; poetry run arpav-ppcv.run-server'
+    ```
+    """
+    # NOTE: we explicitly do not use uvicorn's programmatic running abilities here
+    # because they do not work correctly when called outside an
+    # `if __name__ == __main__` guard and when using its debug features.
+    # For more detail check:
+    #
+    # https://github.com/encode/uvicorn/issues/1045
+    #
+    # This solution works well both in development (where we want to use reload)
+    # and in production, as using os.execvp is actually similar to just running
+    # the standard `uvicorn` cli command (which is what uvicorn docs recommend).
+    settings: config.ArpavPpcvSettings = ctx.obj["settings"]
+    uvicorn_args = [
+        "uvicorn",
+        "arpav_ppcv.webapp.app:create_app",
+        f"--port={settings.bind_port}",
+        f"--host={settings.bind_host}",
+        "--factory",
+    ]
+    if settings.debug:
+        uvicorn_args.extend(
+            [
+                "--reload",
+                f"--reload-dir={str(Path(__file__).parent)}",
+                "--log-level=debug",
+            ]
+        )
+    else:
+        uvicorn_args.extend(["--log-level=info"])
+    if (log_config_file := settings.uvicorn_log_config_file) is not None:
+        uvicorn_args.append(f"--log-config={str(log_config_file)}")
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os.execvp("uvicorn", uvicorn_args)
 
-@app.command()
+
+@app.command(
+    context_settings={
+        "allow_extra_args": True,
+        "ignore_unknown_options": True,
+    }
+)
+def django_admin(ctx: typer.Context, command: str):
+    """Run a django command.
+
+    Run a django management command, just like if you were calling django-admin.
+    """
+    settings: config.ArpavPpcvSettings = ctx.obj["settings"]
+    custom_django_settings = get_custom_django_settings(settings)
+    django_settings.configure(**custom_django_settings)
+    django.setup()
+    management.call_command(command, *ctx.args)
+
+
+@dev_app.command()
 def import_thredds_datasets(
         catalog: Annotated[
-            Optional[list[KnownCatalogIdentifier]],
+            Optional[list[crawler.KnownCatalogIdentifier]],
             typer.Option(default_factory=list)
         ],
         output_base_dir: Annotated[
@@ -100,23 +130,13 @@ def import_thredds_datasets(
                 )
             )
         ] = False,
-        verbose: Annotated[
-            Optional[bool],
-            typer.Option(
-                help=(
-                        "Verbose output"
-                )
-            )
-        ] = False
 ):
-    print(f"{locals()=}")
-    if verbose:
-        logging.basicConfig(level=logging.INFO)
-    relevant_catalogs = catalog if len(catalog) > 0 else list(KnownCatalogIdentifier)
+    relevant_catalogs = (
+        catalog if len(catalog) > 0 else list(crawler.KnownCatalogIdentifier))
     client = httpx.Client()
     for relevant_catalog in relevant_catalogs:
         print(f"Processing catalog {relevant_catalog.value!r}...")
-        catalog_url = _get_catalog_url(relevant_catalog)
+        catalog_url = crawler.get_catalog_url(relevant_catalog)
         contents = crawler.discover_catalog_contents(catalog_url, client)
         print(f"Found {len(contents.get_public_datasets(wildcard_filter))} datasets")
         if output_base_dir is not None:
@@ -127,4 +147,4 @@ def import_thredds_datasets(
                 contents,
                 wildcard_filter,
                 force_download,
-            )
+            )

arpav_ppcv/operations/thredds.py

@@ -0,0 +1,29 @@
+import itertools
+import re
+
+from .. import config
+
+
+def list_dataset_configurations(
+        settings: config.ArpavPpcvSettings
+) -> dict[str, config.ThreddsDatasetSettings]:
+    return settings.thredds_server.datasets
+
+
+def list_dataset_identifiers(
+        dataset_config_identifier: str,
+        dataset_config: config.ThreddsDatasetSettings
+) -> list[str]:
+    pattern_parts = re.findall(
+        r"\{(\w+)\}",
+        dataset_config.dataset_id_pattern.partition("-")[-1])
+    values_to_combine = []
+    for part in pattern_parts:
+        part_allowed_values = dataset_config.allowed_values.get(part, [])
+        values_to_combine.append(part_allowed_values)
+    result = []
+    for combination in itertools.product(*values_to_combine):
+        dataset_id = "-".join((dataset_config_identifier, *combination))
+        result.append(dataset_id)
+    return result
+