Update docs to match new config

.github/workflows/backend.yml

@@ -10,22 +10,17 @@ defaults:
     working-directory: backend
 
 env:
-  DJANGO_SETTINGS_MODULE: metagrid.config.local
-  DOMAIN_NAME: http://localhost:8000
-  DJANGO_SECURE_SSL_REDIRECT: False
-  CORS_ORIGIN_WHITELIST: http://localhost:3000
-  KEYCLOAK_URL: https://esgf-login.ceda.ac.uk/
-  KEYCLOAK_REALM: esgf
-  KEYCLOAK_CLIENT_ID: metagrid-localhost
-  DATABASE_URL: pgsql://postgres:postgres@localhost:5432/postgres
-  REACT_APP_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
-  REACT_APP_WGET_API_URL: https://esgf-node.llnl.gov/esg-search/wget
-  REACT_APP_ESGF_NODE_STATUS_URL: https://aims2.llnl.gov/metagrid-backend/proxy/status
-  REACT_APP_ESGF_SOLR_URL: https://esgf-fedtest.llnl.gov/solr
-  DJANGO_LOGIN_REDIRECT_URL: http://localhost:3000/search
-  DJANGO_LOGOUT_REDIRECT_URL: http://localhost:3000/search
-  GLOBUS_CLIENT_KEY: ${{ secrets.GLOBUS_KEY }}
-  GLOBUS_CLIENT_SECRET: ${{ secrets.GLOBUS_SECRET }}
+  DJANGO_SETTINGS_MODULE: config.settings
+  PGHOST: localhost
+  PGPASSWORD: postgres
+  PGUSER: postgres
+  METAGRID_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
+  METAGRID_WGET_URL: https://esgf-node.llnl.gov/esg-search/wget
+  METAGRID_STATUS_URL: https://esgf-node.llnl.gov/proxy/status
+  METAGRID_SOLR_URL: https://esgf-node.llnl.gov/esg-search
+  METAGRID_SOCIAL_AUTH_GLOBUS_KEY: ${{ secrets.GLOBUS_KEY }}
+  METAGRID_SOCIAL_AUTH_GLOBUS_SECRET: ${{ secrets.GLOBUS_SECRET }}
+  METAGRID_GLOBUS_CLIENT_ID: 21982de0-2b7a-4ba8-bef5-5606ae098201
 
 jobs:
   build:

.gitignore

@@ -183,6 +183,7 @@ tags
 .env
 .envs/*
 !.envs/.local/
+docker-compose.*-overlay.yml
 
 # Documentation artifacts
 docs/site

.pre-commit-config.yaml

@@ -1,44 +1,81 @@
 exclude: "docs|node_modules|migrations|.git|.tox"
-default_stages: [commit]
+default_stages: [ commit ]
 fail_fast: true
 
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: check-yaml
 
-  # Back-end
-  # ------------------------------------------------------------------------------
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
-    hooks:
-      - id: flake8
-        args: ["--config=backend/setup.cfg"]
-        additional_dependencies: [flake8-isort]
+# Back-end
+# ------------------------------------------------------------------------------
+- repo: https://github.com/pycqa/flake8
+  rev: 7.1.1
+  hooks:
+  - id: flake8
+    args: [ "--config=backend/setup.cfg" ]
+    additional_dependencies: [ flake8-isort ]
 
-  - repo: https://github.com/psf/black
-    rev: 24.10.0
-    hooks:
-      - id: black
-        args: ["--config=backend/pyproject.toml"]
+- repo: https://github.com/psf/black
+  rev: 24.10.0
+  hooks:
+  - id: black
+    args: [ "--config=backend/pyproject.toml" ]
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.12.0
-    hooks:
-      - id: mypy
-        additional_dependencies: ["types-requests"]
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.12.0
+  hooks:
+  - id: mypy
+    additional_dependencies: [ "types-requests" ]
 
-  # Front-end
-  # ------------------------------------------------------------------------------
-  # Runs the local node `lint` script to avoid dependencies being out of sync with `frontend/package.json`
-  # https://github.com/pre-commit/pre-commit/issues/945
-  - repo: local
-    hooks:
-      - id: eslint_and_tsc
-        name: eslint_and_tsc
-        language: system
-        files: .+(js|jsx|ts|tsx|json)$
-        entry: bash -c 'cd frontend && yarn precommit'
+- repo: https://github.com/radeklat/settings-doc
+  rev: '4.3.1'
+  hooks:
+  - id: settings-doc-markdown
+    name: Generate backend settings markdown
+    language: system
+    entry: env PYTHONPATH=$PYTHONPATH:backend settings-doc generate --output-format markdown
+    args:
+    - '--class'
+    - 'config.settings.site_specific.MetagridBackendSettings'
+    - '--update'
+    - 'docs/docs/users/configurable_environment_variables.md'
+    - '--between'
+    - '<!-- start generated backend settings markdown -->'
+    - '<!-- end generated backend settings markdown -->'
+    - '--templates'
+    - 'docs/custom_templates'
+    - '--heading-offset'
+    - '3'
+
+  - id: settings-doc-markdown
+    name: Generate frontend settings markdown
+    language: system
+    entry: env PYTHONPATH=$PYTHONPATH:backend settings-doc generate --output-format markdown
+    args:
+    - '--class'
+    - 'config.settings.site_specific.MetagridFrontendSettings'
+    - '--update'
+    - 'docs/docs/users/configurable_environment_variables.md'
+    - '--between'
+    - '<!-- start generated frontend settings markdown -->'
+    - '<!-- end generated frontend settings markdown -->'
+    - '--templates'
+    - 'docs/custom_templates'
+    - '--heading-offset'
+    - '3'
+
+# Front-end
+# ------------------------------------------------------------------------------
+# Runs the local node `lint` script to avoid dependencies being out of sync with `frontend/package.json`
+# https://github.com/pre-commit/pre-commit/issues/945
+- repo: local
+  hooks:
+  - id: eslint_and_tsc
+    name: eslint_and_tsc
+    language: system
+    files: .+(js|jsx|ts|tsx|json)$
+    entry: bash -c 'cd frontend && yarn precommit'

backend/Dockerfile

@@ -22,8 +22,7 @@ RUN pip3 install --no-cache-dir -r /requirements/local.txt
 
 COPY . /app
 
-RUN python /app/manage.py collectstatic --noinput \
-    && mkdir -p /app/staticfiles/.well-known \
+RUN mkdir -p /app/staticfiles/.well-known \
     && cyclonedx-py requirements requirements/base.txt --output-format json --outfile /app/staticfiles/.well-known/bom
 
 WORKDIR /app

backend/config/settings/__init__.py

@@ -0,0 +1,24 @@
+from typing import Any, Iterable
+
+from .site_specific import MetagridBackendSettings, MetagridFrontendSettings
+from .static import DjangoStaticSettings
+
+
+def __dir__() -> Iterable[str]:
+    """The list of available options are retrieved from
+    the dict view of our DjangoSettings object.
+    """
+    return (
+        DjangoStaticSettings.model_fields.keys()
+        | MetagridBackendSettings.model_fields.keys()
+        | MetagridFrontendSettings.model_fields.keys()
+    )
+
+
+def __getattr__(name: str) -> Any:
+    """Turn the module access into a DjangoSettings access"""
+    _combined_settings = DjangoStaticSettings().model_dump()
+    _combined_settings |= MetagridBackendSettings().model_dump()
+    _combined_settings |= MetagridFrontendSettings().model_dump()
+
+    return _combined_settings[name]

backend/config/settings/base.py

@@ -392,10 +392,6 @@
     "REACT_APP_AUTHENTICATION_METHOD": env(
         "REACT_APP_AUTHENTICATION_METHOD", default="keycloak"
     ),
-    "REACT_APP_GLOBUS_REDIRECT": env(
-        "REACT_APP_GLOBUS_REDIRECT",
-        default="http://localhost:8080/cart/items",
-    ),
     "REACT_APP_GLOBUS_CLIENT_ID": env(
         "REACT_APP_GLOBUS_CLIENT_ID", default="frontend"
     ),

backend/config/settings/site_specific.py

@@ -0,0 +1,96 @@
+import secrets
+from typing import Literal, Optional, Sequence
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class MetagridBackendSettings(BaseSettings):
+    """Settings used by the Metagrid backend directly"""
+
+    model_config = SettingsConfigDict(
+        env_prefix="METAGRID_",
+        case_sensitive=True,
+    )
+
+    SEARCH_URL: str = Field(
+        description="The URL at which the ESG-Search api can be queried. A suitable endpoint will return XML in the browser.",
+        examples=["https://esgf-node.llnl.gov/esg-search/search"],
+    )
+    WGET_URL: str = Field(
+        description="The URL at which the ESG-Search wget endpoint can be reached.",
+        examples=["https://esgf-node.llnl.gov/esg-search/wget"],
+    )
+    STATUS_URL: str = Field(
+        description="The URL at which the backend can reach the Node Status API.",
+        examples=["https://esgf-node.llnl.gov/proxy/status"],
+    )
+    KEYCLOAK_CLIENT_ID: str = Field(
+        default="metagrid-localhost",
+        examples=["metagrid-localhost"],
+        description="Used in data migration to register Keycloak social app",
+    )
+    ADMIN_URL: str = Field(
+        examples=["C5PhMrfRDd0x5RY-og2Tk_SwLFs4xWge1j8iM4wx6XQ/"],
+        description="""The default URL for the Django administration interface is `/admin`. This is a well-known fact and attackers will try to access this URL on your site. The solution is to change the URL of the administration interface. If not set, a random secure path will be generated using `secrets.token_urlsafe()` and can be retrieved using a command similar to the following:
+>     `docker compose -f docker-compose.yml -f docker-compose.SITENAME-overlay.yml run --rm django python manage.py get_setting ADMIN_URL`
+
+>     !!! warning "Note"
+>         This path will change each time the backend is restarted.""",
+        default_factory=secrets.token_urlsafe,
+    )
+    ADMINS: Sequence[tuple[str, str]] = Field(
+        default=[],
+        examples=[
+            [
+                ("Author", "[email protected]"),
+                ("Author", "[email protected]"),
+            ]
+        ],
+        description="A list of all the people who get code error notifications. When `DEBUG=False` and `AdminEmailHandler` is configured in `LOGGING` (done by default), Django emails these people the details of exceptions raised in the request/response cycle. Each item in the list should be a tuple of (Full name, email address). "
+        "Reference: <https://docs.djangoproject.com/en/5.1/ref/settings/#admins>",
+    )
+    SOCIAL_AUTH_GLOBUS_KEY: str = Field(
+        examples=["94c44808-9efd-4236-bffd-1185b1071736"],
+        description="The `Client UUID` obtained by registering a `portal, science gateway, or other application you host` with Globus at <https://app.globus.org/settings/developers>",
+    )
+    SOCIAL_AUTH_GLOBUS_SECRET: str = Field(
+        examples=["6aWj3gBYsxUBO++cSXtPzbl4n/sGJdhAmtn70XRoUMA="],
+        description="A `Client Secret` associated with the Client UUID created for `SOCIAL_AUTH_GLOBUS_KEY` at https://app.globus.org/settings/developers",
+    )
+
+
+class MetagridFrontendSettings(BaseSettings):
+    """Settings retrieved by the Metagrid frontend.
+
+    These are kept separate from the Metagrid backend settings to make it easy to serialize them to json and send to the frontend.
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="METAGRID_",
+        case_sensitive=True,
+    )
+
+    AUTHENTICATION_METHOD: Literal["keycloak", "globus"] = Field(
+        default="globus",
+        description="Which authentication method to enable for user sign in on the frontend.",
+    )
+    GLOBUS_CLIENT_ID: str = Field(
+        examples=["536321f7-c0e9-462c-b5c6-34d4a3672076"],
+        description="The `Client UUID` obtained by registering a `a thick client or script that will be installed and run by users on their devices` with Globus at <https://app.globus.org/settings/developers>  This is required even if signing in with Globus is not enabled. It is used for browsing Globus Collections to which files may be sent.",
+    )
+    GLOBUS_NODES: Sequence[str] = Field(
+        default=[
+            "aims3.llnl.gov",
+            "esgf-data1.llnl.gov",
+            "esgf-data2.llnl.gov",
+            "esgf-node.ornl.gov",
+        ],
+        description="The list of data nodes known to be Globus enabled. A data node must be in this list in order to transfer files from it via Globus.",
+    )
+    KEYCLOAK_REALM: Optional[str] = None
+    KEYCLOAK_URL: Optional[str] = None
+    KEYCLOAK_CLIENT_ID: str = ""
+    HOTJAR_ID: Optional[int] = None
+    HOTJAR_SV: Optional[int] = None
+    GOOGLE_ANALYTICS_TRACKING_ID: Optional[str] = None