Adding READMEs to all directories

.env.sample

@@ -54,7 +54,6 @@ ORTHANC_ANON_URL=http://orthanc-anon:8042
 ORTHANC_ANON_USERNAME=
 ORTHANC_ANON_PASSWORD=
 ORTHANC_ANON_AE_TITLE=
-ORTHANC_ANON_HTTP_TIMEOUT=60
 ORTHANC_AUTOROUTE_ANON_TO_ENDPOINT=false
 ENABLE_DICOM_WEB=true
 STUDY_TIME_OFFSET=

.github/workflows/main.yml

@@ -28,11 +28,11 @@ concurrency:
 
 jobs:
     lint:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-24.04
         timeout-minutes: 5
         steps:
             - name: Checkout
-              uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+              uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
             - name: Run pre-commit
               uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
@@ -46,18 +46,18 @@ jobs:
                   docker compose config --quiet
 
     test:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-24.04
         timeout-minutes: 30
         strategy:
             fail-fast: false # run all tests if even if one fails
             matrix:
                 package_dir: [pixl_core, hasher, pixl_dcmd, cli, pixl_export, pixl_imaging, pytest-pixl]
 
         steps:
-            - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
             - name: Init Python
-              uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5
+              uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
               with:
                   python-version: "3.11"
                   cache: "pip"
@@ -78,21 +78,21 @@ jobs:
                   AZURE_KEY_VAULT_SECRET_NAME: test
 
             - name: Upload coverage reports to Codecov
-              uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
+              uses: codecov/codecov-action@7f8b4b4bde536c465e797be725718b88c5d95e0e # v5.1.1
               with:
                   directory: ${{ matrix.package_dir }}
               env:
                   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
     system-test:
         if: ${{ ! github.event.pull_request.draft || contains(github.event.pull_request.title, '[force-system-test]') }}
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-24.04
         timeout-minutes: 30
         steps:
-            - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
-            - uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3
+            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+            - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3
             - name: Init Python
-              uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5
+              uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
               with:
                   python-version: "3.11"
                   cache: "pip"
@@ -179,7 +179,7 @@ jobs:
 
 
             - name: Upload coverage reports to Codecov
-              uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
+              uses: codecov/codecov-action@7f8b4b4bde536c465e797be725718b88c5d95e0e # v5.1.1
               with:
                 directory: test
               env:

.renovaterc.json5

@@ -2,6 +2,36 @@
   $schema: "https://docs.renovatebot.com/renovate-schema.json",
   extends: [
     "github>UCL-ARC/.github//renovate/default-config.json",
-    "schedule:monthly",
+    ":assignAndReview(team:arc-dev)",
+    "group:allNonMajor"
   ],
-}
+  customDatasources: {
+    dicomSpec: {
+      defaultRegistryUrlTemplate: "https://dicom.nema.org/medical/dicom",
+      format: "html",
+    },
+  },
+  customManagers: [
+    {
+      customType: "regex",
+      description: "Update DICOM Spec edition used for validation",
+      fileMatch: [
+        "orthanc/orthanc-anon/plugin/download_dicom_spec.py",
+        "pixl_dcmd/src/pixl_dcmd/main.py",
+      ],
+      matchStrings: [
+        'edition\\s?=\\s?"(?<currentValue>.*?)"\n',
+        '.*\\(edition\\s?=\\s?"(?<currentValue>.*?)"\\)\n',
+      ],
+      depNameTemplate: "dicomSpec",
+      datasourceTemplate: "custom.dicomSpec",
+    },
+  ],
+  packageRules: [
+    {
+      matchDatasources: ["custom.dicomSpec"],
+      extractVersion: "/medical/dicom/(?<version>\\d{4}[a-z])/",
+      versioning: "loose",
+    }
+  ]
+}

bin/README.md

@@ -1,6 +1,24 @@
 ## 'PIXL/bin' Directory Contents
 
-### Subdirectories
+<details>
+<summary>
+<h3> Subdirectories with links to the relevant README </h3> 
+
+</summary>
 
 [linters](./linters/README.md)
 
+</details>
+
+<details>
+<summary>
+<h3> Files </h3> 
+
+</summary>
+
+| **User docs** |
+| :--- | 
+README.md
+
+</details>
+

bin/linters/README.md

@@ -1,6 +1,16 @@
+A directory that contains the files used for linting.
+
 ## 'PIXL/bin/linters' Directory Contents
 
-### Files
+<details>
+<summary>
+<h3> Files </h3> 
+
+</summary>
+
+| **Code** | **User docs** |
+| :--- | :--- |
+| check_headers_exist.sh | README.md |
 
-check_headers_exist.sh
+</details>
 

cli/README.md

@@ -145,15 +145,29 @@ pytest -vs tests/test_docker_commands.py #e.g., for particular tests
 
 ## 'PIXL/cli' Directory Contents
 
-### Subdirectories
+<details>
+<summary>
+<h3> Subdirectories with links to the relevant README </h3> 
+
+</summary>
 
 [src](./src/README.md)
 
 [tests](./tests/README.md)
 
-### Files
+</details>
+
+<details>
+<summary>
+<h3> Files </h3> 
+
+</summary>
+
+| **Configuration** | **User docs** |
+| :--- | :--- |
+| pyproject.toml | README.md |
+
+</details>
 
-pyproject.toml
 
-README.md
 

cli/pyproject.toml

@@ -9,7 +9,7 @@ classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
     "core==0.2.0rc0",
     "click==8.1.7",
-    "tqdm==4.66.4",
+    "tqdm==4.67.1",
 ]
 
 [project.optional-dependencies]

cli/src/README.md

@@ -1,6 +1,26 @@
+This directory contains the source files for the PIXL commands line interface.
+
 ## 'PIXL/cli/src' Directory Contents
 
-### Subdirectories
+<details>
+<summary>
+<h3> Subdirectories with links to the relevant README </h3> 
+
+</summary>
 
 [pixl_cli](./pixl_cli/README.md)
 
+</details>
+
+<details>
+<summary>
+<h3> Files </h3> 
+
+</summary>
+
+| **User docs** |
+| :--- |
+README.md
+
+</details>
+

cli/src/pixl_cli/README.md

@@ -1,18 +1,22 @@
-## 'PIXL/cli/src/pixl_cli' Directory Contents
-
-### Files
-
-main.py
+This directory contains the files necessary to create a command line instance of PIXL.
 
-_config.py
-
-_database.py
+## 'PIXL/cli/src/pixl_cli' Directory Contents
 
-_docker_commands.py
+<details>
+<summary>
+<h3> Files </h3> 
 
-_io.py
+</summary>
 
-_message_processing.py
+| **Code** | **User docs** |
+| :--- | :--- |
+| main.py | README.md |
+| _config.py | |
+| _database.py | |
+| _docker_commands.py | |
+| _io.py | |
+| _message_processing.py | |
+| __init__.py | |
 
-__init__.py
+</details>
 

cli/src/pixl_cli/_database.py

@@ -89,8 +89,8 @@ def _filter_existing_images(
 ) -> pd.DataFrame:
     # DataFrame indices must batch when using df.isin (or df.index.isin)
     # So we re-index the DataFrames to match on the columns we want to compare
-    messages_df_reindexed = messages_df.set_index(["accession_number", "mrn", "study_date"])
-    images_df_reindexed = images_df.set_index(["accession_number", "mrn", "study_date"])
+    messages_df_reindexed = messages_df.set_index(["accession_number", "mrn", "study_uid"])
+    images_df_reindexed = images_df.set_index(["accession_number", "mrn", "study_uid"])
     keep_indices = ~messages_df_reindexed.index.isin(images_df_reindexed.index)
     return messages_df[keep_indices]
 
@@ -101,7 +101,7 @@ def _filter_exported_messages(
 ) -> pd.DataFrame:
     merged = messages_df.merge(
         images_df,
-        on=["accession_number", "mrn", "study_date"],
+        on=["accession_number", "mrn", "study_uid"],
         how="left",
         validate="one_to_one",
         suffixes=(None, None),
@@ -131,7 +131,7 @@ def all_images_for_project(project_slug: str) -> pd.DataFrame:
     PixlSession = sessionmaker(engine)
 
     query = (
-        select(Image.accession_number, Image.study_date, Image.mrn, Image.exported_at)
+        select(Image.accession_number, Image.study_uid, Image.mrn, Image.exported_at)
         .join(Extract)
         .where(Extract.slug == project_slug)
     )

cli/src/pixl_cli/_io.py

@@ -64,19 +64,25 @@ def read_patient_info(resources_path: Path) -> pd.DataFrame:
         messages_df = _load_csv(resources_path)
     else:
         messages_df = _load_parquet(resources_path)
+    # Tidy up dataframe in case of whitespace or no way to identify images
+    unique_columns = ["project_name", "mrn", "accession_number", "study_uid"]
+    filtered_df = messages_df.dropna(subset=["accession_number", "study_uid"], how="all")
+    for column in unique_columns:
+        filtered_df[column] = filtered_df[column].str.strip()
+    filtered_df = filtered_df[
+        ~(filtered_df["accession_number"].eq("") & filtered_df["study_uid"].eq(""))
+    ]
 
-    messages_df = messages_df.sort_values(by=["project_name", "study_date"])
-    messages_df = messages_df.drop_duplicates(
-        subset=["project_name", "mrn", "accession_number", "study_date"]
-    )
+    filtered_df = filtered_df.sort_values(by=["project_name", "study_date"])
+    filtered_df = filtered_df.drop_duplicates(subset=unique_columns)
 
-    if len(messages_df) == 0:
+    if len(filtered_df) == 0:
         msg = f"Failed to find any messages in {resources_path}"
         raise ValueError(msg)
 
-    logger.info("Created {} messages from {}", len(messages_df), resources_path)
+    logger.info("Created {} messages from {}", len(filtered_df), resources_path)
 
-    return messages_df
+    return filtered_df
 
 
 def _load_csv(filepath: Path) -> pd.DataFrame:
@@ -168,7 +174,6 @@ class DF_COLUMNS(StrEnum):  # noqa: N801
     "participant_id": "pseudo_patient_id",
 }
 
-
 MAP_PARQUET_TO_MESSAGE_KEYS = {
     "PrimaryMrn": "mrn",
     "AccessionNumber": "accession_number",

cli/tests/README.md

@@ -1,29 +1,33 @@
+# PIXL cli tests
 
-Remove the db container and associated data
+This directory contains the code for the tests of the PIXL command line interface.
+
+
+
+In order to remove the db container and associated data after the tests have been run use the following command:
 
 ```bash
 docker container rm pixl-test-db -v -f
 ```
 
 ## 'PIXL/cli/tests' Directory Contents
 
-### Files
-
-conftest.py
-
-README.md
-
-test_check_env.py
-
-test_database.py
-
-test_docker_commands.py
-
-test_io.py
+<details>
+<summary>
+<h3> Files </h3> 
 
-test_messages_from_files.py
+</summary>
 
-test_message_processing.py
+| **Code** | **User docs** |
+| :--- | :--- |
+| conftest.py | README.md |
+| test_check_env.py | |
+| test_database.py | |
+| test_docker_commands.py | |
+| test_io.py | |
+| test_messages_from_files.py | |
+| test_message_processing.py | |
+| test_populate.py | |
 
-test_populate.py
+</details>