Add configuration for DESPIAD project

README.md

@@ -148,6 +148,12 @@ The configuration file defines:
 
 - Project name: the `<project-slug>` name of the Project
 - The DICOM dataset modalities to retain (e.g. `["DX", "CR"]` for X-Ray studies)
+- The minimum number of instances required by a series (defaults to 1). May be set higher than 1 to filter out
+  series with a single screenshot containing patient identifiable data
+- A list of series description filters (e.g. `['loc', 'pos']`). Series with descriptions matching any of these
+  filters will be skipped
+- A list of series number filters (e.g. `[3, 4]`). Series with SeriesNumber matching any of these filters will
+  be skipped
 - The [anonymisation operations](/pixl_dcmd/README.md#tag-scheme-anonymisation) to be applied to the DICOM tags,
   by providing a file path to one or multiple YAML files.
   We currently allow two types of files:

orthanc/orthanc-anon/plugin/pixl.py

@@ -331,6 +331,7 @@ def _anonymise_study_instances(
     Return a list of the bytes of anonymised instances, and the anonymised StudyInstanceUID.
     """
     config = load_project_config(project_name)
+    series_to_skip = get_series_to_skip(zipped_study, config.min_instances_per_series)
     anonymised_instances_bytes = []
     skipped_instance_counts = defaultdict(int)
     dicom_validation_errors = {}
@@ -339,6 +340,15 @@ def _anonymise_study_instances(
         with zipped_study.open(file_info) as file:
             logger.debug("Reading file {}", file)
             dataset = dcmread(file)
+
+            if dataset.SeriesInstanceUID in series_to_skip:
+                logger.warning(
+                    "Skipping series {} for study {} due to too few instances",
+                    dataset.SeriesInstanceUID,
+                    study_info,
+                )
+                continue
+
             try:
                 anonymised_instance, instance_validation_errors = _anonymise_dicom_instance(
                     dataset, config
@@ -376,6 +386,33 @@ def _anonymise_study_instances(
     return anonymised_instances_bytes, anonymised_study_uid
 
 
+def get_series_to_skip(zipped_study: ZipFile, min_instances: int) -> set[str]:
+    """
+    Determine which series to skip based on the number of instances in the series.
+
+    If a series has fewer instances than `min_instances`, add it to a set of series to skip.
+
+    Args:
+        zipped_study: ZipFile containing the study
+        min_instances: Minimum number of instances required to include a series
+
+    """
+    if min_instances <= 1:
+        return set()
+
+    series_instances = {}
+    for file_info in zipped_study.infolist():
+        with zipped_study.open(file_info) as file:
+            logger.debug("Reading file {}", file)
+            dataset = dcmread(file)
+            if dataset.SeriesInstanceUID not in series_instances:
+                series_instances[dataset.SeriesInstanceUID] = 1
+                continue
+            series_instances[dataset.SeriesInstanceUID] += 1
+
+    return {series for series, count in series_instances.items() if count < min_instances}
+
+
 def _anonymise_dicom_instance(dataset: pydicom.Dataset, config: PixlConfig) -> tuple[bytes, dict]:
     """Anonymise a DICOM instance."""
     validation_errors = anonymise_dicom_and_update_db(dataset, config=config)

pixl_core/src/core/project_config/pixl_config_model.py

@@ -16,6 +16,7 @@
 
 from __future__ import annotations
 
+import re
 from enum import Enum
 from pathlib import Path
 from typing import Any, Optional
@@ -133,20 +134,47 @@
     """Project-specific configuration for Pixl."""
 
     project: _Project
-    series_filters: Optional[list[str]] = None
+    min_instances_per_series: Optional[int] = 1
+    series_filters: Optional[list[str]] = []  # pydantic makes a deep copy of the empty default list
+    series_number_filters: Optional[list[str]] = []
+    allowed_manufacturers: Optional[str] = ".*"
     tag_operation_files: TagOperationFiles
     destination: _Destination
 
-    def is_series_excluded(self, series_description: str) -> bool:
+    def is_series_description_excluded(self, series_description: str | None) -> bool:
         """
-        Return whether this config excludes the series with the given description
+        Return whether this config excludes the series with the given description.
+
+        Do a simple case-insensitive substring check - this data is ultimately typed by a human, and
+        different image sources may have different conventions for case conversion.
+
         :param series_description: the series description to test
         :returns: True if it should be excluded, False if not
         """
-        if self.series_filters is None or series_description is None:
+        if not self.series_filters or series_description is None:
             return False
-        # Do a simple case-insensitive substring check - this data is ultimately typed by a human,
-        # and different image sources may have different conventions for case conversion.
+
         return any(
             series_description.upper().find(filt.upper()) != -1 for filt in self.series_filters
         )
+
+    def is_series_number_excluded(self, series_number: str | None) -> bool:
+        """
+        Return whether this config excludes the series with the given number
+
+        :param series_number: the series number to test
+        :returns: True if it should be excluded, False if not
+        """
+        if not self.series_number_filters or series_number is None:
+            return False
+
+        return any(series_number.find(filt) != -1 for filt in self.series_number_filters)
+
+    def is_manufacturer_allowed(self, manufacturer: str) -> bool:
+        """
+        Check whether the manufacturer is in the allow-list.
+
+        :param manufacturer: name of the manufacturer
+        :returns: True is the manufacturer is allowed, False if not
+        """
+        return bool(re.search(rf"{self.allowed_manufacturers}", manufacturer, flags=re.IGNORECASE))

pixl_core/tests/project_config/test_project_config.py

@@ -181,4 +181,4 @@ def test_series_filtering(base_yaml_data, series_filters, test_series_desc, expe
     if series_filters is not None:
         base_yaml_data["series_filters"] = series_filters
     cfg = PixlConfig.model_validate(base_yaml_data)
-    assert cfg.is_series_excluded(test_series_desc) == expect_exclude
+    assert cfg.is_series_description_excluded(test_series_desc) == expect_exclude

pixl_dcmd/src/pixl_dcmd/main.py

@@ -57,13 +57,31 @@
 
 
 def _should_exclude_series(dataset: Dataset, cfg: PixlConfig) -> bool:
+    """
+    Check whether the dataset series should be exlucded based on its description
+    and number.
+    """
     series_description = dataset.get("SeriesDescription")
-    if cfg.is_series_excluded(series_description):
+    if cfg.is_series_description_excluded(series_description):
         logger.debug("FILTERING OUT series description: {}", series_description)
         return True
+
+    series_number = dataset.get("SeriesNumber")
+    if cfg.is_series_number_excluded(series_number):
+        logger.debug("FILTERING OUT series number: {}", series_number)
+        return True
+
     return False
 
 
+def _should_exclude_manufacturer(dataset: Dataset, cfg: PixlConfig) -> bool:
+    manufacturer = dataset.get("Manufacturer")
+    should_exclude = not cfg.is_manufacturer_allowed(manufacturer=manufacturer)
+    if should_exclude:
+        logger.debug("FILTERING out manufacturer: {}", manufacturer)
+    return should_exclude
+
+
 def anonymise_dicom_and_update_db(
     dataset: Dataset,
     *,
@@ -125,10 +143,13 @@
     )
 
     # Do before anonymisation in case someone decides to delete the
-    # Series Description tag as part of anonymisation.
+    # Series Description or Manufacturer tags as part of anonymisation.
     if _should_exclude_series(dataset, config):
         msg = "DICOM instance discarded due to its series description"
         raise PixlSkipInstanceError(msg)
+    if _should_exclude_manufacturer(dataset, config):
+        msg = "DICOM instance discarded due to its manufacturer"
+        raise PixlSkipInstanceError(msg)
     if dataset.Modality not in config.project.modalities:
         msg = f"Dropping DICOM Modality: {dataset.Modality}"
         raise PixlSkipInstanceError(msg)

projects/configs/despiad.yaml

@@ -0,0 +1,36 @@
+#  Copyright (c) 2024 University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+project:
+    name: "despiad"
+    modalities:
+      - "CT"
+      - "PT"
+
+tag_operation_files:
+    base:
+        - "base.yaml"
+        - "ct.yaml"
+        - "pet.yaml"
+        - "despiad.yaml"
+    manufacturer_overrides: []
+
+min_instances_per_series: 1
+series_filters: []
+series_number_filters: []
+allowed_manufacturers: ".*"
+
+destination:
+    dicom: "none"
+    parquet: "none"

projects/configs/tag-operations/ct.yaml

@@ -0,0 +1,102 @@
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+- name: Conversion Type
+  group: 0x0008
+  element: 0x0064
+  op: keep
+- name: Spacing Between Slices
+  group: 0x0018
+  element: 0x0088
+  op: keep
+- name: Data Collection Diameter
+  group: 0x0018
+  element: 0x0090
+  op: keep
+- name: Reconstruction Diameter
+  group: 0x0018
+  element: 0x1100
+  op: keep
+- name: Distance Source to Detector
+  group: 0x0018
+  element: 0x1110
+  op: keep
+- name: Distance Source to Patient
+  group: 0x0018
+  element: 0x1111
+  op: keep
+- name: Gantry Detector Tilt
+  group: 0x0018
+  element: 0x1120
+  op: keep
+- name: Table Height
+  group: 0x0018
+  element: 0x1130
+  op: keep
+- name: Rotation Direction
+  group: 0x0018
+  element: 0x1140
+  op: keep
+- name: Exposure Time
+  group: 0x0018
+  element: 0x1150
+  op: keep
+- name: X-Ray Tube Current
+  group: 0x0018
+  element: 0x1151
+  op: keep
+- name: Exposure
+  group: 0x0018
+  element: 0x1152
+  op: keep
+- name: Filter Type
+  group: 0x0018
+  element: 0x1160
+  op: keep
+- name: Generator Power
+  group: 0x0018
+  element: 0x1170
+  op: keep
+- name: Convolution Kernel
+  group: 0x0018
+  element: 0x1210
+  op: keep
+- name: Revolution Time
+  group: 0x0018
+  element: 0x9305
+  op: keep
+- name: Single Collimation Width
+  group: 0x0018
+  element: 0x9306
+  op: keep
+- name: Total Collimation Width
+  group: 0x0018
+  element: 0x9307
+  op: keep
+- name: Table Speed
+  group: 0x0018
+  element: 0x9309
+  op: keep
+- name: Table Feed per Rotation
+  group: 0x0018
+  element: 0x9310
+  op: keep
+- name: Spiral Pitch Factor
+  group: 0x0018
+  element: 0x9311
+  op: keep
+- name: Slice Location
+  group: 0x0020
+  element: 0x1041
+  op: keep

projects/configs/tag-operations/despiad.yaml

@@ -0,0 +1,42 @@
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+- name: Study Date
+  group: 0x0008
+  element: 0x0020
+  op: keep
+- name: Series Date
+  group: 0x0008
+  element: 0x0021
+  op: keep
+- name: Acquisition Date
+  group: 0x0008
+  element: 0x0022
+  op: keep
+- name: Series Time
+  group: 0x0008
+  element: 0x0031
+  op: keep
+- name: Acquisition Time
+  group: 0x0008
+  element: 0x0032
+  op: keep
+- name: Station Name
+  group: 0x0008
+  element: 0x1010
+  op: keep
+- name: Patient's Birth Date
+  group: 0x0010
+  element: 0x0030
+  op: keep