From 909868c8762e9d9c8a75257db6534dc5dd5f73b5 Mon Sep 17 00:00:00 2001 From: Tom Young <39765193+t-young31@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:27:27 +0100 Subject: [PATCH] Packaging and DRYing (#148) * hasher packaging * merge and package buffer and queue * move common api * cli packaging * merge report deid * fix test * fix system test * update workflow * move mypy and isort to precommit * move flake8 and black to pre-commit * remove old refs * test fixes * black update and path fixes * add missing dep * Apply suggestions from code review Co-authored-by: Stef Piatek --------- Co-authored-by: Stef Piatek --- .github/workflows/main.yml | 98 +++--------- .pre-commit-config.yaml | 43 +++++ cli/README.md | 8 +- cli/pyproject.toml | 31 ++++ cli/src/pixl_cli/_version.py | 16 -- cli/src/pixl_cli/main.py | 12 +- cli/src/pixl_cli/tests/README.md | 4 - cli/src/pixl_cli/tests/__init__.py | 13 -- cli/src/requirements.txt | 15 -- cli/src/setup.py | 26 ---- cli/test/wait-until-service-healthy.sh | 17 -- cli/{test => tests}/README.md | 0 cli/{test => tests}/docker-compose.yml | 0 cli/{test => tests}/pixl_config.yml | 0 cli/{test => tests}/run-tests.sh | 21 ++- cli/{test => tests}/test.csv | 0 .../tests/test_queue_start_and_stop.py | 0 docker/ehr-api/Dockerfile | 33 +--- docker/hasher-api/Dockerfile | 22 +-- docker/pacs-api/Dockerfile | 21 +-- hasher/README.md | 10 +- hasher/bin/run-tests.sh | 31 ---- hasher/pyproject.toml | 31 ++++ hasher/src/hasher/__init__.py | 4 +- hasher/src/hasher/_version.py | 16 -- hasher/src/hasher/main.py | 4 +- hasher/src/requirements.txt | 15 -- hasher/src/setup.py | 33 ---- hasher/{src/hasher => }/tests/__init__.py | 0 hasher/{src/hasher => }/tests/conftest.py | 10 +- .../hasher => }/tests/fixtures/__init__.py | 0 hasher/{src/hasher => }/tests/pytest.ini | 0 .../{src/hasher => }/tests/test_endpoints.py | 0 hasher/{src/hasher => }/tests/test_hashing.py | 0 orthanc/orthanc-anon/plugin/pixl.py | 147 ++++++++++-------- orthanc/orthanc-raw/plugin/pixl.py | 14 +- patient_queue/README.md | 19 --- patient_queue/src/patient_queue/_version.py | 16 -- .../src/patient_queue/tests/pytest.ini | 2 - patient_queue/src/requirements.txt | 11 -- patient_queue/src/setup.py | 33 ---- patient_queue/test/Dockerfile.python310 | 23 --- patient_queue/test/run-tests.sh | 34 ---- pixl_core/README.md | 53 +++++++ pixl_core/pyproject.toml | 35 +++++ .../src/core}/__init__.py | 0 .../src/core/models.py | 14 +- .../src/core/patient_queue/__init__.py | 5 +- .../src/core}/patient_queue/_base.py | 0 .../src/core}/patient_queue/producer.py | 2 +- .../src/core}/patient_queue/subscriber.py | 5 +- .../src/core}/patient_queue/utils.py | 0 pixl_core/src/core/router.py | 47 ++++++ .../src/core/token_buffer}/__init__.py | 4 + .../src/core}/token_buffer/tokens.py | 0 .../__init__.py => pixl_core/tests/Dockerfile | 6 + .../tests/conftest.py | 3 + .../tests}/docker-compose.yml | 8 +- .../tests/patient_queue}/test_producer.py | 5 +- .../tests/patient_queue}/test_subscriber.py | 7 +- .../tests/patient_queue}/test_utils.py | 2 +- .../tests/token_buffer}/test_tokens.py | 2 +- pixl_dcmd/bin/run-tests.sh | 12 -- pixl_dcmd/src/pixl_dcmd/main.py | 8 +- pixl_dcmd/src/setup.py | 2 +- pixl_ehr/README.md | 19 ++- pixl_ehr/pyproject.toml | 38 +++++ pixl_ehr/src/pixl_ehr/_databases.py | 5 +- pixl_ehr/src/pixl_ehr/_processing.py | 8 +- pixl_ehr/src/pixl_ehr/_queries.py | 1 - pixl_ehr/src/pixl_ehr/main.py | 59 +------ .../{tests => report_deid}/__init__.py | 3 + .../src/pixl_ehr/report_deid/deid.py | 4 - .../src/pixl_ehr/report_deid}/exclusions.txt | 0 pixl_ehr/src/requirements.txt | 20 --- pixl_ehr/src/setup.py | 31 ---- pixl_ehr/test/run-lint-and-api-tests.sh | 43 ----- pixl_ehr/test/run-tests.sh | 21 --- .../pixl_rd => pixl_ehr}/tests/data/README.md | 0 .../pixl_rd => pixl_ehr}/tests/data/names.csv | 0 .../tests/data/reports.csv | 0 pixl_ehr/{test => tests}/docker-compose.yml | 2 + .../tests}/run-processing-tests.sh | 8 +- .../test => pixl_ehr/tests}/run-tests.sh | 4 +- pixl_ehr/{src/pixl_ehr => }/tests/test_app.py | 7 +- .../tests/test_deidentification.py | 17 +- .../pixl_ehr => }/tests/test_processing.py | 10 +- pixl_pacs/README.md | 12 ++ pixl_pacs/pyproject.toml | 31 ++++ pixl_pacs/src/pixl_pacs/_orthanc.py | 9 +- pixl_pacs/src/pixl_pacs/_processing.py | 6 +- pixl_pacs/src/pixl_pacs/_version.py | 16 -- pixl_pacs/src/pixl_pacs/main.py | 56 +------ pixl_pacs/src/requirements.txt | 17 -- pixl_pacs/src/setup.py | 28 ---- pixl_pacs/test/run-lint.sh | 41 ----- .../Dockerfile.orthanc_raw_test | 0 pixl_pacs/{test => tests}/docker-compose.yml | 6 +- .../orthanc_raw_config/dicom.json | 0 .../orthanc_raw_config/orthanc.json | 0 .../tests/run-tests.sh | 4 +- .../pixl_pacs => }/tests/test_processing.py | 9 +- pixl_rd/README.md | 21 --- pixl_rd/bin/run-tests.sh | 31 ---- pixl_rd/src/pixl_rd/__init__.py | 18 --- pixl_rd/src/pixl_rd/_version.py | 16 -- pixl_rd/src/requirements.txt | 10 -- pixl_rd/src/setup.py | 33 ---- scripts/cmove_all_studies.py | 17 +- scripts/delete_oldest_n_studies.py | 15 +- .../filter_cohort_for_those_present_in_raw.py | 8 +- scripts/list_newest_n_studies.py | 19 +-- setup.cfg | 11 +- test/.env.test | 2 + test/run-system-test.sh | 8 +- test/scripts/check_entry_in_orthanc_anon.sh | 13 +- test/scripts/insert_test_data.sh | 2 +- test/scripts/install_pixl_cli.sh | 24 --- token_buffer/README.md | 9 -- token_buffer/bin/run-tests.sh | 27 ---- token_buffer/src/requirements.txt | 8 - token_buffer/src/setup.py | 33 ---- token_buffer/src/token_buffer/_version.py | 16 -- 123 files changed, 646 insertions(+), 1283 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 cli/pyproject.toml delete mode 100644 cli/src/pixl_cli/_version.py delete mode 100644 cli/src/pixl_cli/tests/README.md delete mode 100644 cli/src/pixl_cli/tests/__init__.py delete mode 100644 cli/src/requirements.txt delete mode 100644 cli/src/setup.py delete mode 100755 cli/test/wait-until-service-healthy.sh rename cli/{test => tests}/README.md (100%) rename cli/{test => tests}/docker-compose.yml (100%) rename cli/{test => tests}/pixl_config.yml (100%) rename cli/{test => tests}/run-tests.sh (74%) rename cli/{test => tests}/test.csv (100%) rename cli/{src/pixl_cli => }/tests/test_queue_start_and_stop.py (100%) delete mode 100755 hasher/bin/run-tests.sh create mode 100644 hasher/pyproject.toml delete mode 100644 hasher/src/hasher/_version.py delete mode 100644 hasher/src/requirements.txt delete mode 100644 hasher/src/setup.py rename hasher/{src/hasher => }/tests/__init__.py (100%) rename hasher/{src/hasher => }/tests/conftest.py (92%) rename hasher/{src/hasher => }/tests/fixtures/__init__.py (100%) rename hasher/{src/hasher => }/tests/pytest.ini (100%) rename hasher/{src/hasher => }/tests/test_endpoints.py (100%) rename hasher/{src/hasher => }/tests/test_hashing.py (100%) delete mode 100644 patient_queue/README.md delete mode 100644 patient_queue/src/patient_queue/_version.py delete mode 100644 patient_queue/src/patient_queue/tests/pytest.ini delete mode 100644 patient_queue/src/requirements.txt delete mode 100644 patient_queue/src/setup.py delete mode 100644 patient_queue/test/Dockerfile.python310 delete mode 100755 patient_queue/test/run-tests.sh create mode 100644 pixl_core/README.md create mode 100644 pixl_core/pyproject.toml rename {patient_queue/src/patient_queue => pixl_core/src/core}/__init__.py (100%) rename token_buffer/src/token_buffer/__init__.py => pixl_core/src/core/models.py (72%) rename pixl_ehr/src/pixl_ehr/_version.py => pixl_core/src/core/patient_queue/__init__.py (89%) rename {patient_queue/src => pixl_core/src/core}/patient_queue/_base.py (100%) rename {patient_queue/src => pixl_core/src/core}/patient_queue/producer.py (97%) rename {patient_queue/src => pixl_core/src/core}/patient_queue/subscriber.py (97%) rename {patient_queue/src => pixl_core/src/core}/patient_queue/utils.py (100%) create mode 100644 pixl_core/src/core/router.py rename {token_buffer/src/token_buffer/tests => pixl_core/src/core/token_buffer}/__init__.py (91%) rename {token_buffer/src => pixl_core/src/core}/token_buffer/tokens.py (100%) rename pixl_pacs/src/pixl_pacs/tests/__init__.py => pixl_core/tests/Dockerfile (79%) rename pixl_rd/src/pixl_rd/tests/__init__.py => pixl_core/tests/conftest.py (94%) rename {patient_queue/test => pixl_core/tests}/docker-compose.yml (91%) rename {patient_queue/src/patient_queue/tests => pixl_core/tests/patient_queue}/test_producer.py (92%) rename {patient_queue/src/patient_queue/tests => pixl_core/tests/patient_queue}/test_subscriber.py (93%) rename {patient_queue/src/patient_queue/tests => pixl_core/tests/patient_queue}/test_utils.py (95%) rename {token_buffer/src/token_buffer/tests => pixl_core/tests/token_buffer}/test_tokens.py (97%) create mode 100644 pixl_ehr/pyproject.toml rename pixl_ehr/src/pixl_ehr/{tests => report_deid}/__init__.py (90%) rename pixl_rd/src/pixl_rd/main.py => pixl_ehr/src/pixl_ehr/report_deid/deid.py (99%) rename {pixl_rd/src/pixl_rd => pixl_ehr/src/pixl_ehr/report_deid}/exclusions.txt (100%) delete mode 100644 pixl_ehr/src/requirements.txt delete mode 100644 pixl_ehr/src/setup.py delete mode 100755 pixl_ehr/test/run-lint-and-api-tests.sh delete mode 100755 pixl_ehr/test/run-tests.sh rename {pixl_rd/src/pixl_rd => pixl_ehr}/tests/data/README.md (100%) rename {pixl_rd/src/pixl_rd => pixl_ehr}/tests/data/names.csv (100%) rename {pixl_rd/src/pixl_rd => pixl_ehr}/tests/data/reports.csv (100%) rename pixl_ehr/{test => tests}/docker-compose.yml (98%) rename {pixl_pacs/test => pixl_ehr/tests}/run-processing-tests.sh (87%) rename {pixl_pacs/test => pixl_ehr/tests}/run-tests.sh (89%) rename pixl_ehr/{src/pixl_ehr => }/tests/test_app.py (96%) rename {pixl_rd/src/pixl_rd => pixl_ehr}/tests/test_deidentification.py (99%) rename pixl_ehr/{src/pixl_ehr => }/tests/test_processing.py (98%) create mode 100644 pixl_pacs/pyproject.toml delete mode 100644 pixl_pacs/src/pixl_pacs/_version.py delete mode 100644 pixl_pacs/src/requirements.txt delete mode 100644 pixl_pacs/src/setup.py delete mode 100755 pixl_pacs/test/run-lint.sh rename pixl_pacs/{test => tests}/Dockerfile.orthanc_raw_test (100%) rename pixl_pacs/{test => tests}/docker-compose.yml (96%) rename pixl_pacs/{test => tests}/orthanc_raw_config/dicom.json (100%) rename pixl_pacs/{test => tests}/orthanc_raw_config/orthanc.json (100%) rename pixl_ehr/test/run-processing-tests.sh => pixl_pacs/tests/run-tests.sh (92%) rename pixl_pacs/{src/pixl_pacs => }/tests/test_processing.py (97%) delete mode 100644 pixl_rd/README.md delete mode 100755 pixl_rd/bin/run-tests.sh delete mode 100644 pixl_rd/src/pixl_rd/__init__.py delete mode 100644 pixl_rd/src/pixl_rd/_version.py delete mode 100644 pixl_rd/src/requirements.txt delete mode 100644 pixl_rd/src/setup.py delete mode 100755 test/scripts/install_pixl_cli.sh delete mode 100644 token_buffer/README.md delete mode 100755 token_buffer/bin/run-tests.sh delete mode 100644 token_buffer/src/requirements.txt delete mode 100644 token_buffer/src/setup.py delete mode 100644 token_buffer/src/token_buffer/_version.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee7fc6242..ed85833c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,6 +32,11 @@ jobs: - name: Ensure copyright and license header are present run: ./.github/linters/check_headers_exist.sh + - name: Run pre-commit + uses: pre-commit/action@v3.0.0 + with: + extra_args: --all-files + - name: Validate Docker Compose config file working-directory: . run: | @@ -50,49 +55,19 @@ jobs: python-version: '3.10' cache: 'pip' - - name: Install Python dependencies + - name: Install package run: | - python -m pip install --upgrade pip - pip install -r hasher/src/requirements.txt + pip install hasher/[test] - name: Run tests working-directory: hasher - run: | - bin/run-tests.sh + run: pytest env: ENV: test AZURE_KEY_VAULT_NAME: test AZURE_KEY_VAULT_SECRET_NAME: test - report-de-id-tests: - runs-on: ubuntu-22.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v3 - - - name: Init Python - uses: actions/setup-python@v4 - with: - python-version: '3.10.6' # Numpy croaks on 3.10.7+ - cache: 'pip' - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r pixl_rd/src/requirements.txt - - - name: Download presidio data - run: | - python -m spacy download en_core_web_lg - - - name: Run tests - working-directory: pixl_rd - run: | - bin/run-tests.sh - env: - ENV: test - - token-buffer-tests: + core-tests: runs-on: ubuntu-22.04 timeout-minutes: 10 steps: @@ -106,13 +81,13 @@ jobs: - name: Install Python dependencies run: | - python -m pip install --upgrade pip - pip install -r token_buffer/src/requirements.txt + pip install pixl_core/[test] - name: Run tests - working-directory: token_buffer + working-directory: pixl_core/tests run: | - bin/run-tests.sh + docker compose up --build --exit-code-from test + docker compose down env: ENV: test @@ -140,30 +115,6 @@ jobs: env: ENV: test - patient-queue-tests: - runs-on: ubuntu-22.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v3 - - - name: Init Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - cache: 'pip' - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r patient_queue/src/requirements.txt - - - name: Run tests - working-directory: patient_queue/test - run: | - ./run-tests.sh - env: - ENV: test - cli-tests: runs-on: ubuntu-22.04 timeout-minutes: 10 @@ -178,14 +129,10 @@ jobs: - name: Install Python dependencies run: | - python -m pip install --upgrade pip - pip install -r cli/src/requirements.txt - pip install -r token_buffer/src/requirements.txt token_buffer/src/ - pip install -r patient_queue/src/requirements.txt - pip install patient_queue/src/ + pip install pixl_core/[test] cli/[test] - name: Run tests - working-directory: cli/test + working-directory: cli/tests run: | ./run-tests.sh @@ -203,15 +150,11 @@ jobs: - name: Install Python dependencies run: | - python -m pip install --upgrade pip - pip install -r pixl_ehr/src/requirements.txt - pip install -r pixl_rd/src/requirements.txt pixl_rd/src/ - pip install -r token_buffer/src/requirements.txt token_buffer/src/ - pip install -r patient_queue/src/requirements.txt patient_queue/src/ + pip install pixl_core/[test] pixl_ehr/[test] python -m spacy download en_core_web_lg - name: Run tests - working-directory: pixl_ehr/test + working-directory: pixl_ehr/tests env: INFORMDB_PAT: ${{ secrets.INFORMDB_PAT }} run: | @@ -231,12 +174,9 @@ jobs: - name: Install Python dependencies run: | - python -m pip install --upgrade pip - pip install -r pixl_pacs/src/requirements.txt - pip install -r token_buffer/src/requirements.txt token_buffer/src/ - pip install -r patient_queue/src/requirements.txt patient_queue/src/ + pip install pixl_core/[test] pixl_pacs/[test] - name: Run tests - working-directory: pixl_pacs/test + working-directory: pixl_pacs/tests run: | ./run-tests.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..e4f817417 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +repos: + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black", "--settings-path", "setup.cfg"] + + - repo: https://github.com/ambv/black + rev: 23.9.1 + hooks: + - id: black + + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + args: ["--config", "setup.cfg"] + exclude: "tests|orthanc" + + - repo: local + hooks: + - id: mypy # This does not work with the official mypy hook + name: mypy + language: python + pass_filenames: false + entry: mypy + args: ['--config-file=setup.cfg'] + additional_dependencies: ['mypy', 'types-PyYAML', 'types-requests'] diff --git a/cli/README.md b/cli/README.md index 59f154dac..6745c73a5 100644 --- a/cli/README.md +++ b/cli/README.md @@ -10,9 +10,15 @@ stopped cleanly. ## Installation ```bash -cd src && pip install -r requirements.txt . +pip install -e ../pixl_core/ . ``` +## Test +```bash +./tests/run-tests.sh +``` + + ## Usage > **Note** diff --git a/cli/pyproject.toml b/cli/pyproject.toml new file mode 100644 index 000000000..07be7b0d4 --- /dev/null +++ b/cli/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "pixl_cli" +version = "0.0.4" +authors = [ + { name="PIXL authors" }, +] +description = "PIXL command line interface" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3" +] +dependencies = [ + "core", + "click==8.1.3", + "coloredlogs==15.0.1", + "pandas==1.5.1", + "PyYAML==6.0" +] + +[project.optional-dependencies] +test = [ + "pytest==7.4.*" +] + +[project.scripts] +pixl = "pixl_cli.main:cli" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" diff --git a/cli/src/pixl_cli/_version.py b/cli/src/pixl_cli/_version.py deleted file mode 100644 index 523d681be..000000000 --- a/cli/src/pixl_cli/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version_info__ = ("0", "0", "4") -__version__ = ".".join(__version_info__) diff --git a/cli/src/pixl_cli/main.py b/cli/src/pixl_cli/main.py index be475bd93..f2e5df893 100644 --- a/cli/src/pixl_cli/main.py +++ b/cli/src/pixl_cli/main.py @@ -20,14 +20,15 @@ import pandas as pd import click -from patient_queue.producer import PixlProducer -from patient_queue.subscriber import PixlBlockingConsumer -from patient_queue.utils import serialise, deserialise -from pixl_cli._logging import logger, set_log_level -from pixl_cli._utils import clear_file, remove_file_if_it_exists, string_is_non_empty +from core.patient_queue.producer import PixlProducer +from core.patient_queue.subscriber import PixlBlockingConsumer +from core.patient_queue.utils import deserialise, serialise import requests import yaml +from ._logging import logger, set_log_level +from ._utils import clear_file, remove_file_if_it_exists, string_is_non_empty + def _load_config(filename: str = "pixl_config.yml") -> dict: """CLI configuration generated from a .yaml file""" @@ -77,7 +78,6 @@ def populate(csv_filename: str, queues: str, restart: bool) -> None: for queue in queues.split(","): with PixlProducer(queue_name=queue, **config["rabbitmq"]) as producer: - state_filepath = state_filepath_for_queue(queue) if state_filepath.exists() and restart: logger.info(f"Extracting messages from state: {state_filepath}") diff --git a/cli/src/pixl_cli/tests/README.md b/cli/src/pixl_cli/tests/README.md deleted file mode 100644 index 383f95974..000000000 --- a/cli/src/pixl_cli/tests/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# PIXL CLI tests - -All required test services must be up before running these tests. Also note that RabbitMQ tests here use default credentials -as opposed to what is specified in the environment file in the main directory. diff --git a/cli/src/pixl_cli/tests/__init__.py b/cli/src/pixl_cli/tests/__init__.py deleted file mode 100644 index ff5b83333..000000000 --- a/cli/src/pixl_cli/tests/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/cli/src/requirements.txt b/cli/src/requirements.txt deleted file mode 100644 index 560bf821d..000000000 --- a/cli/src/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -black==22.8.0 -flake8==5.0.4 -flake8-return==1.1.3 -isort==5.10.1 -environs==9.5.0 -mypy==0.991 -pytest==7.1.3 -click==8.1.3 -coloredlogs==15.0.1 -pika==1.3.1 -pandas==1.5.1 -PyYAML==6.0 -requests==2.28.1 -types-PyYAML==6.0.* -types-requests==2.28.* diff --git a/cli/src/setup.py b/cli/src/setup.py deleted file mode 100644 index 6858f7547..000000000 --- a/cli/src/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from setuptools import setup, find_packages - -exec(open("pixl_cli/_version.py").read()) - -setup( - name="pixl_cli", - version=__version__, # noqa: F821 - packages=find_packages("."), - author="Tom Young", - url="https://github.com/UCLH-Foundry/PIXL", - entry_points={"console_scripts": ["pixl = pixl_cli.main:cli"]}, - description="Command line interaction with PIXL", -) diff --git a/cli/test/wait-until-service-healthy.sh b/cli/test/wait-until-service-healthy.sh deleted file mode 100755 index 96bcb4df4..000000000 --- a/cli/test/wait-until-service-healthy.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -while ! docker ps | grep "$1" | grep -q healthy ;do - sleep 5 -done diff --git a/cli/test/README.md b/cli/tests/README.md similarity index 100% rename from cli/test/README.md rename to cli/tests/README.md diff --git a/cli/test/docker-compose.yml b/cli/tests/docker-compose.yml similarity index 100% rename from cli/test/docker-compose.yml rename to cli/tests/docker-compose.yml diff --git a/cli/test/pixl_config.yml b/cli/tests/pixl_config.yml similarity index 100% rename from cli/test/pixl_config.yml rename to cli/tests/pixl_config.yml diff --git a/cli/test/run-tests.sh b/cli/tests/run-tests.sh similarity index 74% rename from cli/test/run-tests.sh rename to cli/tests/run-tests.sh index ca9926f11..9acee9c43 100755 --- a/cli/test/run-tests.sh +++ b/cli/tests/run-tests.sh @@ -14,21 +14,20 @@ # limitations under the License. set -eux pipefail +function wait_until_service_healthy() { + while ! docker ps | grep "$1" | grep -q healthy ;do + sleep 5 + done +} + THIS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${THIS_DIR%/*}" cd "$PACKAGE_DIR" || exit -pip install -r src/requirements.txt - -CONF_FILE=../setup.cfg -mypy --config-file ${CONF_FILE} src/pixl_cli -isort --settings-path ${CONF_FILE} src/pixl_cli -black src/pixl_cli -flake8 --config ${CONF_FILE} src/pixl_cli - -cd test/ +pip install "../pixl_core/[test]" ".[test]" +cd tests/ docker compose up -d -./wait-until-service-healthy.sh queue -pytest ../src/pixl_cli +wait_until_service_healthy queue +pytest docker compose down diff --git a/cli/test/test.csv b/cli/tests/test.csv similarity index 100% rename from cli/test/test.csv rename to cli/tests/test.csv diff --git a/cli/src/pixl_cli/tests/test_queue_start_and_stop.py b/cli/tests/test_queue_start_and_stop.py similarity index 100% rename from cli/src/pixl_cli/tests/test_queue_start_and_stop.py rename to cli/tests/test_queue_start_and_stop.py diff --git a/docker/ehr-api/Dockerfile b/docker/ehr-api/Dockerfile index 30cfb88be..9a6d3b9ca 100644 --- a/docker/ehr-api/Dockerfile +++ b/docker/ehr-api/Dockerfile @@ -14,6 +14,8 @@ FROM python:3.10.6-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +ARG TEST="false" + RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install --yes --no-install-recommends procps ca-certificates \ @@ -21,38 +23,17 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ RUN sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen RUN apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN mkdir /app WORKDIR /app -ENV VIRTUAL_ENV=/app/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - # Download presido data. Before other dependencies to cache more effectively RUN pip install spacy && python -m spacy download en_core_web_lg -RUN mkdir local_deps -COPY ./token_buffer/src/requirements.txt local_deps/r0.txt -COPY ./pixl_rd/src/requirements.txt local_deps/r1.txt -COPY ./patient_queue/src/requirements.txt local_deps/r2.txt -RUN for i in {0..2}; do pip install -r "local_deps/r${i}.txt"; done - -COPY ./pixl_ehr/src/requirements.txt . -# patch is needed for image to work on a M1 mac.. -RUN sed -i '/psycopg2-binary==2.9.5/c\psycopg2==2.9.3' requirements.txt && \ - pip install --no-cache-dir -U pip wheel setuptools \ - && pip install --no-cache-dir -r requirements.txt - -# install the pacakges after the dependencies for better caching -COPY ./token_buffer/src/ local_deps/token_buffer/ -COPY ./pixl_rd/src/ local_deps/pixl_rd/ -COPY ./patient_queue/src/ local_deps/patient_queue/ -RUN for dir in local_deps/*/; do pip install --no-cache-dir $dir; done - -COPY ./pixl_ehr/src ./pixl_ehr -RUN pip install --no-cache-dir ./pixl_ehr +COPY ./pixl_core/ core/ +COPY ./pixl_ehr/ . +RUN --mount=type=cache,target=/root/.cache \ + pip install core/ . && \ + if [ "$TEST" = "true" ]; then pip install core/[test] .[test]; fi HEALTHCHECK CMD /usr/bin/curl -f http://0.0.0.0:8000/heart-beat || exit 1 -WORKDIR /app/pixl_ehr ENTRYPOINT ["uvicorn", "pixl_ehr.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/hasher-api/Dockerfile b/docker/hasher-api/Dockerfile index 14252a846..82cf04b51 100644 --- a/docker/hasher-api/Dockerfile +++ b/docker/hasher-api/Dockerfile @@ -14,7 +14,6 @@ FROM python:3.10.7-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] - # OS setup RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -22,25 +21,12 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ iproute2 git curl libpq-dev curl gnupg g++ locales RUN sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen - # clean up +# clean up RUN apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* -# Directory setup -RUN mkdir /app WORKDIR /app +COPY ./hasher/ . +RUN --mount=type=cache,target=/root/.cache \ + pip install . -ENV VIRTUAL_ENV=/app/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -# 3rd party deps -COPY ./hasher/src/requirements.txt . -RUN pip install --no-cache-dir -U pip wheel setuptools \ - && pip install --no-cache-dir -r requirements.txt - -# hasher -COPY ./hasher/src ./hasher -RUN pip install --no-cache-dir ./hasher - -WORKDIR /app/hasher ENTRYPOINT ["uvicorn", "hasher.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/pacs-api/Dockerfile b/docker/pacs-api/Dockerfile index 7f38e79be..16fb97121 100644 --- a/docker/pacs-api/Dockerfile +++ b/docker/pacs-api/Dockerfile @@ -14,6 +14,8 @@ FROM python:3.10.6-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +ARG TEST="false" + RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install --yes --no-install-recommends procps ca-certificates \ @@ -21,23 +23,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ RUN sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen RUN apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN mkdir /app WORKDIR /app -ENV VIRTUAL_ENV=/app/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -RUN mkdir local_deps -COPY ./token_buffer/src/ local_deps/token_buffer/ -COPY ./patient_queue/src/ local_deps/patient_queue/ -RUN for dir in local_deps/*/; do pip install --no-cache-dir -r "${dir}requirements.txt" $dir; done - -COPY ./pixl_pacs/src ./pixl_pacs -RUN pip install --no-cache-dir -r ./pixl_pacs/requirements.txt -RUN pip install --no-cache-dir ./pixl_pacs +COPY ./pixl_core/ core/ +COPY ./pixl_pacs/ . +RUN --mount=type=cache,target=/root/.cache \ + pip install core/ . && \ + if [ "$TEST" = "true" ]; then pip install core/[test] .[test]; fi HEALTHCHECK CMD /usr/bin/curl -f http://0.0.0.0:8000/heart-beat || exit 1 -WORKDIR /app/pixl_pacs ENTRYPOINT ["uvicorn", "pixl_pacs.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/hasher/README.md b/hasher/README.md index aa962890f..9d0cc33f6 100644 --- a/hasher/README.md +++ b/hasher/README.md @@ -83,7 +83,7 @@ HASHER_API_AZ_KEY_VAULT_SECRET_NAME= It is assumed you have a Python virtual environment configured using a tool like Conda or pyenv. Install the dependencies from inside the _PIXL/hasher/src_ directory: ```bash -pip install -r requirements.txt +pip install -e . ``` ### Setup @@ -98,13 +98,9 @@ uvicorn hasher.main:app --host=0.0.0.0 --port=8000 --reload ``` ### Test -from the _PIXL/hasher/src_ directory: -```bash -bin/run-tests.sh -``` -or +From this directory: ```bash -PIXL_ENV=test pytest --ff hasher/tests +pytest ``` to skip linting and run only the last failed test. diff --git a/hasher/bin/run-tests.sh b/hasher/bin/run-tests.sh deleted file mode 100755 index f661e61bb..000000000 --- a/hasher/bin/run-tests.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e pipefail - -BIN_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) && pwd) -HASHER_DIR="${BIN_DIR%/*}" -cd $HASHER_DIR - -CONF_FILE=../setup.cfg - -mypy --config-file ${CONF_FILE} src/hasher - -isort --settings-path ${CONF_FILE} src/hasher - -black src/hasher - -flake8 --config ${CONF_FILE} - -ENV=test pytest src/hasher/tests diff --git a/hasher/pyproject.toml b/hasher/pyproject.toml new file mode 100644 index 000000000..f72df7db6 --- /dev/null +++ b/hasher/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "hasher" +version = "0.0.1" +authors = [ + { name="PIXL authors" }, +] +description = "Service to securely hash identifiers" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3" +] +dependencies = [ + "azure-identity==1.12.0", + "azure-keyvault==4.2.0", + "fastapi==0.103.2", + "hypothesis==6.56.0", + "environs==9.5.0", + "requests==2.31.0", + "uvicorn==0.23.2", +] + +[project.optional-dependencies] +test = [ + "pytest==7.4.*", + "httpx==0.24.*" +] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" diff --git a/hasher/src/hasher/__init__.py b/hasher/src/hasher/__init__.py index 07cd35ff8..eec699afd 100644 --- a/hasher/src/hasher/__init__.py +++ b/hasher/src/hasher/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._version import __version__, __version_info__ +import importlib.metadata + +__version__ = importlib.metadata.version("hasher") icon = "🪢" diff --git a/hasher/src/hasher/_version.py b/hasher/src/hasher/_version.py deleted file mode 100644 index c2ef34e98..000000000 --- a/hasher/src/hasher/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version_info__ = ("0", "0", "1") -__version__ = ".".join(__version_info__) diff --git a/hasher/src/hasher/main.py b/hasher/src/hasher/main.py index e6bcf6ceb..2bddd71be 100644 --- a/hasher/src/hasher/main.py +++ b/hasher/src/hasher/main.py @@ -18,8 +18,8 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -from hasher import __version__, icon, settings -from hasher.endpoints import router +from . import __version__, icon, settings +from .endpoints import router logger = logging.getLogger(__name__) diff --git a/hasher/src/requirements.txt b/hasher/src/requirements.txt deleted file mode 100644 index c16aeef74..000000000 --- a/hasher/src/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -azure-identity==1.12.0 -azure-keyvault==4.2.0 -black==22.8.0 -fastapi==0.103.2 -flake8==5.0.4 -flake8-return==1.1.3 -hypothesis==6.56.0 -isort==5.10.1 -environs==9.5.0 -mypy==1.6.0 -httpx==0.25.0 -pytest==7.4.2 -pytest-mock==3.8.2 -requests==2.31.0 -uvicorn==0.23.2 diff --git a/hasher/src/setup.py b/hasher/src/setup.py deleted file mode 100644 index 796dcfd50..000000000 --- a/hasher/src/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import find_packages, setup - -exec(open("hasher/_version.py").read()) - -setup( - name="hasher", - version=__version__, # noqa: F821 - description="Service to securely hash identifiers", - packages=find_packages( - include=[ - "hasher*", - ], - exclude=[ - "*tests", - "*.tests.*", - ], - ), - python_requires=">=3.10", -) diff --git a/hasher/src/hasher/tests/__init__.py b/hasher/tests/__init__.py similarity index 100% rename from hasher/src/hasher/tests/__init__.py rename to hasher/tests/__init__.py diff --git a/hasher/src/hasher/tests/conftest.py b/hasher/tests/conftest.py similarity index 92% rename from hasher/src/hasher/tests/conftest.py rename to hasher/tests/conftest.py index d259fc72e..c099d16e9 100644 --- a/hasher/src/hasher/tests/conftest.py +++ b/hasher/tests/conftest.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import os -import hasher.hashing +import pytest -pytest_plugins = [ - "hasher.tests.fixtures", -] +os.environ["ENV"] = "test" @pytest.fixture @@ -26,4 +24,6 @@ def dummy_key(monkeypatch): """ Fixture to set up a dummy key to use for hashing tests """ + import hasher.hashing + monkeypatch.setattr(hasher.hashing, "fetch_key_from_vault", lambda: "test-key") diff --git a/hasher/src/hasher/tests/fixtures/__init__.py b/hasher/tests/fixtures/__init__.py similarity index 100% rename from hasher/src/hasher/tests/fixtures/__init__.py rename to hasher/tests/fixtures/__init__.py diff --git a/hasher/src/hasher/tests/pytest.ini b/hasher/tests/pytest.ini similarity index 100% rename from hasher/src/hasher/tests/pytest.ini rename to hasher/tests/pytest.ini diff --git a/hasher/src/hasher/tests/test_endpoints.py b/hasher/tests/test_endpoints.py similarity index 100% rename from hasher/src/hasher/tests/test_endpoints.py rename to hasher/tests/test_endpoints.py diff --git a/hasher/src/hasher/tests/test_hashing.py b/hasher/tests/test_hashing.py similarity index 100% rename from hasher/src/hasher/tests/test_hashing.py rename to hasher/tests/test_hashing.py diff --git a/orthanc/orthanc-anon/plugin/pixl.py b/orthanc/orthanc-anon/plugin/pixl.py index 4c4e98ab8..53175ec90 100644 --- a/orthanc/orthanc-anon/plugin/pixl.py +++ b/orthanc/orthanc-anon/plugin/pixl.py @@ -11,93 +11,96 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +from io import BytesIO +import json +import logging import os +import pprint +import sys +import threading +from time import sleep import traceback -from decouple import config -from io import BytesIO -from time import sleep +from decouple import config from pydicom import dcmread, dcmwrite from pydicom.filebase import DicomFileLike - -import hashlib -import json -import orthanc -import pprint import requests -import sys -import threading import yaml -import logging - +import orthanc import pixl_dcmd -def AzureAccessToken(): - AZ_DICOM_ENDPOINT_CLIENT_ID = config('AZ_DICOM_ENDPOINT_CLIENT_ID') - AZ_DICOM_ENDPOINT_CLIENT_SECRET = config('AZ_DICOM_ENDPOINT_CLIENT_SECRET') - AZ_DICOM_ENDPOINT_TENANT_ID = config('AZ_DICOM_ENDPOINT_TENANT_ID') +def AzureAccessToken(): + AZ_DICOM_ENDPOINT_CLIENT_ID = config("AZ_DICOM_ENDPOINT_CLIENT_ID") + AZ_DICOM_ENDPOINT_CLIENT_SECRET = config("AZ_DICOM_ENDPOINT_CLIENT_SECRET") + AZ_DICOM_ENDPOINT_TENANT_ID = config("AZ_DICOM_ENDPOINT_TENANT_ID") - url = "https://login.microsoft.com/" + AZ_DICOM_ENDPOINT_TENANT_ID \ - + "/oauth2/token" + url = "https://login.microsoft.com/" + AZ_DICOM_ENDPOINT_TENANT_ID + "/oauth2/token" payload = { - 'client_id': AZ_DICOM_ENDPOINT_CLIENT_ID, - 'grant_type': 'client_credentials', - 'client_secret': AZ_DICOM_ENDPOINT_CLIENT_SECRET, - 'resource': 'https://dicom.healthcareapis.azure.com' + "client_id": AZ_DICOM_ENDPOINT_CLIENT_ID, + "grant_type": "client_credentials", + "client_secret": AZ_DICOM_ENDPOINT_CLIENT_SECRET, + "resource": "https://dicom.healthcareapis.azure.com", } response = requests.post(url, data=payload) - #logging.info(f"{payload}") - #logging.info(f"{response.content}") access_token = response.json()["access_token"] return access_token + def AzureDICOMTokenRefresh(): global TIMER TIMER = None orthanc.LogWarning("Refreshing Azure DICOM token") - ORTHANC_USERNAME = config('ORTHANC_USERNAME') - ORTHANC_PASSWORD = config('ORTHANC_PASSWORD') + ORTHANC_USERNAME = config("ORTHANC_USERNAME") + ORTHANC_PASSWORD = config("ORTHANC_PASSWORD") - AZ_DICOM_TOKEN_REFRESH_SECS = int(config('AZ_DICOM_TOKEN_REFRESH_SECS')) - AZ_DICOM_ENDPOINT_NAME = config('AZ_DICOM_ENDPOINT_NAME') - AZ_DICOM_ENDPOINT_URL = config('AZ_DICOM_ENDPOINT_URL') - AZ_DICOM_HTTP_TIMEOUT = int(config('HTTP_TIMEOUT')) + AZ_DICOM_TOKEN_REFRESH_SECS = int(config("AZ_DICOM_TOKEN_REFRESH_SECS")) + AZ_DICOM_ENDPOINT_NAME = config("AZ_DICOM_ENDPOINT_NAME") + AZ_DICOM_ENDPOINT_URL = config("AZ_DICOM_ENDPOINT_URL") + AZ_DICOM_HTTP_TIMEOUT = int(config("HTTP_TIMEOUT")) try: access_token = AzureAccessToken() # logging.info(f"{access_token}") except Exception: - orthanc.LogError("Failed to get an Azure access token. Retrying in 30 seconds\n" - + traceback.format_exc()) + orthanc.LogError( + "Failed to get an Azure access token. Retrying in 30 seconds\n" + + traceback.format_exc() + ) sleep(30) return AzureDICOMTokenRefresh() bearer_str = "Bearer " + access_token dicomweb_config = { - "Url" : AZ_DICOM_ENDPOINT_URL, - "HttpHeaders" : { - "Authorization" : bearer_str, + "Url": AZ_DICOM_ENDPOINT_URL, + "HttpHeaders": { + "Authorization": bearer_str, }, "HasDelete": True, - "Timeout" : AZ_DICOM_HTTP_TIMEOUT + "Timeout": AZ_DICOM_HTTP_TIMEOUT, } - #logging.info(f"{dicomweb_config}") + # logging.info(f"{dicomweb_config}") - headers = {'content-type': 'application/json'} + headers = {"content-type": "application/json"} url = "http://localhost:8042/dicom-web/servers/" + AZ_DICOM_ENDPOINT_NAME try: - requests.put(url, auth=(ORTHANC_USERNAME, ORTHANC_PASSWORD), headers=headers, data=json.dumps(dicomweb_config)) + requests.put( + url, + auth=(ORTHANC_USERNAME, ORTHANC_PASSWORD), + headers=headers, + data=json.dumps(dicomweb_config), + ) except requests.exceptions.RequestException as e: orthanc.LogError("Failed to update DICOMweb token") raise SystemExit(e) @@ -107,41 +110,42 @@ def AzureDICOMTokenRefresh(): TIMER = threading.Timer(AZ_DICOM_TOKEN_REFRESH_SECS, AzureDICOMTokenRefresh) TIMER.start() -def SendViaStow(resourceId): - ORTHANC_USERNAME = config('ORTHANC_USERNAME') - ORTHANC_PASSWORD = config('ORTHANC_PASSWORD') +def SendViaStow(resourceId): + ORTHANC_USERNAME = config("ORTHANC_USERNAME") + ORTHANC_PASSWORD = config("ORTHANC_PASSWORD") - AZ_DICOM_ENDPOINT_NAME = config('AZ_DICOM_ENDPOINT_NAME') + AZ_DICOM_ENDPOINT_NAME = config("AZ_DICOM_ENDPOINT_NAME") url = "http://localhost:8042/dicom-web/servers/" + AZ_DICOM_ENDPOINT_NAME + "/stow" - headers = {'content-type': 'application/json'} + headers = {"content-type": "application/json"} - payload = { - "Resources" : [ - resourceId - ], - "Synchronous" : False - } + payload = {"Resources": [resourceId], "Synchronous": False} logging.info(f"{payload}") try: - requests.post(url, auth=(ORTHANC_USERNAME, ORTHANC_PASSWORD), headers=headers, data=json.dumps(payload)) + requests.post( + url, + auth=(ORTHANC_USERNAME, ORTHANC_PASSWORD), + headers=headers, + data=json.dumps(payload), + ) except requests.exceptions.RequestException as e: orthanc.LogError("Failed to send via STOW") + def ShouldAutoRoute(): return os.environ.get("ORTHANC_AUTOROUTE_ANON_TO_AZURE", "false").lower() == "true" -def OnChange(changeType, level, resource): +def OnChange(changeType, level, resource): if not ShouldAutoRoute(): return if changeType == orthanc.ChangeType.STABLE_STUDY and ShouldAutoRoute(): - print('Stable study: %s' % resource) + print("Stable study: %s" % resource) SendViaStow(resource) if changeType == orthanc.ChangeType.ORTHANC_STARTED: @@ -152,58 +156,63 @@ def OnChange(changeType, level, resource): orthanc.LogWarning("Stopping the scheduler") TIMER.cancel() + def OnHeartBeat(output, uri, **request): orthanc.LogWarning("OK") - output.AnswerBuffer('OK\n', 'text/plain') + output.AnswerBuffer("OK\n", "text/plain") + def ReceivedInstanceCallback(receivedDicom, origin): """Modifies a DICOM instance received by Orthanc and applies anonymisation.""" if origin == orthanc.InstanceOrigin.REST_API: - orthanc.LogWarning('DICOM instance received from the REST API') + orthanc.LogWarning("DICOM instance received from the REST API") elif origin == orthanc.InstanceOrigin.DICOM_PROTOCOL: - orthanc.LogWarning('DICOM instance received from the DICOM protocol') - + orthanc.LogWarning("DICOM instance received from the DICOM protocol") + # Read the bytes as DICOM/ dataset = dcmread(BytesIO(receivedDicom)) # Drop anything that is not an X-Ray - if not (dataset.Modality == 'DX' or dataset.Modality == 'CR'): - orthanc.LogWarning('Dropping DICOM that is not X-Ray') + if not (dataset.Modality == "DX" or dataset.Modality == "CR"): + orthanc.LogWarning("Dropping DICOM that is not X-Ray") return orthanc.ReceivedInstanceAction.DISCARD, None # Attempt to anonymise and drop the study if any exceptions occur try: return AnonymiseCallback(dataset) except Exception as e: - orthanc.LogWarning('Failed to anonymize study due to\n' + traceback.format_exc()) + orthanc.LogWarning( + "Failed to anonymize study due to\n" + traceback.format_exc() + ) return orthanc.ReceivedInstanceAction.DISCARD, None def AnonymiseCallback(dataset): - - orthanc.LogWarning('***Anonymising received instance***') + orthanc.LogWarning("***Anonymising received instance***") # Rip out all private tags/ dataset.remove_private_tags() - orthanc.LogWarning('Removed private tags') + orthanc.LogWarning("Removed private tags") # Rip out overlays/ dataset = pixl_dcmd.remove_overlays(dataset) - orthanc.LogWarning('Removed overlays') + orthanc.LogWarning("Removed overlays") # Apply anonymisation. - with open('/etc/orthanc/tag-operations.yaml', 'r') as file: + with open("/etc/orthanc/tag-operations.yaml", "r") as file: # Load tag operations scheme from YAML. tags = yaml.safe_load(file) # Apply scheme to instance - dataset = pixl_dcmd.apply_tag_scheme(dataset,tags) + dataset = pixl_dcmd.apply_tag_scheme(dataset, tags) # Apply whitelist - dataset = pixl_dcmd.enforce_whitelist(dataset,tags) + dataset = pixl_dcmd.enforce_whitelist(dataset, tags) # Write anoymised instance to disk. - return orthanc.ReceivedInstanceAction.MODIFY, pixl_dcmd.write_dataset_to_bytes(dataset) + return orthanc.ReceivedInstanceAction.MODIFY, pixl_dcmd.write_dataset_to_bytes( + dataset + ) orthanc.RegisterOnChangeCallback(OnChange) orthanc.RegisterReceivedInstanceCallback(ReceivedInstanceCallback) -orthanc.RegisterRestCallback('/heart-beat', OnHeartBeat) \ No newline at end of file +orthanc.RegisterRestCallback("/heart-beat", OnHeartBeat) diff --git a/orthanc/orthanc-raw/plugin/pixl.py b/orthanc/orthanc-raw/plugin/pixl.py index 29e98f321..05d1cf869 100644 --- a/orthanc/orthanc-raw/plugin/pixl.py +++ b/orthanc/orthanc-raw/plugin/pixl.py @@ -12,22 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import os + import orthanc + def OnChange(changeType, level, resourceId): - # Taken from: + # Taken from: # https://book.orthanc-server.com/plugins/python.html#auto-routing-studies if changeType == orthanc.ChangeType.STABLE_STUDY and ShouldAutoRoute(): - print('Stable study: %s' % resourceId) - orthanc.RestApiPost('/modalities/PIXL-Anon/store', resourceId) + print("Stable study: %s" % resourceId) + orthanc.RestApiPost("/modalities/PIXL-Anon/store", resourceId) + def OnHeartBeat(output, uri, **request): orthanc.LogWarning("OK") - output.AnswerBuffer('OK\n', 'text/plain') + output.AnswerBuffer("OK\n", "text/plain") + def ShouldAutoRoute(): return os.environ.get("ORTHANC_AUTOROUTE_RAW_TO_ANON", "false").lower() == "true" orthanc.RegisterOnChangeCallback(OnChange) -orthanc.RegisterRestCallback('/heart-beat', OnHeartBeat) \ No newline at end of file +orthanc.RegisterRestCallback("/heart-beat", OnHeartBeat) diff --git a/patient_queue/README.md b/patient_queue/README.md deleted file mode 100644 index 08fc57eaf..000000000 --- a/patient_queue/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Patient queue - -Mechanism that allows driver to populate queues that can then be consumed by different services, e.g. patient data -or image download. - -Two queues are currently planned: -1. for download and de-identification of image data (default "pacs") -2. for download and de-identification of EHR demographic data (default "ehr") - -The image anonymisation will be triggered automatically once the image has been downloaded to the raw Orthanc server. - -## RabbitMQ - -RabbitMQ is used for the queue implementation. - -The client of choice for RabbitMQ at this point in time is [pika](https://pika.readthedocs.io/en/stable/), which provides both a synchronous and -asynchronous way of transferring messages. The former is geared towards high data throughput whereas the latter is geared towards stability. -The asynchronous mode of transferring messages is a lot more complex as it is based on the -[asyncio event loop](https://docs.python.org/3/library/asyncio-eventloop.html). diff --git a/patient_queue/src/patient_queue/_version.py b/patient_queue/src/patient_queue/_version.py deleted file mode 100644 index da6422c2d..000000000 --- a/patient_queue/src/patient_queue/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version_info__ = ("0", "0", "3") -__version__ = ".".join(__version_info__) diff --git a/patient_queue/src/patient_queue/tests/pytest.ini b/patient_queue/src/patient_queue/tests/pytest.ini deleted file mode 100644 index d280de047..000000000 --- a/patient_queue/src/patient_queue/tests/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -asyncio_mode = auto \ No newline at end of file diff --git a/patient_queue/src/requirements.txt b/patient_queue/src/requirements.txt deleted file mode 100644 index d754d77f2..000000000 --- a/patient_queue/src/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -pika==1.3.1 -aio_pika==8.2.4 -black==22.8.0 -flake8==5.0.4 -flake8-return==1.1.3 -isort==5.10.1 -environs==9.5.0 -mypy==1.6.0 -pytest==7.1.3 -pytest-asyncio==0.20.2 -python-decouple==3.6 \ No newline at end of file diff --git a/patient_queue/src/setup.py b/patient_queue/src/setup.py deleted file mode 100644 index 5edc2037f..000000000 --- a/patient_queue/src/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import find_packages, setup - -exec(open("patient_queue/_version.py").read()) - -setup( - name="patient_queue", - version=__version__, # noqa: F821 - description="Service to create queues for inter-service communication", - packages=find_packages( - include=[ - "patient_queue*", - ], - exclude=[ - "*tests", - "*.tests.*", - ], - ), - python_requires=">=3.10", -) diff --git a/patient_queue/test/Dockerfile.python310 b/patient_queue/test/Dockerfile.python310 deleted file mode 100644 index 3ed9246d3..000000000 --- a/patient_queue/test/Dockerfile.python310 +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM python:3.10.6-slim-bullseye -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] - -# source code -COPY ./patient_queue/src ./patient_queue -COPY ./token_buffer/src ./token_buffer -RUN pip install -r ./patient_queue/requirements.txt -RUN pip install -r ./token_buffer/requirements.txt -RUN pip install --no-cache-dir ./token_buffer && \ - pip install --no-cache-dir ./patient_queue diff --git a/patient_queue/test/run-tests.sh b/patient_queue/test/run-tests.sh deleted file mode 100755 index 7da78c68f..000000000 --- a/patient_queue/test/run-tests.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -eo pipefail - -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -QUEUE_DIR="${BIN_DIR%/*}" -cd "$QUEUE_DIR/test" || exit - -CONF_FILE=../../setup.cfg - -mypy --config-file ${CONF_FILE} ../src/patient_queue -isort --settings-path ${CONF_FILE} ../src/patient_queue -black ../src/patient_queue -flake8 --config ${CONF_FILE} ../src/patient_queue - -docker compose -f docker-compose.yml up -d --build -docker exec pixl-test-python /bin/bash -c "pytest /patient_queue/patient_queue/tests/" -docker compose -f docker-compose.yml down diff --git a/pixl_core/README.md b/pixl_core/README.md new file mode 100644 index 000000000..fced850ed --- /dev/null +++ b/pixl_core/README.md @@ -0,0 +1,53 @@ +# Core + +This directory contains a Python module with core PIXL functionality utilised by both the EHR and PACS APIs to +interact with RabbitMQ and ensure suitable rate limiting of requests to the upstream services. + +### Install +```bash +pip install -e . +``` + +### Test + +```bash +pip install .[test] +pytest -m "not pika" +``` +or the full set with +```bash +cd tests +docker compose up --build --exit-code-from test +docker compose down +``` + +## Token buffer + +The token buffer is needed to limit the download rate for images from PAX/VNA. Current specification suggests that a +rate limit of five images per second should be sufficient, however that may have to be altered dynamically through +command line interaction. + +The current implementation of the token buffer uses the +[token bucket implementation from Falconry](https://github.com/falconry/token-bucket/). Furthermore, the token buffer is +not set up as a service as it is only needed for the image download rate. + + +## Patient queue + +Mechanism that allows driver to populate queues that can then be consumed by different services, e.g. patient data +or image download. + +Two queues are currently planned: +1. for download and de-identification of image data (default "pacs") +2. for download and de-identification of EHR demographic data (default "ehr") + +The image anonymisation will be triggered automatically once the image has been downloaded to the raw Orthanc server. + +### RabbitMQ + +RabbitMQ is used for the queue implementation. + +The client of choice for RabbitMQ at this point in time is [pika](https://pika.readthedocs.io/en/stable/), which provides both a synchronous and +asynchronous way of transferring messages. The former is geared towards high data throughput whereas the latter is geared towards stability. +The asynchronous mode of transferring messages is a lot more complex as it is based on the +[asyncio event loop](https://docs.python.org/3/library/asyncio-eventloop.html). diff --git a/pixl_core/pyproject.toml b/pixl_core/pyproject.toml new file mode 100644 index 000000000..00571b98f --- /dev/null +++ b/pixl_core/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "core" +version = "0.0.1" +authors = [ + { name="PIXL core functionality" }, +] +description = "" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3" +] +dependencies = [ + "fastapi==0.103.2", + "token-bucket==0.3.0", + "python-decouple==3.6", + "pika==1.3.1", + "aio_pika==8.2.4", + "environs==9.5.0", + "requests==2.31.0" +] + +[project.optional-dependencies] +test = [ + "pytest==7.4.2", + "pytest-asyncio==0.21.1", + "httpx==0.24.*" +] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +markers = ["pika"] diff --git a/patient_queue/src/patient_queue/__init__.py b/pixl_core/src/core/__init__.py similarity index 100% rename from patient_queue/src/patient_queue/__init__.py rename to pixl_core/src/core/__init__.py diff --git a/token_buffer/src/token_buffer/__init__.py b/pixl_core/src/core/models.py similarity index 72% rename from token_buffer/src/token_buffer/__init__.py rename to pixl_core/src/core/models.py index 7589828ce..7044fcc32 100644 --- a/token_buffer/src/token_buffer/__init__.py +++ b/pixl_core/src/core/models.py @@ -11,9 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import dataclass -from token_buffer.tokens import TokenBucket +from pydantic import BaseModel -from ._version import __version__, __version_info__ +from .token_buffer import TokenBucket -__all__ = ["TokenBucket"] + +@dataclass +class AppState: + token_bucket = TokenBucket(rate=0, capacity=5) + + +class TokenRefreshUpdate(BaseModel): + rate: float diff --git a/pixl_ehr/src/pixl_ehr/_version.py b/pixl_core/src/core/patient_queue/__init__.py similarity index 89% rename from pixl_ehr/src/pixl_ehr/_version.py rename to pixl_core/src/core/patient_queue/__init__.py index fddacd579..5ba50475e 100644 --- a/pixl_ehr/src/pixl_ehr/_version.py +++ b/pixl_core/src/core/patient_queue/__init__.py @@ -12,5 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version_info__ = ("0", "0", "2") -__version__ = ".".join(__version_info__) +from .subscriber import PixlConsumer + +__all__ = ["PixlConsumer"] diff --git a/patient_queue/src/patient_queue/_base.py b/pixl_core/src/core/patient_queue/_base.py similarity index 100% rename from patient_queue/src/patient_queue/_base.py rename to pixl_core/src/core/patient_queue/_base.py diff --git a/patient_queue/src/patient_queue/producer.py b/pixl_core/src/core/patient_queue/producer.py similarity index 97% rename from patient_queue/src/patient_queue/producer.py rename to pixl_core/src/core/patient_queue/producer.py index aee6430d9..9e1ce6d77 100644 --- a/patient_queue/src/patient_queue/producer.py +++ b/pixl_core/src/core/patient_queue/producer.py @@ -16,7 +16,7 @@ from time import sleep from typing import List -from patient_queue._base import PixlBlockingInterface +from ._base import PixlBlockingInterface LOGGER = logging.getLogger(__name__) diff --git a/patient_queue/src/patient_queue/subscriber.py b/pixl_core/src/core/patient_queue/subscriber.py similarity index 97% rename from patient_queue/src/patient_queue/subscriber.py rename to pixl_core/src/core/patient_queue/subscriber.py index c41db6ea2..329745cac 100644 --- a/patient_queue/src/patient_queue/subscriber.py +++ b/pixl_core/src/core/patient_queue/subscriber.py @@ -17,9 +17,9 @@ from typing import Any, Awaitable, Callable import aio_pika -from patient_queue._base import PixlBlockingInterface, PixlQueueInterface +from core.token_buffer.tokens import TokenBucket -from token_buffer import TokenBucket +from ._base import PixlBlockingInterface, PixlQueueInterface LOGGER = logging.getLogger(__name__) @@ -55,7 +55,6 @@ async def run(self, callback: Callable[[bytes], Awaitable[None]]) -> None: """ async with self._queue.iterator() as queue_iter: async for message in queue_iter: - if not self.token_bucket.has_token: await asyncio.gather( message.reject(requeue=True), diff --git a/patient_queue/src/patient_queue/utils.py b/pixl_core/src/core/patient_queue/utils.py similarity index 100% rename from patient_queue/src/patient_queue/utils.py rename to pixl_core/src/core/patient_queue/utils.py diff --git a/pixl_core/src/core/router.py b/pixl_core/src/core/router.py new file mode 100644 index 000000000..7cf791e4a --- /dev/null +++ b/pixl_core/src/core/router.py @@ -0,0 +1,47 @@ +# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fastapi import APIRouter, HTTPException, status + +from .models import AppState, TokenRefreshUpdate + +state = AppState() +router = APIRouter() + + +@router.get("/heart-beat", summary="Health Check") +async def heart_beat() -> str: + return "OK" + + +@router.post( + "/token-bucket-refresh-rate", summary="Update the refresh rate in items per second" +) +async def update_tb_refresh_rate(item: TokenRefreshUpdate) -> str: + if not isinstance(item.rate, float) or item.rate < 0: + raise HTTPException( + status_code=status.HTTP_406_NOT_ACCEPTABLE, + detail=f"Refresh rate mush be a positive integer. Had {item.rate}", + ) + + state.token_bucket.rate = float(item.rate) + return "Successfully updated the refresh rate" + + +@router.get( + "/token-bucket-refresh-rate", + summary="Get the refresh rate in items per second", + response_model=TokenRefreshUpdate, +) +async def get_tb_refresh_rate() -> TokenRefreshUpdate: + return TokenRefreshUpdate(rate=state.token_bucket.rate) diff --git a/token_buffer/src/token_buffer/tests/__init__.py b/pixl_core/src/core/token_buffer/__init__.py similarity index 91% rename from token_buffer/src/token_buffer/tests/__init__.py rename to pixl_core/src/core/token_buffer/__init__.py index a61138626..57573ef6b 100644 --- a/token_buffer/src/token_buffer/tests/__init__.py +++ b/pixl_core/src/core/token_buffer/__init__.py @@ -11,3 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from .tokens import TokenBucket + +__all__ = ["TokenBucket"] diff --git a/token_buffer/src/token_buffer/tokens.py b/pixl_core/src/core/token_buffer/tokens.py similarity index 100% rename from token_buffer/src/token_buffer/tokens.py rename to pixl_core/src/core/token_buffer/tokens.py diff --git a/pixl_pacs/src/pixl_pacs/tests/__init__.py b/pixl_core/tests/Dockerfile similarity index 79% rename from pixl_pacs/src/pixl_pacs/tests/__init__.py rename to pixl_core/tests/Dockerfile index ff5b83333..5a2123ea7 100644 --- a/pixl_pacs/src/pixl_pacs/tests/__init__.py +++ b/pixl_core/tests/Dockerfile @@ -11,3 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +FROM python:3.10.6-slim-bullseye +SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] + +WORKDIR /core +COPY . . +RUN pip install --no-cache-dir .[test] diff --git a/pixl_rd/src/pixl_rd/tests/__init__.py b/pixl_core/tests/conftest.py similarity index 94% rename from pixl_rd/src/pixl_rd/tests/__init__.py rename to pixl_core/tests/conftest.py index a61138626..e35e72ce8 100644 --- a/pixl_rd/src/pixl_rd/tests/__init__.py +++ b/pixl_core/tests/conftest.py @@ -11,3 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + +os.environ["ENV"] = "test" diff --git a/patient_queue/test/docker-compose.yml b/pixl_core/tests/docker-compose.yml similarity index 91% rename from patient_queue/test/docker-compose.yml rename to pixl_core/tests/docker-compose.yml index 053f32a8f..143787f57 100644 --- a/patient_queue/test/docker-compose.yml +++ b/pixl_core/tests/docker-compose.yml @@ -34,11 +34,11 @@ services: timeout: 30s retries: 3 - python310: + test: container_name: pixl-test-python build: - context: ../.. - dockerfile: patient_queue/test/Dockerfile.python310 + context: .. + dockerfile: tests/Dockerfile depends_on: queue: condition: service_healthy @@ -47,4 +47,4 @@ services: RABBITMQ_PASSWORD: guest RABBITMQ_HOST: queue RABBITMQ_PORT: 5672 - entrypoint: "tail -f /dev/null" + entrypoint: "pytest" diff --git a/patient_queue/src/patient_queue/tests/test_producer.py b/pixl_core/tests/patient_queue/test_producer.py similarity index 92% rename from patient_queue/src/patient_queue/tests/test_producer.py rename to pixl_core/tests/patient_queue/test_producer.py index 28826b333..8145ca296 100644 --- a/patient_queue/src/patient_queue/tests/test_producer.py +++ b/pixl_core/tests/patient_queue/test_producer.py @@ -11,17 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from patient_queue.producer import PixlProducer +from core.patient_queue.producer import PixlProducer +import pytest TEST_QUEUE = "test_publish" +@pytest.mark.pika def test_create_pixl_producer() -> None: """Checks that PixlProducer can be instantiated.""" with PixlProducer(queue_name=TEST_QUEUE) as pp: assert pp.connection_open +@pytest.mark.pika def test_publish() -> None: """Checks that after publishing, there is one message in the queue. Will only work if nothing has been added to queue before.""" diff --git a/patient_queue/src/patient_queue/tests/test_subscriber.py b/pixl_core/tests/patient_queue/test_subscriber.py similarity index 93% rename from patient_queue/src/patient_queue/tests/test_subscriber.py rename to pixl_core/tests/patient_queue/test_subscriber.py index c31db4518..8133d4d59 100644 --- a/patient_queue/src/patient_queue/tests/test_subscriber.py +++ b/pixl_core/tests/patient_queue/test_subscriber.py @@ -16,10 +16,10 @@ from typing import Any, Coroutine, Generator from unittest import TestCase -from patient_queue.producer import PixlProducer -from patient_queue.subscriber import PixlBlockingConsumer, PixlConsumer +from core.patient_queue.producer import PixlProducer +from core.patient_queue.subscriber import PixlBlockingConsumer, PixlConsumer +from core.token_buffer.tokens import TokenBucket import pytest -from token_buffer.tokens import TokenBucket TEST_QUEUE = "test_consume" MESSAGE_BODY = "test".encode("utf-8") @@ -68,6 +68,7 @@ async def consume(msg: bytes) -> None: assert counter == 1 +@pytest.mark.pika def test_consume_all() -> None: """Checks that all messages are returned that have been published before for graceful shutdown.""" diff --git a/patient_queue/src/patient_queue/tests/test_utils.py b/pixl_core/tests/patient_queue/test_utils.py similarity index 95% rename from patient_queue/src/patient_queue/tests/test_utils.py rename to pixl_core/tests/patient_queue/test_utils.py index 869c1e81f..5a467b28a 100644 --- a/patient_queue/src/patient_queue/tests/test_utils.py +++ b/pixl_core/tests/patient_queue/test_utils.py @@ -14,7 +14,7 @@ from datetime import datetime as dt import json -from patient_queue.utils import deserialise, serialise +from core.patient_queue.utils import deserialise, serialise def test_serialise() -> None: diff --git a/token_buffer/src/token_buffer/tests/test_tokens.py b/pixl_core/tests/token_buffer/test_tokens.py similarity index 97% rename from token_buffer/src/token_buffer/tests/test_tokens.py rename to pixl_core/tests/token_buffer/test_tokens.py index 50a4b3821..db7b2b49f 100644 --- a/token_buffer/src/token_buffer/tests/test_tokens.py +++ b/pixl_core/tests/token_buffer/test_tokens.py @@ -14,7 +14,7 @@ import time -from token_buffer import TokenBucket +from core.token_buffer import TokenBucket def test_retrieve_token() -> None: diff --git a/pixl_dcmd/bin/run-tests.sh b/pixl_dcmd/bin/run-tests.sh index 5dbed5d4f..9b0f2c2d6 100755 --- a/pixl_dcmd/bin/run-tests.sh +++ b/pixl_dcmd/bin/run-tests.sh @@ -18,16 +18,4 @@ BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${BIN_DIR%/*}" cd "$PACKAGE_DIR" -echo $PACKAGE_DIR - -CONF_FILE=../setup.cfg - -mypy --config-file ${CONF_FILE} src/pixl_dcmd - -isort --settings-path ${CONF_FILE} src/pixl_dcmd - -black src/pixl_dcmd - -flake8 --config ${CONF_FILE} - ENV=test pytest src/pixl_dcmd/tests diff --git a/pixl_dcmd/src/pixl_dcmd/main.py b/pixl_dcmd/src/pixl_dcmd/main.py index 5dce0bfed..b194e7370 100644 --- a/pixl_dcmd/src/pixl_dcmd/main.py +++ b/pixl_dcmd/src/pixl_dcmd/main.py @@ -15,9 +15,9 @@ from io import BytesIO import logging from os import PathLike +from random import randint import re from typing import Any, BinaryIO, Union -from random import randint import arrow from decouple import config @@ -104,7 +104,6 @@ def get_encrypted_uid(uid: str, salt: bytes) -> str: # For each subcomponent of the suffix: for idx, item in enumerate(suffix_elements): - h = hashlib.sha512() h.update(item.encode("utf-8")) # Add subcomponent. h.update(salt) # Apply salt. @@ -182,11 +181,9 @@ def enforce_whitelist(dataset: dict, tags: dict) -> dict: # For every element: for de in dataset: - keep_el = False # For every entry in the YAML: for i in range(0, len(tags)): - grp = tags[i]["group"] el = tags[i]["element"] op = tags[i]["op"] @@ -242,7 +239,6 @@ def apply_tag_scheme(dataset: dict, tags: dict) -> dict: # For every entry in the YAML: for i in range(0, len(tags)): - name = tags[i]["name"] grp = tags[i]["group"] el = tags[i]["element"] @@ -281,9 +277,7 @@ def apply_tag_scheme(dataset: dict, tags: dict) -> dict: # Handle UIDs that should be encrypted. elif op == "hash-uid": - if [grp, el] in dataset: - message = "Changing: {name} (0x{grp:04x},0x{el:04x})".format( name=name, grp=grp, el=el ) diff --git a/pixl_dcmd/src/setup.py b/pixl_dcmd/src/setup.py index d536c0e8f..1d11939c8 100644 --- a/pixl_dcmd/src/setup.py +++ b/pixl_dcmd/src/setup.py @@ -18,7 +18,7 @@ setup( name="pixl_dcmd", - version=__version__, # noqa: F821 + version=__version__, # type: ignore # noqa: F821 description="DICOM de-identifier", packages=find_packages( exclude=[ diff --git a/pixl_ehr/README.md b/pixl_ehr/README.md index 7021cc804..0670b55b8 100644 --- a/pixl_ehr/README.md +++ b/pixl_ehr/README.md @@ -7,12 +7,25 @@ postgres database. ## Installation -Local installation is possible to run the api tests, but require installing -all the associated PIXL pip packages found at the repo root. +```bash +pip install -e ../pixl_core/ . +python -m spacy download en_core_web_lg # Download spacy language model for deidentification +``` + +## Test + +```bash +pip install -e ../pixl_core/[test] .[test] +pytest -m "not processing" +``` +and the processing tests with +```bash +./tests/run-processing-tests.sh +``` ## Usage -Usage should be from the CLI driver, which interacts with the endpoint. +Usage should be from the CLI driver, which calls the HTTP endpoints. ## Notes diff --git a/pixl_ehr/pyproject.toml b/pixl_ehr/pyproject.toml new file mode 100644 index 000000000..79b76c552 --- /dev/null +++ b/pixl_ehr/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "pixl_ehr" +version = "0.0.2" +authors = [ + { name="PIXL authors" }, +] +description = "PIXL electronic health record extractor" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3" +] +dependencies = [ + "core", + "uvicorn==0.23.2", + "python-decouple==3.6", + "psycopg2==2.9.5", + "azure-identity==1.12.0", + "azure-storage-blob==12.14.1", + "PyYAML==6.0", + "presidio-analyzer==2.2.29", + "presidio-anonymizer==2.2.29" +] + +[project.optional-dependencies] +test = [ + "pytest==7.4.2" +] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +markers = ["processing"] + +[tool.setuptools.package-data] +pixl_ehr = ["sql/*.sql", "report_deid/*.txt"] diff --git a/pixl_ehr/src/pixl_ehr/_databases.py b/pixl_ehr/src/pixl_ehr/_databases.py index 140a1178d..c799024b8 100644 --- a/pixl_ehr/src/pixl_ehr/_databases.py +++ b/pixl_ehr/src/pixl_ehr/_databases.py @@ -15,9 +15,10 @@ from typing import TYPE_CHECKING, List, Optional from decouple import config -from pixl_ehr._queries import SQLQuery import psycopg2 as pypg +from pixl_ehr._queries import SQLQuery + logger = logging.getLogger("uvicorn") if TYPE_CHECKING: @@ -34,7 +35,6 @@ def __init__( password: Optional[str] = None, host: Optional[str] = None, ) -> None: - connection_string = ( f"dbname={db_name} user={username} password={password} host={host}" ) @@ -54,7 +54,6 @@ def execute(self, query: SQLQuery) -> Optional[tuple]: return None if row is None else tuple(row) def execute_or_raise(self, query: SQLQuery, error_str: str = "Failed") -> tuple: - result = self.execute(query) if result is None: diff --git a/pixl_ehr/src/pixl_ehr/_processing.py b/pixl_ehr/src/pixl_ehr/_processing.py index 763f51b95..40a0d388a 100644 --- a/pixl_ehr/src/pixl_ehr/_processing.py +++ b/pixl_ehr/src/pixl_ehr/_processing.py @@ -20,13 +20,14 @@ from pathlib import Path from typing import Optional +from core.patient_queue.utils import deserialise from decouple import config -from patient_queue.utils import deserialise +import requests + from pixl_ehr._databases import EMAPStar, PIXLDatabase from pixl_ehr._queries import SQLQuery -import requests -from pixl_rd import deidentify_text +from .report_deid import deidentify_text logger = logging.getLogger("uvicorn") logger.setLevel(os.environ.get("LOG_LEVEL", "WARNING")) @@ -222,7 +223,6 @@ def emap_name(self) -> str: """Name of this observation type in an EMAP star schema, e.g. HEIGHT""" def update(self, data: PatientEHRData) -> None: - if data.acquisition_datetime is None: raise RuntimeError("Cannot update a height without an acquisition") diff --git a/pixl_ehr/src/pixl_ehr/_queries.py b/pixl_ehr/src/pixl_ehr/_queries.py index fdfe1e70c..7bc62ad6c 100644 --- a/pixl_ehr/src/pixl_ehr/_queries.py +++ b/pixl_ehr/src/pixl_ehr/_queries.py @@ -17,7 +17,6 @@ class SQLQuery: def __init__(self, filepath: Path, context: dict): - self.values: List[str] = [] self._filepath = filepath self._lines = open(filepath, "r").readlines() diff --git a/pixl_ehr/src/pixl_ehr/main.py b/pixl_ehr/src/pixl_ehr/main.py index a9fdf7c98..d75c91d07 100644 --- a/pixl_ehr/src/pixl_ehr/main.py +++ b/pixl_ehr/src/pixl_ehr/main.py @@ -12,82 +12,39 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio -from dataclasses import dataclass +import importlib.metadata import logging from azure.identity import EnvironmentCredential from azure.storage.blob import BlobServiceClient +from core.patient_queue import PixlConsumer +from core.router import router, state from decouple import config -from fastapi import FastAPI, HTTPException, status +from fastapi import FastAPI from fastapi.responses import JSONResponse -from patient_queue.subscriber import PixlConsumer -from pixl_ehr._databases import PIXLDatabase -from pixl_ehr._processing import process_message -from pydantic import BaseModel -from token_buffer import TokenBucket - -from ._version import __version__ +from ._databases import PIXLDatabase +from ._processing import process_message QUEUE_NAME = "ehr" app = FastAPI( title="ehr-api", description="EHR extraction service", - version=__version__, + version=importlib.metadata.version("pixl_ehr"), default_response_class=JSONResponse, ) +app.include_router(router) logger = logging.getLogger("uvicorn") -@dataclass -class AppState: - token_bucket = TokenBucket(rate=0, capacity=5) - - -state = AppState() - - @app.on_event("startup") async def startup_event() -> None: async with PixlConsumer(QUEUE_NAME, token_bucket=state.token_bucket) as consumer: asyncio.create_task(consumer.run(callback=process_message)) -@app.get("/heart-beat", summary="Health Check") -async def heart_beat() -> str: - return "OK" - - -class TokenRefreshUpdate(BaseModel): - rate: float - - -@app.post( - "/token-bucket-refresh-rate", summary="Update the refresh rate in items per second" -) -async def update_tb_refresh_rate(item: TokenRefreshUpdate) -> str: - - if not isinstance(item.rate, float) or item.rate < 0: - raise HTTPException( - status_code=status.HTTP_406_NOT_ACCEPTABLE, - detail=f"Refresh rate mush be a positive float. Had {item.rate}", - ) - - state.token_bucket.rate = item.rate - return "Successfully updated the refresh rate" - - -@app.get( - "/token-bucket-refresh-rate", - summary="Get the refresh rate in items per second", - response_model=TokenRefreshUpdate, -) -async def get_tb_refresh_rate() -> BaseModel: - return TokenRefreshUpdate(rate=state.token_bucket.rate) - - @app.get( "/az-copy-current", summary="Copy the current state of the PIXL anon EHR schema to azure", diff --git a/pixl_ehr/src/pixl_ehr/tests/__init__.py b/pixl_ehr/src/pixl_ehr/report_deid/__init__.py similarity index 90% rename from pixl_ehr/src/pixl_ehr/tests/__init__.py rename to pixl_ehr/src/pixl_ehr/report_deid/__init__.py index ff5b83333..ac59346c4 100644 --- a/pixl_ehr/src/pixl_ehr/tests/__init__.py +++ b/pixl_ehr/src/pixl_ehr/report_deid/__init__.py @@ -11,3 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .deid import deidentify_text + +__all__ = ["deidentify_text"] diff --git a/pixl_rd/src/pixl_rd/main.py b/pixl_ehr/src/pixl_ehr/report_deid/deid.py similarity index 99% rename from pixl_rd/src/pixl_rd/main.py rename to pixl_ehr/src/pixl_ehr/report_deid/deid.py index cc0b2c1f3..fd1d42c32 100644 --- a/pixl_rd/src/pixl_rd/main.py +++ b/pixl_ehr/src/pixl_ehr/report_deid/deid.py @@ -49,7 +49,6 @@ def deidentify_text(text: str) -> str: def _presidio_anonymise(text: str) -> str: - results = _analyzer.analyze( text=text, entities=["DATE_TIME", "PERSON"], @@ -62,7 +61,6 @@ def _presidio_anonymise(text: str) -> str: def _remove_case_insensitive_patterns(text: str) -> str: - patterns = ( r"reporting corresponds to ([^:]+)", # Remove any words between ...to and : r"(\S+@\S+)", # Matches any email address @@ -80,7 +78,6 @@ def _remove_case_insensitive_patterns(text: str) -> str: def _remove_case_sensitive_patterns(text: str) -> str: - patterns = ( r"((?:[A-Z][a-z]+) (?:[A-Z][a-z]+)-(?:[A-Z][a-z]+))", # Hyphenated full names r"\.\s{0,2}((?:[A-Z][a-z]+\s?){2})", # Remove two title case after a full stop @@ -124,7 +121,6 @@ def _possible_professions_str() -> str: def _remove_linebreaks_after_title_case_lines(text: str) -> str: - lines = text.split("\n") text = "" for i, line in enumerate(lines): diff --git a/pixl_rd/src/pixl_rd/exclusions.txt b/pixl_ehr/src/pixl_ehr/report_deid/exclusions.txt similarity index 100% rename from pixl_rd/src/pixl_rd/exclusions.txt rename to pixl_ehr/src/pixl_ehr/report_deid/exclusions.txt diff --git a/pixl_ehr/src/requirements.txt b/pixl_ehr/src/requirements.txt deleted file mode 100644 index 25cb515e7..000000000 --- a/pixl_ehr/src/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -black==22.8.0 -fastapi==0.103.2 -flake8==5.0.4 -flake8-return==1.1.3 -hypothesis==6.56.0 -isort==5.10.1 -environs==9.5.0 -mypy==1.6.0 -httpx==0.25.0 -pytest==7.4.2 -requests==2.31.0 -uvicorn==0.23.2 -pydantic==1.10.13 -types-requests==2.31.0 -psycopg2-binary==2.9.5 -azure-identity==1.12.0 -azure-storage-blob==12.14.1 -PyYAML==6.0 -types-PyYAML==6.0.* -python-decouple==3.6 diff --git a/pixl_ehr/src/setup.py b/pixl_ehr/src/setup.py deleted file mode 100644 index 9ee103096..000000000 --- a/pixl_ehr/src/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from setuptools import setup, find_packages - -exec(open("pixl_ehr/_version.py").read()) - -setup( - name="pixl_ehr", - version=__version__, # noqa: F821 - author="Tom Young", - url="https://github.com/UCLH-Foundry/PIXL", - description="PIXL electronic health record extractor", - packages=find_packages( - exclude=["*tests", "*.tests.*"], - ), - package_data={ - "pixl_ehr": ["sql/*.sql"], - }, - python_requires=">=3.10", -) diff --git a/pixl_ehr/test/run-lint-and-api-tests.sh b/pixl_ehr/test/run-lint-and-api-tests.sh deleted file mode 100755 index b51819336..000000000 --- a/pixl_ehr/test/run-lint-and-api-tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -euxo pipefail - -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -PACKAGE_DIR="${BIN_DIR%/*}" -cd "$PACKAGE_DIR" || exit - -pip install -r src/requirements.txt - -CONF_FILE=../setup.cfg -mypy --config-file ${CONF_FILE} src/pixl_ehr -isort --settings-path ${CONF_FILE} src/pixl_ehr -black src/pixl_ehr -flake8 --config ${CONF_FILE} src/pixl_ehr - -ENV="test" pytest src/pixl_ehr/tests/test_app.py diff --git a/pixl_ehr/test/run-tests.sh b/pixl_ehr/test/run-tests.sh deleted file mode 100755 index c89a77016..000000000 --- a/pixl_ehr/test/run-tests.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -euxo pipefail - -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -PACKAGE_DIR="${BIN_DIR%/*}" - -. "${PACKAGE_DIR}"/test/run-lint-and-api-tests.sh -. "${PACKAGE_DIR}"/test/run-processing-tests.sh diff --git a/pixl_rd/src/pixl_rd/tests/data/README.md b/pixl_ehr/tests/data/README.md similarity index 100% rename from pixl_rd/src/pixl_rd/tests/data/README.md rename to pixl_ehr/tests/data/README.md diff --git a/pixl_rd/src/pixl_rd/tests/data/names.csv b/pixl_ehr/tests/data/names.csv similarity index 100% rename from pixl_rd/src/pixl_rd/tests/data/names.csv rename to pixl_ehr/tests/data/names.csv diff --git a/pixl_rd/src/pixl_rd/tests/data/reports.csv b/pixl_ehr/tests/data/reports.csv similarity index 100% rename from pixl_rd/src/pixl_rd/tests/data/reports.csv rename to pixl_ehr/tests/data/reports.csv diff --git a/pixl_ehr/test/docker-compose.yml b/pixl_ehr/tests/docker-compose.yml similarity index 98% rename from pixl_ehr/test/docker-compose.yml rename to pixl_ehr/tests/docker-compose.yml index fe0ad0cad..e2030919e 100644 --- a/pixl_ehr/test/docker-compose.yml +++ b/pixl_ehr/tests/docker-compose.yml @@ -20,6 +20,8 @@ services: build: context: ../../ dockerfile: ./docker/ehr-api/Dockerfile + args: + TEST: true depends_on: queue: condition: service_healthy diff --git a/pixl_pacs/test/run-processing-tests.sh b/pixl_ehr/tests/run-processing-tests.sh similarity index 87% rename from pixl_pacs/test/run-processing-tests.sh rename to pixl_ehr/tests/run-processing-tests.sh index 8da69ae7f..9afbef241 100755 --- a/pixl_pacs/test/run-processing-tests.sh +++ b/pixl_ehr/tests/run-processing-tests.sh @@ -30,8 +30,12 @@ set -euxo pipefail THIS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${THIS_DIR%/*}" -cd "$PACKAGE_DIR"/test || exit +cd "$PACKAGE_DIR"/tests || exit + +if [ -z "${INFORMDB_PAT+x}" ]; then + echo "INFORMDB_PAT must be set as an environment variable" && exit 1 +fi docker compose up -d --build -docker exec pixl-test-pacs-api /bin/bash -c "pytest pixl_pacs/tests/test_processing.py" +docker exec pixl-test-ehr-api /bin/bash -c "pytest -m processing" docker compose down diff --git a/pixl_pacs/test/run-tests.sh b/pixl_ehr/tests/run-tests.sh similarity index 89% rename from pixl_pacs/test/run-tests.sh rename to pixl_ehr/tests/run-tests.sh index 6d8dc441b..f40822ad9 100755 --- a/pixl_pacs/test/run-tests.sh +++ b/pixl_ehr/tests/run-tests.sh @@ -17,5 +17,5 @@ set -euxo pipefail BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${BIN_DIR%/*}" -. "${PACKAGE_DIR}"/test/run-lint.sh -. "${PACKAGE_DIR}"/test/run-processing-tests.sh +pytest -m "not processing" +. "${PACKAGE_DIR}"/tests/run-processing-tests.sh diff --git a/pixl_ehr/src/pixl_ehr/tests/test_app.py b/pixl_ehr/tests/test_app.py similarity index 96% rename from pixl_ehr/src/pixl_ehr/tests/test_app.py rename to pixl_ehr/tests/test_app.py index 8f82f7ab4..567b1cfe1 100644 --- a/pixl_ehr/src/pixl_ehr/tests/test_app.py +++ b/pixl_ehr/tests/test_app.py @@ -15,8 +15,10 @@ This file contains unit tests for the API that do not require any test services """ from fastapi.testclient import TestClient -from pixl_ehr.main import AppState, app, state +from pixl_ehr.main import app, state + +AppState = state.__class__ client = TestClient(app) @@ -30,19 +32,16 @@ def test_initial_state_has_no_token() -> None: def test_updating_the_token_refresh_rate_to_negative_fails() -> None: - response = client.post("/token-bucket-refresh-rate", json={"rate": -1}) assert response.is_error def test_updating_the_token_refresh_rate_to_string_fails() -> None: - response = client.post("/token-bucket-refresh-rate", json={"rate": "a string"}) assert response.is_error def test_updating_the_token_refresh_rate_updates_state() -> None: - response = client.post("/token-bucket-refresh-rate", json={"rate": 1}) assert state.token_bucket.has_token assert response.status_code == 200 diff --git a/pixl_rd/src/pixl_rd/tests/test_deidentification.py b/pixl_ehr/tests/test_deidentification.py similarity index 99% rename from pixl_rd/src/pixl_rd/tests/test_deidentification.py rename to pixl_ehr/tests/test_deidentification.py index 78c3c4f70..bfcc9528e 100644 --- a/pixl_rd/src/pixl_rd/tests/test_deidentification.py +++ b/pixl_ehr/tests/test_deidentification.py @@ -16,7 +16,9 @@ from pathlib import Path from typing import List, Tuple -from pixl_rd.main import ( +import pytest + +from pixl_ehr.report_deid.deid import ( _remove_any_excluded_words, _remove_any_trailing_tags, _remove_case_insensitive_patterns, @@ -24,7 +26,6 @@ _remove_linebreaks_after_title_case_lines, deidentify_text, ) -import pytest THIS_DIR = Path(os.path.dirname(os.path.abspath(__file__))) @@ -42,11 +43,9 @@ def _tuple_from(line: str) -> Tuple[str, str]: def test_patient_name_is_redacted(required_accuracy: float = 0.85) -> None: - results = [] for first_name, last_name in _patient_names_from_names_csv(): - report_text = f"{first_name} {last_name} was x-rayed and the prognosis is X" anon_text = deidentify_text(report_text) results.append(first_name not in anon_text and last_name not in anon_text) @@ -56,7 +55,6 @@ def test_patient_name_is_redacted(required_accuracy: float = 0.85) -> None: def test_signed_by_section_is_removed() -> None: - first_name, last_name, date = info = "John", "Doe", "01/01/20" anon_text = deidentify_text( @@ -69,7 +67,6 @@ def test_signed_by_section_is_removed() -> None: @pytest.mark.parametrize("id_name", ["GMC", "HCPC"]) def test_block_with_excluded_identifiers_are_removed(id_name: str) -> None: - header, footer = "A xray report with information", "Other text" first_name, last_name, num = info = "John", "Doe", "0123456" @@ -90,9 +87,7 @@ def test_block_with_excluded_identifiers_are_removed(id_name: str) -> None: # using ":" or " " as a delimiter is not redacted by Presidio @pytest.mark.parametrize("delimiter", ["/", "-"]) def test_possible_dates_are_removed(delimiter: str) -> None: - for day, month, year in [(1, 3, 2019)]: - date_strings = [ f"{day}{delimiter}{month}{delimiter}{year}", f"{month}{delimiter}{day}{delimiter}{year}", @@ -105,7 +100,6 @@ def test_possible_dates_are_removed(delimiter: str) -> None: def test_accession_nums_gmc_nhs_email() -> None: - gmc_number = "12345" email_address = "jon.smith@nhs.net" accession_number = "RRV012734923" @@ -123,7 +117,6 @@ def test_accession_nums_gmc_nhs_email() -> None: def test_linebreaks_are_removed_from_possible_identifying_section() -> None: - text = "A report.\nJohn Smith\nReporting Radiographer\nOther text after" expected_text = "A report.\nJohn Smith Reporting Radiographer Other text after" @@ -132,7 +125,6 @@ def test_linebreaks_are_removed_from_possible_identifying_section() -> None: @pytest.mark.parametrize("initials", ["JS", "AJ", "AO\t", "ER "]) def test_initials_are_removed_from_end_of_string(initials: str) -> None: - text = f"Some text. {initials}" assert initials.strip() not in _remove_case_sensitive_patterns(text) @@ -179,13 +171,11 @@ def test_name_from_exclusion_list_is_removed(name: str) -> None: @pytest.mark.parametrize("date_str", ["14 Jun 2022", "1 Jan 2022", "21 March 2022"]) def test_abbreviated_date_is_removed(date_str: str) -> None: - text = f"A sentence {date_str} then other things" assert date_str not in _remove_case_insensitive_patterns(text) def test_remove_trailing_tags() -> None: - text = "A sentence XXX> other things" expected_text = "A sentence XXX other things" assert _remove_any_trailing_tags(text) == expected_text @@ -193,6 +183,5 @@ def test_remove_trailing_tags() -> None: @pytest.mark.parametrize("digits_str", ["14", "01", "7", "31"]) def test_remove_up_to_two_digits_before_datetime(digits_str: str) -> None: - text = f"A sentence referring to {digits_str} then other text" assert digits_str not in _remove_case_insensitive_patterns(text) diff --git a/pixl_ehr/src/pixl_ehr/tests/test_processing.py b/pixl_ehr/tests/test_processing.py similarity index 98% rename from pixl_ehr/src/pixl_ehr/tests/test_processing.py rename to pixl_ehr/tests/test_processing.py index 9eed2c6b1..ede5400b2 100644 --- a/pixl_ehr/src/pixl_ehr/tests/test_processing.py +++ b/pixl_ehr/tests/test_processing.py @@ -20,13 +20,14 @@ from datetime import datetime from typing import List +from core.patient_queue.utils import serialise from decouple import config -from patient_queue.utils import serialise -from pixl_ehr._databases import PIXLDatabase, WriteableDatabase -from pixl_ehr._processing import process_message from psycopg2.errors import UniqueViolation import pytest +from pixl_ehr._databases import PIXLDatabase, WriteableDatabase +from pixl_ehr._processing import process_message + pytest_plugins = ("pytest_asyncio",) @@ -85,7 +86,6 @@ def execute_query_string(self, query: str, values: list) -> tuple: def insert_row_into_emap_star_schema( table_name: str, col_names: List[str], values: List ) -> None: - db = WritableEMAPStar() cols = ",".join(col_names) vals = ",".join("%s" for _ in range(len(col_names))) @@ -157,9 +157,9 @@ def insert_data_into_emap_star_schema() -> None: ) +@pytest.mark.processing @pytest.mark.asyncio async def test_message_processing() -> None: - insert_data_into_emap_star_schema() await process_message(message_body) diff --git a/pixl_pacs/README.md b/pixl_pacs/README.md index d1fab7511..e7f5d8c5a 100644 --- a/pixl_pacs/README.md +++ b/pixl_pacs/README.md @@ -6,6 +6,18 @@ different processing to transfer DICOM studies from the VNA to the "raw" Orthanc instance, from which the anonymisation and push over DICOMWeb to are automatic. +## Installation + +```bash +pip install -e ../pixl_core/ . +``` + +## Test + +```bash +./tests/run-tests.sh +``` + ## Usage Usage should be from the CLI driver, which interacts with the endpoint. diff --git a/pixl_pacs/pyproject.toml b/pixl_pacs/pyproject.toml new file mode 100644 index 000000000..325ba2ac7 --- /dev/null +++ b/pixl_pacs/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "pixl_pacs" +version = "0.0.2" +authors = [ + { name="PIXL authors" }, +] +description = "PIXL image extractor" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3" +] +dependencies = [ + "core", + "uvicorn==0.23.2", + "pydicom==2.3.0", + "python-decouple==3.6" +] + +[project.optional-dependencies] +test = [ + "pytest==7.4.2", + "pytest-asyncio==0.21.1" +] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +markers = ["processing"] diff --git a/pixl_pacs/src/pixl_pacs/_orthanc.py b/pixl_pacs/src/pixl_pacs/_orthanc.py index 88337c984..18cb590d1 100644 --- a/pixl_pacs/src/pixl_pacs/_orthanc.py +++ b/pixl_pacs/src/pixl_pacs/_orthanc.py @@ -16,8 +16,8 @@ import logging from typing import Any, Optional -import requests from decouple import config +import requests from requests.auth import HTTPBasicAuth logger = logging.getLogger("uvicorn") @@ -25,7 +25,6 @@ class Orthanc(ABC): def __init__(self, url: str, username: str, password: str): - self._url = url.rstrip("/") self._username = username self._password = password @@ -52,7 +51,7 @@ def query_remote(self, data: dict, modality: str) -> Optional[str]: response = self._post( f"/modalities/{modality}/query", data=data, - timeout=config("PIXL_QUERY_TIMEOUT", default=10, cast=float) + timeout=config("PIXL_QUERY_TIMEOUT", default=10, cast=float), ) logger.debug(f"Query response: {response}") @@ -77,7 +76,9 @@ def _get(self, path: str) -> Any: def _post(self, path: str, data: dict, timeout: Optional[float] = None) -> Any: return _deserialise( - requests.post(f"{self._url}{path}", json=data, auth=self._auth, timeout=timeout) + requests.post( + f"{self._url}{path}", json=data, auth=self._auth, timeout=timeout + ) ) diff --git a/pixl_pacs/src/pixl_pacs/_processing.py b/pixl_pacs/src/pixl_pacs/_processing.py index 06a2cd511..8c683ee52 100644 --- a/pixl_pacs/src/pixl_pacs/_processing.py +++ b/pixl_pacs/src/pixl_pacs/_processing.py @@ -18,8 +18,9 @@ import os from time import time -from patient_queue.utils import deserialise +from core.patient_queue.utils import deserialise from decouple import config + from pixl_pacs._orthanc import Orthanc, PIXLRawOrthanc logger = logging.getLogger("uvicorn") @@ -48,8 +49,7 @@ async def process_message(message_body: bytes) -> None: start_time = time() while job_state != "Success": - - if (time() - start_time) > float(config("PIXL_DICOM_TRANSFER_TIMEOUT")): + if (time() - start_time) > config("PIXL_DICOM_TRANSFER_TIMEOUT", cast=float): raise TimeoutError( f"Failed to transfer {message_body.decode()} within " f"{config('PIXL_DICOM_TRANSFER_TIMEOUT')} seconds" diff --git a/pixl_pacs/src/pixl_pacs/_version.py b/pixl_pacs/src/pixl_pacs/_version.py deleted file mode 100644 index fddacd579..000000000 --- a/pixl_pacs/src/pixl_pacs/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version_info__ = ("0", "0", "2") -__version__ = ".".join(__version_info__) diff --git a/pixl_pacs/src/pixl_pacs/main.py b/pixl_pacs/src/pixl_pacs/main.py index 2faa89e25..c27d521cf 100644 --- a/pixl_pacs/src/pixl_pacs/main.py +++ b/pixl_pacs/src/pixl_pacs/main.py @@ -12,72 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio -from dataclasses import dataclass +import importlib.metadata import logging -from fastapi import FastAPI, HTTPException, status +from core.patient_queue.subscriber import PixlConsumer +from core.router import router, state +from fastapi import FastAPI from fastapi.responses import JSONResponse -from patient_queue.subscriber import PixlConsumer -from pixl_pacs._processing import process_message -from pydantic import BaseModel -from token_buffer import TokenBucket - -from ._version import __version__ +from ._processing import process_message QUEUE_NAME = "pacs" app = FastAPI( title="pacs-api", description="PACS extraction service", - version=__version__, + version=importlib.metadata.version("pixl_pacs"), default_response_class=JSONResponse, ) +app.include_router(router) logger = logging.getLogger("uvicorn") -@dataclass -class AppState: - token_bucket = TokenBucket(rate=0, capacity=5) - - -state = AppState() - - @app.on_event("startup") async def startup_event() -> None: async with PixlConsumer(QUEUE_NAME, token_bucket=state.token_bucket) as consumer: asyncio.create_task(consumer.run(callback=process_message)) - - -@app.get("/heart-beat", summary="Health Check") -async def heart_beat() -> str: - return "OK" - - -class TokenRefreshUpdate(BaseModel): - rate: float - - -@app.post( - "/token-bucket-refresh-rate", summary="Update the refresh rate in items per second" -) -async def update_tb_refresh_rate(item: TokenRefreshUpdate) -> str: - - if not isinstance(item.rate, float) or item.rate < 0: - raise HTTPException( - status_code=status.HTTP_406_NOT_ACCEPTABLE, - detail=f"Refresh rate mush be a positive integer. Had {item.rate}", - ) - - state.token_bucket.rate = float(item.rate) - return "Successfully updated the refresh rate" - - -@app.get( - "/token-bucket-refresh-rate", summary="Get the refresh rate in items per second", - response_model=TokenRefreshUpdate, -) -async def get_tb_refresh_rate() -> BaseModel: - return TokenRefreshUpdate(rate=state.token_bucket.rate) diff --git a/pixl_pacs/src/requirements.txt b/pixl_pacs/src/requirements.txt deleted file mode 100644 index ec4928d0c..000000000 --- a/pixl_pacs/src/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -black==22.8.0 -flake8==5.0.4 -flake8-return==1.1.3 -isort==5.10.1 -environs==9.5.0 -mypy==0.991 -pytest==7.1.3 -pytest-asyncio==0.20.2 -click==8.1.3 -coloredlogs==15.0.1 -fastapi==0.85.2 -pydantic==1.9.2 -uvicorn==0.19.0 -requests==2.28.1 -types-requests==2.28.* -pydicom==2.3.0 -python-decouple==3.6 diff --git a/pixl_pacs/src/setup.py b/pixl_pacs/src/setup.py deleted file mode 100644 index d5af86c00..000000000 --- a/pixl_pacs/src/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from setuptools import setup, find_packages - -exec(open("pixl_pacs/_version.py").read()) - -setup( - name="pixl_pacs", - version=__version__, # noqa: F821 - author="Tom Young", - url="https://github.com/UCLH-Foundry/PIXL", - description="PIXL image extractor", - packages=find_packages( - exclude=["*tests", "*.tests.*"], - ), - python_requires=">=3.10", -) diff --git a/pixl_pacs/test/run-lint.sh b/pixl_pacs/test/run-lint.sh deleted file mode 100755 index 177f7e9c4..000000000 --- a/pixl_pacs/test/run-lint.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -euxo pipefail - -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -PACKAGE_DIR="${BIN_DIR%/*}" -cd "$PACKAGE_DIR" || exit - -pip install -r src/requirements.txt - -CONF_FILE=../setup.cfg -mypy --config-file ${CONF_FILE} src/pixl_pacs -isort --settings-path ${CONF_FILE} src/pixl_pacs -black src/pixl_pacs -flake8 --config ${CONF_FILE} src/pixl_pacs diff --git a/pixl_pacs/test/Dockerfile.orthanc_raw_test b/pixl_pacs/tests/Dockerfile.orthanc_raw_test similarity index 100% rename from pixl_pacs/test/Dockerfile.orthanc_raw_test rename to pixl_pacs/tests/Dockerfile.orthanc_raw_test diff --git a/pixl_pacs/test/docker-compose.yml b/pixl_pacs/tests/docker-compose.yml similarity index 96% rename from pixl_pacs/test/docker-compose.yml rename to pixl_pacs/tests/docker-compose.yml index 7418550e0..97846f19f 100644 --- a/pixl_pacs/test/docker-compose.yml +++ b/pixl_pacs/tests/docker-compose.yml @@ -26,6 +26,8 @@ services: build: context: ../../ dockerfile: ./docker/pacs-api/Dockerfile + args: + TEST: true depends_on: queue: condition: service_healthy @@ -113,7 +115,9 @@ services: queue: container_name: pixl-test-queue - image: rabbitmq:3.11.2-management + build: + context: ../.. + dockerfile: docker/queue/Dockerfile healthcheck: test: rabbitmq-diagnostics -q ping interval: 30s diff --git a/pixl_pacs/test/orthanc_raw_config/dicom.json b/pixl_pacs/tests/orthanc_raw_config/dicom.json similarity index 100% rename from pixl_pacs/test/orthanc_raw_config/dicom.json rename to pixl_pacs/tests/orthanc_raw_config/dicom.json diff --git a/pixl_pacs/test/orthanc_raw_config/orthanc.json b/pixl_pacs/tests/orthanc_raw_config/orthanc.json similarity index 100% rename from pixl_pacs/test/orthanc_raw_config/orthanc.json rename to pixl_pacs/tests/orthanc_raw_config/orthanc.json diff --git a/pixl_ehr/test/run-processing-tests.sh b/pixl_pacs/tests/run-tests.sh similarity index 92% rename from pixl_ehr/test/run-processing-tests.sh rename to pixl_pacs/tests/run-tests.sh index e3f72cd6e..80e59a137 100755 --- a/pixl_ehr/test/run-processing-tests.sh +++ b/pixl_pacs/tests/run-tests.sh @@ -30,8 +30,8 @@ set -euxo pipefail THIS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PACKAGE_DIR="${THIS_DIR%/*}" -cd "$PACKAGE_DIR"/test || exit +cd "$PACKAGE_DIR"/tests || exit docker compose up -d --build -docker exec pixl-test-ehr-api /bin/bash -c "pytest pixl_ehr/tests/test_processing.py" +docker exec pixl-test-pacs-api /bin/bash -c "pytest -m processing" docker compose down diff --git a/pixl_pacs/src/pixl_pacs/tests/test_processing.py b/pixl_pacs/tests/test_processing.py similarity index 97% rename from pixl_pacs/src/pixl_pacs/tests/test_processing.py rename to pixl_pacs/tests/test_processing.py index 65bd87a99..ba3898364 100644 --- a/pixl_pacs/src/pixl_pacs/tests/test_processing.py +++ b/pixl_pacs/tests/test_processing.py @@ -18,14 +18,15 @@ from datetime import datetime import os -from patient_queue.utils import serialise -from pixl_pacs._orthanc import Orthanc, PIXLRawOrthanc -from pixl_pacs._processing import ImagingStudy, process_message +from core.patient_queue.utils import serialise from decouple import config from pydicom import dcmread from pydicom.data import get_testdata_file import pytest +from pixl_pacs._orthanc import Orthanc, PIXLRawOrthanc +from pixl_pacs._processing import ImagingStudy, process_message + pytest_plugins = ("pytest_asyncio",) ACCESSION_NUMBER = "abc" @@ -64,9 +65,9 @@ def add_image_to_fake_vna(image_filename: str = "test.dcm") -> None: vna.upload(image_filename) +@pytest.mark.processing @pytest.mark.asyncio async def test_image_processing() -> None: - add_image_to_fake_vna() study = ImagingStudy.from_message(message_body) orthanc_raw = PIXLRawOrthanc() diff --git a/pixl_rd/README.md b/pixl_rd/README.md deleted file mode 100644 index c3ee00837..000000000 --- a/pixl_rd/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# PIXL Report De-identifier - -Reports are de-identified based on a combined [Presido](https://microsoft.github.io/presidio/) + -[regex](https://en.wikipedia.org/wiki/Regular_expression) approach and aims to -remove all absolute identifiers (e.g. NHS number), almost all >99% full names -and most dates/partial names. - -### Notes - -- Linebreaks are not always preserved and some partial text overlaps of masks exist -- Presido identifies nouns as names on occasion - - -*** - -### Local installation - -```bash -pip install -r requirements.txt -python -m spacy download en_core_web_lg # Download spacy language model -``` diff --git a/pixl_rd/bin/run-tests.sh b/pixl_rd/bin/run-tests.sh deleted file mode 100755 index 1665d271a..000000000 --- a/pixl_rd/bin/run-tests.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eo pipefail - -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -PACKAGE_DIR="${BIN_DIR%/*}" -cd "$PACKAGE_DIR" - -CONF_FILE=../setup.cfg - -mypy --config-file ${CONF_FILE} src/pixl_rd - -isort --settings-path ${CONF_FILE} src/pixl_rd - -black src/pixl_rd - -flake8 --config ${CONF_FILE} - -ENV="test" pytest src/pixl_rd diff --git a/pixl_rd/src/pixl_rd/__init__.py b/pixl_rd/src/pixl_rd/__init__.py deleted file mode 100644 index 9da3e1961..000000000 --- a/pixl_rd/src/pixl_rd/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from pixl_rd.main import deidentify_text - -from ._version import __version__, __version_info__ - -__all__ = ["deidentify_text"] diff --git a/pixl_rd/src/pixl_rd/_version.py b/pixl_rd/src/pixl_rd/_version.py deleted file mode 100644 index fddacd579..000000000 --- a/pixl_rd/src/pixl_rd/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version_info__ = ("0", "0", "2") -__version__ = ".".join(__version_info__) diff --git a/pixl_rd/src/requirements.txt b/pixl_rd/src/requirements.txt deleted file mode 100644 index 746869322..000000000 --- a/pixl_rd/src/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -black==22.8.0 -flake8==5.0.4 -flake8-return==1.1.3 -isort==5.10.1 -environs==9.5.0 -mypy==1.6.0 -pytest==7.4.2 -presidio-analyzer==2.2.29 -presidio-anonymizer==2.2.29 -python-decouple==3.6 diff --git a/pixl_rd/src/setup.py b/pixl_rd/src/setup.py deleted file mode 100644 index 9f1a576b4..000000000 --- a/pixl_rd/src/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import find_packages, setup - -exec(open("pixl_rd/_version.py").read()) - -setup( - name="pixl_rd", - version=__version__, # noqa: F821 - description="Radiology report de-identifier", - packages=find_packages( - exclude=[ - "*tests", - "*.tests.*", - ], - ), - package_data={ - "pixl_rd": ["*.txt"], - }, - python_requires=">=3.10", -) diff --git a/scripts/cmove_all_studies.py b/scripts/cmove_all_studies.py index bdb89e542..582087548 100644 --- a/scripts/cmove_all_studies.py +++ b/scripts/cmove_all_studies.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """C-Move all studies from Orthanc Raw to Anon""" -import os -import requests import argparse - +from datetime import datetime, timedelta from json import JSONDecodeError +import os from typing import Any -from datetime import datetime, timedelta + +import requests from requests.auth import HTTPBasicAuth os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost" @@ -30,7 +30,7 @@ def __init__( url=f"http://localhost:{os.environ['ORTHANC_PORT']}", username=os.environ["ORTHANC_USERNAME"], password=os.environ["ORTHANC_PASSWORD"], - anon_aet=os.environ["ORTHANC_ANON_AE_TITLE"] + anon_aet=os.environ["ORTHANC_ANON_AE_TITLE"], ): self._url = url.rstrip("/") self._username = username @@ -68,17 +68,15 @@ def _deserialise(response: requests.Response) -> Any: def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() parser.add_argument( "start_date", - help="Date from which to trigger C-Move from in the format: YYYY-MM-DD" + help="Date from which to trigger C-Move from in the format: YYYY-MM-DD", ) return parser.parse_args() if __name__ == "__main__": - args = parse_args() orthanc = Orthanc() @@ -89,8 +87,7 @@ def parse_args() -> argparse.Namespace: date = start_date + timedelta(days=i) query_id = orthanc.query_remote( - data={"Level": "Study", - "Query": {"StudyDate": date.strftime('%Y%m%d')}} + data={"Level": "Study", "Query": {"StudyDate": date.strftime("%Y%m%d")}} ) print(f"Driving C-Move for study {i} {query_id} on {date}") diff --git a/scripts/delete_oldest_n_studies.py b/scripts/delete_oldest_n_studies.py index 62d408885..f3a32020b 100644 --- a/scripts/delete_oldest_n_studies.py +++ b/scripts/delete_oldest_n_studies.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """Delete a number of studies from an Orthanc instance""" -import os -import requests import argparse - +from datetime import datetime from json import JSONDecodeError +import os from typing import Any, List, Optional -from datetime import datetime + +import requests from requests.auth import HTTPBasicAuth os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost" @@ -67,10 +67,7 @@ def query_local(self, data: dict) -> Any: def delete(self, study: Study) -> None: """Delete a study from Orthanc""" - response = requests.delete( - f"{self._url}/studies/{study.uid}", - auth=self._auth - ) + response = requests.delete(f"{self._url}/studies/{study.uid}", auth=self._auth) if response.status_code != 200: raise RuntimeError(f"Failed to delete: {study.uid}") @@ -100,7 +97,6 @@ def _deserialise(response: requests.Response) -> Any: def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() parser.add_argument( "number_to_delete", type=int, help="Number of the oldest studies to delete" @@ -112,7 +108,6 @@ def parse_args() -> argparse.Namespace: if __name__ == "__main__": - args = parse_args() print(f"Deleting the oldest {args.number_to_delete} studies") diff --git a/scripts/filter_cohort_for_those_present_in_raw.py b/scripts/filter_cohort_for_those_present_in_raw.py index d91631bdb..358c0b43b 100644 --- a/scripts/filter_cohort_for_those_present_in_raw.py +++ b/scripts/filter_cohort_for_those_present_in_raw.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """Filter a cohort .csv file for those that are not present in Orthanc raw""" +from json import JSONDecodeError import os -import requests import sys - -from json import JSONDecodeError from typing import Any, List + +import requests from requests.auth import HTTPBasicAuth os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost" @@ -84,7 +84,6 @@ def _deserialise(response: requests.Response) -> Any: if __name__ == "__main__": - filename = sys.argv[1] orthanc = Orthanc() @@ -94,7 +93,6 @@ def _deserialise(response: requests.Response) -> Any: with open(filename, "r") as file: with open(f"{filename.rstrip('.csv')}_filtered.csv", "w") as new_file: for line in file: - if any(a in line for a in present_accession_numbers): continue diff --git a/scripts/list_newest_n_studies.py b/scripts/list_newest_n_studies.py index 08efdae62..b211c845c 100644 --- a/scripts/list_newest_n_studies.py +++ b/scripts/list_newest_n_studies.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """List the newest studies""" -import os -import requests import argparse - +from datetime import datetime from json import JSONDecodeError +import os from typing import Any, List, Optional -from datetime import datetime + +import requests from requests.auth import HTTPBasicAuth os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost" @@ -94,7 +94,6 @@ def _deserialise(response: requests.Response) -> Any: def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() parser.add_argument( "number_to_list", type=int, help="Number of the newest studies to list" @@ -103,7 +102,6 @@ def parse_args() -> argparse.Namespace: if __name__ == "__main__": - args = parse_args() print(f"Listing the newest {args.number_to_list} studies") @@ -114,9 +112,12 @@ def parse_args() -> argparse.Namespace: for study in studies: study.received_time = orthanc.received_time(study) - for i, study in enumerate(sorted(studies, key=lambda x: x.received_time, reverse=True)): + sorted_studies = sorted(studies, key=lambda x: x.received_time, reverse=True) + for i, study in enumerate(sorted_studies): if i == args.number_to_list: break - print(f"Study received at {study.received_time}." - f"Accession number {orthanc.accession_number(study)}") + print( + f"Study received at {study.received_time}." + f"Accession number {orthanc.accession_number(study)}" + ) diff --git a/setup.cfg b/setup.cfg index 679afcf36..41c4a807f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,12 +28,5 @@ warn_unused_configs = True disallow_untyped_defs = True no_implicit_optional = True ignore_missing_imports = True - -[mypy-hasher.settings] -ignore_errors = True -[mypy-hasher.tests.*] -ignore_errors = True -[mypy-token_buffer.settings] -ignore_errors = True -[mypy-token_buffer.tests.*] -ignore_errors = True +exclude = orthanc|build|tests +packages = cli,pixl_core,pixl_ehr,pixl_pacs,hasher,pixl_dcmd,orthanc diff --git a/test/.env.test b/test/.env.test index 92ae63a25..faf865764 100644 --- a/test/.env.test +++ b/test/.env.test @@ -1,5 +1,7 @@ ENV=test DEBUG=True +PIXL_DICOM_TRANSFER_TIMEOUT=120 +PIXL_QUERY_TIMEOUT=120 # PIXL PostgreSQL instance PIXL_DB_HOST=postgres diff --git a/test/run-system-test.sh b/test/run-system-test.sh index a5158685c..375846c0b 100755 --- a/test/run-system-test.sh +++ b/test/run-system-test.sh @@ -18,18 +18,20 @@ PACKAGE_DIR="${BIN_DIR%/*}" cd "${PACKAGE_DIR}/test" # Note: this doesn't work as a single command -docker compose --env-file .env.test -p test up -d --build --remove-orphans +docker compose --env-file .env.test -p test up -d --remove-orphans cd .. && \ docker compose --env-file test/.env.test -p test up -d --build && \ - cd - + cd "${PACKAGE_DIR}/test" ./scripts/insert_test_data.sh -./scripts/install_pixl_cli.sh + +pip install "${PACKAGE_DIR}/pixl_core" "${PACKAGE_DIR}/cli" pixl populate data/test.csv pixl start sleep 65 # need to wait until the DICOM image is "stable" = 60s ./scripts/check_entry_in_pixl_anon.sh ./scripts/check_entry_in_orthanc_anon.sh +cd "${PACKAGE_DIR}" docker compose -f docker-compose.yml -f ../docker-compose.yml -p test down docker volume rm test_postgres-data test_orthanc-raw-data test_orthanc-anon-data diff --git a/test/scripts/check_entry_in_orthanc_anon.sh b/test/scripts/check_entry_in_orthanc_anon.sh index b221d11bc..ed5be78b9 100755 --- a/test/scripts/check_entry_in_orthanc_anon.sh +++ b/test/scripts/check_entry_in_orthanc_anon.sh @@ -14,13 +14,6 @@ # limitations under the License. set -eux pipefail -_result=$(curl -f -X POST \ - -u orthanc_anon_username:orthanc_anon_password \ - http://localhost:7003/tools/find \ - --data '{ - "Level" : "Instance", - "Query" : {} - }' - ) -# Check that result does not contain an empty list -echo "$_result" | grep --invert-match "\[\]" +# This could be much improved by having more realistic test data some of +# which actually was persisted +docker logs test-orthanc-anon-1 2>&1 | grep "DICOM instance received" diff --git a/test/scripts/insert_test_data.sh b/test/scripts/insert_test_data.sh index 02025b6a5..c28a0b706 100755 --- a/test/scripts/insert_test_data.sh +++ b/test/scripts/insert_test_data.sh @@ -20,7 +20,7 @@ _sql_command=" insert into star.mrn(mrn_id, mrn, research_opt_out) values (1234, 'patient_identifier', false); insert into star.core_demographic(mrn_id, sex) values (1234, 'F'); " -docker exec -it test-fake-star-db /bin/bash -c "psql -U postgres -c \"$_sql_command\"" || true +docker exec -it test-fake-star-db /bin/bash -c "psql -U postgres -d emap -c \"$_sql_command\"" || true # Uses an accession number of "123456789" curl -X POST -u orthanc:orthanc http://localhost:8043/instances \ diff --git a/test/scripts/install_pixl_cli.sh b/test/scripts/install_pixl_cli.sh deleted file mode 100755 index 29c5fbfb2..000000000 --- a/test/scripts/install_pixl_cli.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eux pipefail - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -REPO_ROOT_DIR="${SCRIPT_DIR}/../.." - -for dir in "token_buffer" "patient_queue" "cli" -do - src_path="${REPO_ROOT_DIR}/${dir}/src" - pip install -r "${src_path}/requirements.txt" "$src_path" -done diff --git a/token_buffer/README.md b/token_buffer/README.md deleted file mode 100644 index 914607884..000000000 --- a/token_buffer/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Token buffer - -The token buffer is needed to limit the download rate for images from PAX/VNA. Current specification suggests that a -rate limit of five images per second should be sufficient, however that may have to be altered dynamically through -command line interaction. - -The current implementation of the token buffer uses the -[token bucket implementation from Falconry](https://github.com/falconry/token-bucket/). Furthermore, the token buffer is -not set up as a service as it is only needed for the image download rate. \ No newline at end of file diff --git a/token_buffer/bin/run-tests.sh b/token_buffer/bin/run-tests.sh deleted file mode 100755 index 07cb15af8..000000000 --- a/token_buffer/bin/run-tests.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eo pipefail - -BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -PACKAGE_DIR="${BIN_DIR%/*}" -cd "$PACKAGE_DIR" - -CONF_FILE=../setup.cfg -mypy --config-file ${CONF_FILE} src/token_buffer -isort --settings-path ${CONF_FILE} src/token_buffer -black src/token_buffer -flake8 --config ${CONF_FILE} src/token_buffer - -ENV=test pytest src/token_buffer/tests diff --git a/token_buffer/src/requirements.txt b/token_buffer/src/requirements.txt deleted file mode 100644 index 570843307..000000000 --- a/token_buffer/src/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -black==22.8.0 -flake8==5.0.4 -flake8-return==1.1.3 -isort==5.10.1 -mypy==1.6.0 -pytest==7.4.2 -token-bucket==0.3.0 -python-decouple==3.6 diff --git a/token_buffer/src/setup.py b/token_buffer/src/setup.py deleted file mode 100644 index b4fb6ad0d..000000000 --- a/token_buffer/src/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import find_packages, setup - -exec(open("token_buffer/_version.py").read()) - -setup( - name="token_buffer", - version=__version__, # noqa: F821 - description="Service to create and manage a token bucket", - packages=find_packages( - include=[ - "token_buffer*", - ], - exclude=[ - "*tests", - "*.tests.*", - ], - ), - python_requires=">=3.10", -) diff --git a/token_buffer/src/token_buffer/_version.py b/token_buffer/src/token_buffer/_version.py deleted file mode 100644 index fddacd579..000000000 --- a/token_buffer/src/token_buffer/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2022 University College London Hospitals NHS Foundation Trust -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version_info__ = ("0", "0", "2") -__version__ = ".".join(__version_info__)