Packaging and DRYing (#148)

* hasher packaging * merge and package buffer and queue * move common api * cli packaging * merge report deid * fix test * fix system test * update workflow * move mypy and isort to precommit * move flake8 and black to pre-commit * remove old refs * test fixes * black update and path fixes * add missing dep * Apply suggestions from code review Co-authored-by: Stef Piatek <[email protected]> --------- Co-authored-by: Stef Piatek <[email protected]>
SAFEHR-data · Oct 24, 2023 · 909868c · 909868c
1 parent f808da2
commit 909868c
Show file tree

Hide file tree

Showing 123 changed files with 646 additions and 1,283 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -32,6 +32,11 @@ jobs:
       - name: Ensure copyright and license header are present
         run: ./.github/linters/check_headers_exist.sh
 
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files
+
       - name: Validate Docker Compose config file
         working-directory: .
         run: |
@@ -50,49 +55,19 @@ jobs:
           python-version: '3.10'
           cache: 'pip'
 
-      - name: Install Python dependencies
+      - name: Install package
         run: |
-          python -m pip install --upgrade pip
-          pip install -r hasher/src/requirements.txt
+          pip install hasher/[test]
 
       - name: Run tests
         working-directory: hasher
-        run: |
-          bin/run-tests.sh
+        run: pytest
         env:
           ENV: test
           AZURE_KEY_VAULT_NAME: test
           AZURE_KEY_VAULT_SECRET_NAME: test
 
-  report-de-id-tests:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Init Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10.6'  # Numpy croaks on 3.10.7+
-          cache: 'pip'
-
-      - name: Install Python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r pixl_rd/src/requirements.txt
-
-      - name: Download presidio data
-        run: |
-          python -m spacy download en_core_web_lg
-
-      - name: Run tests
-        working-directory: pixl_rd
-        run: |
-          bin/run-tests.sh
-        env:
-          ENV: test
-
-  token-buffer-tests:
+  core-tests:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
@@ -106,13 +81,13 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r token_buffer/src/requirements.txt
+           pip install pixl_core/[test]
 
       - name: Run tests
-        working-directory: token_buffer
+        working-directory: pixl_core/tests
         run: |
-          bin/run-tests.sh
+          docker compose up --build --exit-code-from test
+          docker compose down
         env:
           ENV: test
 
@@ -140,30 +115,6 @@ jobs:
         env:
           ENV: test
 
-  patient-queue-tests:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Init Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-          cache: 'pip'
-
-      - name: Install Python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r patient_queue/src/requirements.txt
-
-      - name: Run tests
-        working-directory: patient_queue/test
-        run: |
-          ./run-tests.sh
-        env:
-          ENV: test
-
   cli-tests:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
@@ -178,14 +129,10 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r cli/src/requirements.txt
-          pip install -r token_buffer/src/requirements.txt token_buffer/src/
-          pip install -r patient_queue/src/requirements.txt
-          pip install patient_queue/src/
+          pip install pixl_core/[test] cli/[test]
 
       - name: Run tests
-        working-directory: cli/test
+        working-directory: cli/tests
         run: |
           ./run-tests.sh
 
@@ -203,15 +150,11 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r pixl_ehr/src/requirements.txt
-          pip install -r pixl_rd/src/requirements.txt pixl_rd/src/
-          pip install -r token_buffer/src/requirements.txt token_buffer/src/
-          pip install -r patient_queue/src/requirements.txt patient_queue/src/
+          pip install pixl_core/[test] pixl_ehr/[test]
           python -m spacy download en_core_web_lg
 
       - name: Run tests
-        working-directory: pixl_ehr/test
+        working-directory: pixl_ehr/tests
         env:
           INFORMDB_PAT: ${{ secrets.INFORMDB_PAT }}
         run: |
@@ -231,12 +174,9 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r pixl_pacs/src/requirements.txt
-          pip install -r token_buffer/src/requirements.txt token_buffer/src/
-          pip install -r patient_queue/src/requirements.txt patient_queue/src/
+          pip install pixl_core/[test] pixl_pacs/[test]
 
       - name: Run tests
-        working-directory: pixl_pacs/test
+        working-directory: pixl_pacs/tests
         run: |
           ./run-tests.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,43 @@
+#  Copyright (c) University College London Hospitals NHS Foundation Trust
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: ["--profile", "black", "--settings-path", "setup.cfg"]
+
+  - repo: https://github.com/ambv/black
+    rev: 23.9.1
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    - id: flake8
+      args: ["--config", "setup.cfg"]
+      exclude: "tests|orthanc"
+
+  - repo: local
+    hooks:
+      - id: mypy  # This does not work with the official mypy hook
+        name: mypy
+        language: python
+        pass_filenames: false
+        entry: mypy
+        args: ['--config-file=setup.cfg']
+        additional_dependencies: ['mypy', 'types-PyYAML', 'types-requests']
diff --git a/cli/README.md b/cli/README.md
@@ -10,9 +10,15 @@ stopped cleanly.
 ## Installation
 
 ```bash
-cd src && pip install -r requirements.txt . 
+pip install -e ../pixl_core/ .
 ```
 
+## Test
+```bash
+./tests/run-tests.sh
+```
+
+
 ## Usage
 
 > **Note**

diff --git a/cli/pyproject.toml b/cli/pyproject.toml
@@ -0,0 +1,31 @@
+[project]
+name = "pixl_cli"
+version = "0.0.4"
+authors = [
+  { name="PIXL authors" },
+]
+description = "PIXL command line interface"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3"
+]
+dependencies = [
+    "core",
+    "click==8.1.3",
+    "coloredlogs==15.0.1",
+    "pandas==1.5.1",
+    "PyYAML==6.0"
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest==7.4.*"
+]
+
+[project.scripts]
+pixl = "pixl_cli.main:cli"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
diff --git a/cli/src/pixl_cli/_version.py b/cli/src/pixl_cli/_version.py
diff --git a/cli/src/pixl_cli/main.py b/cli/src/pixl_cli/main.py
@@ -20,14 +20,15 @@
 import pandas as pd
 
 import click
-from patient_queue.producer import PixlProducer
-from patient_queue.subscriber import PixlBlockingConsumer
-from patient_queue.utils import serialise, deserialise
-from pixl_cli._logging import logger, set_log_level
-from pixl_cli._utils import clear_file, remove_file_if_it_exists, string_is_non_empty
+from core.patient_queue.producer import PixlProducer
+from core.patient_queue.subscriber import PixlBlockingConsumer
+from core.patient_queue.utils import deserialise, serialise
 import requests
 import yaml
 
+from ._logging import logger, set_log_level
+from ._utils import clear_file, remove_file_if_it_exists, string_is_non_empty
+
 
 def _load_config(filename: str = "pixl_config.yml") -> dict:
     """CLI configuration generated from a .yaml file"""
@@ -77,7 +78,6 @@ def populate(csv_filename: str, queues: str, restart: bool) -> None:
 
     for queue in queues.split(","):
         with PixlProducer(queue_name=queue, **config["rabbitmq"]) as producer:
-
             state_filepath = state_filepath_for_queue(queue)
             if state_filepath.exists() and restart:
                 logger.info(f"Extracting messages from state: {state_filepath}")

diff --git a/cli/src/pixl_cli/tests/README.md b/cli/src/pixl_cli/tests/README.md
diff --git a/cli/src/pixl_cli/tests/__init__.py b/cli/src/pixl_cli/tests/__init__.py
diff --git a/cli/src/requirements.txt b/cli/src/requirements.txt
diff --git a/cli/src/setup.py b/cli/src/setup.py
diff --git a/cli/test/wait-until-service-healthy.sh b/cli/test/wait-until-service-healthy.sh
diff --git a/cli/test/README.md → cli/tests/README.md b/cli/test/README.md → cli/tests/README.md
diff --git a/cli/test/docker-compose.yml → cli/tests/docker-compose.yml b/cli/test/docker-compose.yml → cli/tests/docker-compose.yml
diff --git a/cli/test/pixl_config.yml → cli/tests/pixl_config.yml b/cli/test/pixl_config.yml → cli/tests/pixl_config.yml