From 107751850d1433caf953bed9c6219d6f8b8250f3 Mon Sep 17 00:00:00 2001 From: Hugo MacDermott-Opeskin Date: Sun, 11 Feb 2024 19:28:15 +1100 Subject: [PATCH] Add initial testing (#26) * try get initial tests working * add in env file * fix names and rename file * fix * fix again * move to ci file * Revert "move to ci file" This reverts commit 73a9418e26c5605f218ea5dca4839c6da3130cea. * collapse to one file * fix indents * try some stuff * add ci to correct path * fix TTY issues * try waiting * try with healthcheck instead * try making sure service is healthy * add initial testing * add updates * make model tests work * make tests even better * improve again * get tests partially working * try * inject OE license * fix settings stuff * HOLY GUACAMOLE it was the OE version * fix wf * gix path? * Update ci.yaml * Update ci.yaml * Update test_views.py * Update test_views.py * Update test_views.py --- .github/workflows/ci.yaml | 66 +++++++++++++++++++ Dockerfile | 17 ++++- argos/argos/settings.py | 17 +++++ argos/argos_viewer/models.py | 16 ++++- .../argos_viewer/target_pdb_detail_view.html | 1 - argos/argos_viewer/tests.py | 3 - argos/argos_viewer/tests/__init__.py | 0 argos/argos_viewer/tests/test_models.py | 43 ++++++++++++ argos/argos_viewer/tests/test_oe_license.py | 6 ++ argos/argos_viewer/tests/test_urls.py | 28 ++++++++ argos/argos_viewer/tests/test_views.py | 55 ++++++++++++++++ argos/argos_viewer/views.py | 13 ++-- devtools/conda-envs/argos-ubuntu-latest.yml | 15 +++-- devtools/deployment/.env.example | 2 +- docker-compose-dev.yml | 17 +++-- docker-compose-prod.yml | 10 ++- 16 files changed, 287 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 argos/argos_viewer/templates/argos_viewer/target_pdb_detail_view.html delete mode 100644 argos/argos_viewer/tests.py create mode 100644 argos/argos_viewer/tests/__init__.py create mode 100644 argos/argos_viewer/tests/test_models.py create mode 100644 argos/argos_viewer/tests/test_oe_license.py create mode 100644 argos/argos_viewer/tests/test_urls.py create mode 100644 argos/argos_viewer/tests/test_views.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..aacea74 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,66 @@ +name: ci + +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + schedule: + # Nightly tests run on main by default: + # Scheduled workflows run on the latest commit on the default or base branch. + # (from https://help.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule) + - cron: "0 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +defaults: + run: + shell: bash -l {0} + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install docker-compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + + - name: Copy example env file + run: | + cp devtools/deployment/.env.example .env + + - name: Make OpenEye directory + run: | + mkdir -p ~/.OpenEye + + - name: Copy OpenEye license file + env: + OE_LICENSE: ${{ secrets.OE_LICENSE }} + run: | + echo "$OE_LICENSE" > ~/.OpenEye/oe_license.txt + + - name: Build containers + run: | + docker-compose -f docker-compose-dev.yml build --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) + docker container ls + docker-compose -f docker-compose-dev.yml up -d + + - name: Chown volume + run: | + ls -alsh + ls -alsh argos + sudo chown -R $(id -u):$(id -g) argos/pdb_data + + - name: Run tests + run: | + docker-compose -f docker-compose-dev.yml exec -T web bash -c "cd argos && /opt/conda/bin/python manage.py test --no-input" diff --git a/Dockerfile b/Dockerfile index 2737df3..ca28ae2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,17 @@ # Use the conda-forge base image with Python FROM mambaorg/micromamba:jammy + +ARG USER_ID +ARG GROUP_ID + +USER root + +RUN addgroup --gid $GROUP_ID user +RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user +USER user + + # set environment variables ENV PYTHONUNBUFFERED 1 @@ -13,15 +24,17 @@ COPY . /argos RUN micromamba config append channels conda-forge RUN micromamba config append channels openeye -COPY --chown=$MAMBA_USER:$MAMBA_USER devtools/conda-envs/argos-ubuntu-latest.yml /tmp/env.yaml +COPY --chown=user:user devtools/conda-envs/argos-ubuntu-latest.yml /tmp/env.yaml RUN micromamba install -y -n base git -f /tmp/env.yaml && \ micromamba clean --all --yes USER root RUN mkdir /openeye +RUN chown -R user:user /openeye RUN mkdir /argos/pdb_data -USER $MAMBA_USER +RUN chown -R user:user /argos/pdb_data +USER user ENV OE_LICENSE=/openeye/oe_license.txt diff --git a/argos/argos/settings.py b/argos/argos/settings.py index fd7e91a..6f95457 100644 --- a/argos/argos/settings.py +++ b/argos/argos/settings.py @@ -48,6 +48,7 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django_cleanup.apps.CleanupConfig', 'argos_viewer.apps.ArgosViewerConfig', ] @@ -89,6 +90,7 @@ DATABASES = { 'default': env.db(), + } @@ -146,3 +148,18 @@ # Redirect to home URL after login (Default redirects to /accounts/profile/) LOGIN_REDIRECT_URL = '/' + + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": {"class": "logging.StreamHandler"}, + }, + "loggers": { + "django": { + "handlers": ["console"], + "level": "INFO", + }, + } +} diff --git a/argos/argos_viewer/models.py b/argos/argos_viewer/models.py index 8b094f8..9a8966c 100644 --- a/argos/argos_viewer/models.py +++ b/argos/argos_viewer/models.py @@ -1,12 +1,26 @@ +from typing import Any from django.db import models from django.core.validators import FileExtensionValidator - +import os class PDBFile(models.Model): file = models.FileField(validators=[FileExtensionValidator(['pdb'])], upload_to="pdb_data") + def delete(self, *args, **kwargs): + # Delete the file when the instance is deleted + if self.file: + if os.path.isfile(self.file.path): + os.remove(self.file.path) + super(PDBFile, self).delete(*args, **kwargs) class TargetPDBFile(models.Model): pdb_file = models.ForeignKey(PDBFile, on_delete=models.CASCADE) target = models.CharField(max_length=200) upload_date = models.DateTimeField(auto_now_add=True) + + + def delete(self, *args, **kwargs): + # Delete the associated PDBFile instance, which should trigger its delete method + if self.pdb_file: + self.pdb_file.delete() + super(TargetPDBFile, self).delete(*args, **kwargs) \ No newline at end of file diff --git a/argos/argos_viewer/templates/argos_viewer/target_pdb_detail_view.html b/argos/argos_viewer/templates/argos_viewer/target_pdb_detail_view.html deleted file mode 100644 index f3ca97f..0000000 --- a/argos/argos_viewer/templates/argos_viewer/target_pdb_detail_view.html +++ /dev/null @@ -1 +0,0 @@ -{{html_content}} \ No newline at end of file diff --git a/argos/argos_viewer/tests.py b/argos/argos_viewer/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/argos/argos_viewer/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/argos/argos_viewer/tests/__init__.py b/argos/argos_viewer/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/argos/argos_viewer/tests/test_models.py b/argos/argos_viewer/tests/test_models.py new file mode 100644 index 0000000..b914619 --- /dev/null +++ b/argos/argos_viewer/tests/test_models.py @@ -0,0 +1,43 @@ +from django.test import TestCase +from argos_viewer.models import PDBFile, TargetPDBFile +from django.core.files.uploadedfile import SimpleUploadedFile +from django.utils import timezone +import os + +class PDBFileModelTest(TestCase): + def setUp(self): + self.file = SimpleUploadedFile("test.pdb", b"pdb file content") + self.pdb_file = PDBFile.objects.create(file=self.file) + + + def test_pdb_file_created(self): + self.assertTrue(os.path.exists(self.pdb_file.file.path)) + + def test_file_extension(self): + self.assertEqual(self.pdb_file.file.name.split(".")[-1], "pdb") + +class TargetPDBFileModelTest(TestCase): + def setUp(self): + self.file = SimpleUploadedFile("test.pdb", b"pdb file content") + self.pdb_file = PDBFile.objects.create(file=self.file) + self.target_pdb_file = TargetPDBFile.objects.create(pdb_file=self.pdb_file, target="test_target", upload_date=timezone.now()) + + def tearDown(self): + if os.path.exists(self.target_pdb_file.pdb_file.file.path): + os.remove(self.target_pdb_file.pdb_file.file.path) + + def test_target_pdb_file_created(self): + self.assertTrue(TargetPDBFile.objects.exists()) + + def test_target(self): + self.assertEqual(self.target_pdb_file.target, "test_target") + + def test_upload_date(self): + self.assertIsNotNone(self.target_pdb_file.upload_date) + + def test_pdb_file_deletion_on_target_pdb_file_deletion(self): + self.target_pdb_file.delete() + self.assertFalse(os.path.exists(self.pdb_file.file.path)) + + + diff --git a/argos/argos_viewer/tests/test_oe_license.py b/argos/argos_viewer/tests/test_oe_license.py new file mode 100644 index 0000000..17662ee --- /dev/null +++ b/argos/argos_viewer/tests/test_oe_license.py @@ -0,0 +1,6 @@ +from django.test import TestCase + +class OELicenseTests(TestCase): + def test_oechem_is_licensed(self): + from asapdiscovery.data.openeye import oechem + self.assertTrue(oechem.OEChemIsLicensed()) \ No newline at end of file diff --git a/argos/argos_viewer/tests/test_urls.py b/argos/argos_viewer/tests/test_urls.py new file mode 100644 index 0000000..52f6e2c --- /dev/null +++ b/argos/argos_viewer/tests/test_urls.py @@ -0,0 +1,28 @@ +from django.test import SimpleTestCase +from django.urls import reverse, resolve +from argos_viewer.views import home, upload_sucessful, TargetPDBListView, target_pdb_detail_view, failed, no_fitness_data + +class TestUrls(SimpleTestCase): + def test_home_url_resolves(self): + url = reverse('home') + self.assertEqual(resolve(url).func, home) + + def test_upload_sucessful_url_resolves(self): + url = reverse('upload_sucessful') + self.assertEqual(resolve(url).func, upload_sucessful) + + def test_pdb_files_url_resolves(self): + url = reverse('pdb_files') + self.assertEqual(resolve(url).func.view_class, TargetPDBListView) + + def test_detail_url_resolves(self): + url = reverse('detail', args=[1]) # assuming pk=1 + self.assertEqual(resolve(url).func, target_pdb_detail_view) + + def test_failed_url_resolves(self): + url = reverse('failed') + self.assertEqual(resolve(url).func, failed) + + def test_no_fitness_data_url_resolves(self): + url = reverse('no_fitness_data', args=['target_name']) + self.assertEqual(resolve(url).func, no_fitness_data) diff --git a/argos/argos_viewer/tests/test_views.py b/argos/argos_viewer/tests/test_views.py new file mode 100644 index 0000000..e103e96 --- /dev/null +++ b/argos/argos_viewer/tests/test_views.py @@ -0,0 +1,55 @@ +from django.test import TestCase, Client, RequestFactory +from django.urls import reverse +from argos_viewer.models import PDBFile, TargetPDBFile +from django.core.files.uploadedfile import SimpleUploadedFile +from django.utils import timezone +from django.contrib.auth.models import User +from asapdiscovery.data.testing.test_resources import fetch_test_file + + +class ViewTests(TestCase): + def setUp(self): + self._test_file = fetch_test_file("Mpro-P2660_0A_bound.pdb") + with open(self._test_file, "r") as f: + self._file_contents = f.read() + # Create a user + self.user = User.objects.create_user(username='testuser', password='12345') + self.file = SimpleUploadedFile("test.pdb", self._file_contents.encode()) + self.pdb_file = PDBFile.objects.create(file=self.file) + self.target_pdb_file = TargetPDBFile.objects.create(pdb_file=self.pdb_file, target="SARS-CoV-2-Mpro", upload_date=timezone.now()) + + def test_index_view(self): + response = self.client.get(reverse('index')) + self.assertEqual(response.status_code, 302) # Redirects to home + + def test_home_view_GET(self): + self.client.force_login(self.user) + response = self.client.get(reverse('home')) + self.assertEqual(response.status_code, 200) + self.assertTemplateUsed(response, 'argos_viewer/home.html') + + def test_upload_successful_view(self): + self.client.force_login(self.user) + response = self.client.get(reverse('upload_sucessful')) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'upload worked!') + + + def test_target_pdb_detail_view_GET(self): + self.client.force_login(self.user) + response = self.client.get(reverse('detail', args=[self.target_pdb_file.pk]), follow=True) + self.assertEqual(response.status_code, 200) + + def test_failed_view_GET(self): + self.client.force_login(self.user) + response = self.client.get(reverse('failed')) + self.assertEqual(response.status_code, 200) + self.assertTemplateUsed(response, 'argos_viewer/failed.html') + + def test_no_fitness_data_view_GET(self): + self.client.force_login(self.user) + response = self.client.get(reverse('no_fitness_data', args=['target_name'])) + self.assertEqual(response.status_code, 200) + self.assertTemplateUsed(response, 'argos_viewer/no_fitness_data.html') + + # Add tests for POST requests as well if necessary diff --git a/argos/argos_viewer/views.py b/argos/argos_viewer/views.py index e1d67ba..60454c3 100644 --- a/argos/argos_viewer/views.py +++ b/argos/argos_viewer/views.py @@ -15,6 +15,10 @@ from asapdiscovery.data.fitness import target_has_fitness_data import tempfile +import logging + +logger = logging.getLogger('django') + def index(request): context = {} return redirect("home") @@ -74,14 +78,13 @@ def target_pdb_detail_view(request, pk): target_kwargs={"target_name": "unknown"}, ) - tf = tempfile.NamedTemporaryFile() + tf = tempfile.NamedTemporaryFile() html_viz = HTMLVisualizer( - [c.ligand.to_oemol()], [tf], obj.target, c.target.to_oemol(), color_method="fitness", align=False - ) - # align=True is broken, see https://github.com/choderalab/asapdiscovery/issues/709 + [c.ligand.to_oemol()], [tf], obj.target, c.target.to_oemol(), color_method="fitness", align=True) html = html_viz.make_poses_html()[0] + logger.debug("Made pose html") except Exception as e: - print(f"rendering failed with exception {e}") + logger.error(f"rendering failed with exception {e}") return redirect("failed") return HttpResponse(html) diff --git a/devtools/conda-envs/argos-ubuntu-latest.yml b/devtools/conda-envs/argos-ubuntu-latest.yml index 4ac5d49..6e627ad 100644 --- a/devtools/conda-envs/argos-ubuntu-latest.yml +++ b/devtools/conda-envs/argos-ubuntu-latest.yml @@ -6,20 +6,27 @@ dependencies: # Base depends - pip - - python =3.10 - - git + - python >=3.10,<3.11 + # argos deps - django - gunicorn + - django-cleanup + + + # Testing + - pytest + - pytest-cov + - pooch + # asapdiscovery deps # Others - appdirs - - openeye-toolkits + - openeye-toolkits <2023.2.3 - pydantic >=1.10.8,<2.0.0a0 - # data - requests - boto3 diff --git a/devtools/deployment/.env.example b/devtools/deployment/.env.example index 9b13331..f0222d3 100644 --- a/devtools/deployment/.env.example +++ b/devtools/deployment/.env.example @@ -5,4 +5,4 @@ DATABASE_URL=postgresql://django_traefik:django_traefik@db:5432/django_traefik HOST_DOMAIN=localhost POSTGRES_USER=django_traefik POSTGRES_PASSWORD=django_traefik -POSTGRES_DB=django_traefik +POSTGRES_DB=django_traefik \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3926887..0e1f80d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -16,21 +16,30 @@ services: env_file: - .env depends_on: - - db + db: + condition: service_healthy + labels: - "traefik.enable=true" - "traefik.http.routers.django.rule=Host(`django.localhost`)" db: image: postgres:15-alpine + + healthcheck: + test: pg_isready -U ${POSTGRES_USER:?err} -d ${POSTGRES_DB:?err} + interval: 5s + timeout: 10s + retries: 120 + volumes: - postgres_data:/var/lib/postgresql/data/ expose: - 5432 environment: - - POSTGRES_USER=django_traefik - - POSTGRES_PASSWORD=django_traefik - - POSTGRES_DB=django_traefik + - POSTGRES_USER=${POSTGRES_USER:?err} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?err} + - POSTGRES_DB=${POSTGRES_DB:?err} traefik: image: traefik:latest diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 9151d41..0c0a751 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -17,7 +17,8 @@ services: env_file: - .env depends_on: - - db + db: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.django.rule=Host(`${HOST_DOMAIN:?err}`)" @@ -27,6 +28,13 @@ services: db: image: postgres:15-alpine + + healthcheck: + test: pg_isready -U ${POSTGRES_USER:?err} -d ${POSTGRES_DB:?err} + interval: 5s + timeout: 10s + retries: 120 + volumes: - postgres_data_prod:/var/lib/postgresql/data/ expose: