Refactor/distributed-tests

.github/workflows/push-pr_workflow.yml

@@ -10,7 +10,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0 # Checkout the whole history, in case the target is way far behind
 
@@ -40,14 +40,14 @@ jobs:
       MAX_COMPLEXITY: 15
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
 
     - name: Check cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
@@ -95,14 +95,14 @@ jobs:
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Check cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
@@ -112,8 +112,7 @@ jobs:
         python3 -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
-        pip freeze
-
+
     - name: Install singularity
       run: |
         sudo apt-get update && sudo apt-get install -y \
@@ -145,48 +144,31 @@ jobs:
         merlin example feature_demo
         pip3 install -r feature_demo/requirements.txt
 
-    - name: Run pytest over unit test suite
-      run: |
-        python3 -m pytest -v --order-scope=module tests/unit/
-
     - name: Run integration test suite for local tests
       run: |
         python3 tests/integration/run_tests.py --verbose --local
 
-  Distributed-test-suite:
+  Unit-tests:
     runs-on: ubuntu-latest
-    services:
-      # rabbitmq:
-      #   image: rabbitmq:latest
-      #   ports:
-      #     - 5672:5672
-      #   options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379
+    env: 
+      GO_VERSION: 1.18.1
+      SINGULARITY_VERSION: 3.9.9
+      OS: linux
+      ARCH: amd64
 
     strategy:
       matrix:
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Check cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
@@ -197,33 +179,104 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
 
-    - name: Install merlin and setup redis as the broker
+    - name: Install singularity
+      run: |
+        sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          pkg-config
+        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
+        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+        rm go$GO_VERSION.$OS-$ARCH.tar.gz
+        export PATH=$PATH:/usr/local/go/bin
+        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
+        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
+        cd singularity-ce-$SINGULARITY_VERSION
+        ./mconfig && \
+          make -C ./builddir && \
+          sudo make -C ./builddir install
+
+    - name: Install merlin to run unit tests
       run: |
         pip3 install -e .
-        merlin config --broker redis
+        merlin config
 
     - name: Install CLI task dependencies generated from the 'feature demo' workflow
       run: |
         merlin example feature_demo
         pip3 install -r feature_demo/requirements.txt
 
+    - name: Run pytest over unit test suite
+      run: |
+        python3 -m pytest -v --order-scope=module tests/unit/
+
+  Integration-tests:
+    runs-on: ubuntu-latest
+    env: 
+      GO_VERSION: 1.18.1
+      SINGULARITY_VERSION: 3.9.9
+      OS: linux
+      ARCH: amd64
+
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Check cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip3 install -r requirements/dev.txt
+
+    - name: Install merlin
+      run: |
+        pip3 install -e .
+        merlin config
+
+    - name: Install singularity
+      run: |
+        sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          pkg-config
+        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
+        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+        rm go$GO_VERSION.$OS-$ARCH.tar.gz
+        export PATH=$PATH:/usr/local/go/bin
+        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
+        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
+        cd singularity-ce-$SINGULARITY_VERSION
+        ./mconfig && \
+          make -C ./builddir && \
+          sudo make -C ./builddir install
+
+    - name: Install CLI task dependencies generated from the 'feature demo' workflow
+      run: |
+        merlin example feature_demo
+        pip3 install -r feature_demo/requirements.txt
+
+    # TODO remove the --ignore statement once those tests are fixed
     - name: Run integration test suite for distributed tests
-      env:
-        REDIS_HOST: redis
-        REDIS_PORT: 6379
-      run: |
-        python3 tests/integration/run_tests.py --verbose --distributed
-
-    # - name: Setup rabbitmq config
-    #   run: |
-    #     merlin config --test rabbitmq
-
-    # - name: Run integration test suite for rabbitmq
-    #   env:
-    #     AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }}
-    #     RABBITMQ_USER: Jimmy_Space
-    #     RABBITMQ_PASS: Alexander_Rules
-    #   ports:
-    #     - ${{ job.services.rabbitmq.ports['5672'] }}
-    #   run: |
-    #     python3 tests/integration/run_tests.py --verbose --ids 31 32
+      run: |
+        python3 -m pytest -v --ignore tests/integration/test_celeryadapter.py tests/integration/

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to Merlin will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Added additional tests for the `merlin run` and `merlin purge` commands
+- Aliased types to represent different types of pytest fixtures
+- New test condition `StepFinishedFilesCount` to help search for `MERLIN_FINISHED` files in output workspaces
+- Added "Unit-tests" GitHub action to run the unit test suite
+- Added `CeleryTaskManager` context manager to the test suite to ensure tasks are safely purged from queues if tests fail
+
+### Changed
+- Ported all distributed tests of the integration test suite to pytest
+  - There is now a `commands/` directory and a `workflows/` directory under the integration suite to house these tests
+  - Removed the "Distributed-tests" GitHub action as these tests will now be run under "Integration-tests"
+
 ## [1.12.2]
 ### Added
 - Conflict handler option to the `dict_deep_merge` function in `utils.py`

merlin/celery.py

@@ -114,9 +114,11 @@ def route_for_task(name, args, kwargs, options, task=None, **kw):  # pylint: dis
     BROKER_URI = None
     RESULTS_BACKEND_URI = None
 
+app_name = "merlin_test_app" if os.getenv("CELERY_ENV") == "test" else "merlin"
+
 # initialize app with essential properties
 app: Celery = patch_celery().Celery(
-    "merlin",
+    app_name,
     broker=BROKER_URI,
     backend=RESULTS_BACKEND_URI,
     broker_use_ssl=BROKER_SSL,

merlin/examples/dev_workflows/multiple_workers.yaml

@@ -46,11 +46,11 @@ merlin:
     resources:
         workers:
             step_1_merlin_test_worker:
-                args: -l INFO
+                args: -l INFO --concurrency 1
                 steps: [step_1]
             step_2_merlin_test_worker:
-                args: -l INFO
+                args: -l INFO --concurrency 1
                 steps: [step_2]
             other_merlin_test_worker:
-                args: -l INFO
+                args: -l INFO --concurrency 1
                 steps: [step_3, step_4]

tests/README.md

@@ -4,16 +4,20 @@ This directory utilizes pytest to create and run our test suite.
 
 This directory is organized like so:
 - `conftest.py` - The script containing common fixtures for our tests
+- `constants.py` - Constant values to be used throughout the test suite.
+- `fixture_data_classes.py` - Dataclasses to help group pytest fixtures together, reducing the required number of imports.
+- `fixture_types.py` - Aliases for type hinting fixtures.
 - `context_managers/` - The directory containing context managers used for testing
     - `celery_workers_manager.py` - A context manager used to manage celery workers for integration testing
     - `server_manager.py` - A context manager used to manage the redis server used for integration testing
 - `fixtures/` - The directory containing specific test module fixtures
     - `<test_module_name>.py` - Fixtures for specific test modules
 - `integration/` - The directory containing integration tests
-    <!-- - `test_*.py` - The actual test scripts to run -->
     - `definitions.py` - The test definitions
     - `run_tests.py` - The script to run the tests defined in `definitions.py`
     - `conditions.py` - The conditions to test against
+    - `commands/` - The directory containing tests for commands of the Merlin library.
+    - `workflows/` The directory containing tests for entire workflow runs.
 - `unit/` - The directory containing unit tests
     - `test_*.py` - The actual test scripts to run