diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 1d3c9d958..abc2b2ac7 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 # Checkout the whole history, in case the target is way far behind @@ -40,14 +40,14 @@ jobs: MAX_COMPLEXITY: 15 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.x' - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} @@ -95,14 +95,14 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} @@ -112,8 +112,7 @@ jobs: python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - pip freeze - + - name: Install singularity run: | sudo apt-get update && sudo apt-get install -y \ @@ -145,48 +144,31 @@ jobs: merlin example feature_demo pip3 install -r feature_demo/requirements.txt - - name: Run pytest over unit test suite - run: | - python3 -m pytest -v --order-scope=module tests/unit/ - - name: Run integration test suite for local tests run: | python3 tests/integration/run_tests.py --verbose --local - Distributed-test-suite: + Unit-tests: runs-on: ubuntu-latest - services: - # rabbitmq: - # image: rabbitmq:latest - # ports: - # - 5672:5672 - # options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5 - # Label used to access the service container - redis: - # Docker Hub image - image: redis - # Set health checks to wait until redis has started - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 strategy: matrix: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} @@ -197,33 +179,104 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Install merlin and setup redis as the broker + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install + + - name: Install merlin to run unit tests run: | pip3 install -e . - merlin config --broker redis + merlin config - name: Install CLI task dependencies generated from the 'feature demo' workflow run: | merlin example feature_demo pip3 install -r feature_demo/requirements.txt + - name: Run pytest over unit test suite + run: | + python3 -m pytest -v --order-scope=module tests/unit/ + + Integration-tests: + runs-on: ubuntu-latest + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 + + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Check cache + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip3 install -r requirements/dev.txt + + - name: Install merlin + run: | + pip3 install -e . + merlin config + + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install + + - name: Install CLI task dependencies generated from the 'feature demo' workflow + run: | + merlin example feature_demo + pip3 install -r feature_demo/requirements.txt + + # TODO remove the --ignore statement once those tests are fixed - name: Run integration test suite for distributed tests - env: - REDIS_HOST: redis - REDIS_PORT: 6379 - run: | - python3 tests/integration/run_tests.py --verbose --distributed - - # - name: Setup rabbitmq config - # run: | - # merlin config --test rabbitmq - - # - name: Run integration test suite for rabbitmq - # env: - # AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }} - # RABBITMQ_USER: Jimmy_Space - # RABBITMQ_PASS: Alexander_Rules - # ports: - # - ${{ job.services.rabbitmq.ports['5672'] }} - # run: | - # python3 tests/integration/run_tests.py --verbose --ids 31 32 + run: | + python3 -m pytest -v --ignore tests/integration/test_celeryadapter.py tests/integration/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd534b1b..d2ef4eeaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- Added additional tests for the `merlin run` and `merlin purge` commands +- Aliased types to represent different types of pytest fixtures +- New test condition `StepFinishedFilesCount` to help search for `MERLIN_FINISHED` files in output workspaces +- Added "Unit-tests" GitHub action to run the unit test suite +- Added `CeleryTaskManager` context manager to the test suite to ensure tasks are safely purged from queues if tests fail +- Added `command-tests`, `workflow-tests`, and `integration-tests` to the Makefile + +### Changed +- Ported all distributed tests of the integration test suite to pytest + - There is now a `commands/` directory and a `workflows/` directory under the integration suite to house these tests + - Removed the "Distributed-tests" GitHub action as these tests will now be run under "Integration-tests" +- Removed `e2e-distributed*` definitions from the Makefile + ## [1.12.2] ### Added - Conflict handler option to the `dict_deep_merge` function in `utils.py` diff --git a/Makefile b/Makefile index a261b1d99..9e1592925 100644 --- a/Makefile +++ b/Makefile @@ -34,12 +34,13 @@ include config.mk .PHONY : install-workflow-deps .PHONY : install-dev .PHONY : unit-tests +.PHONY : command-tests +.PHONY : workflow-tests +.PHONY : integration-tests .PHONY : e2e-tests .PHONY : e2e-tests-diagnostic .PHONY : e2e-tests-local .PHONY : e2e-tests-local-diagnostic -.PHONY : e2e-tests-distributed -.PHONY : e2e-tests-distributed-diagnostic .PHONY : tests .PHONY : check-flake8 .PHONY : check-black @@ -89,6 +90,18 @@ unit-tests: . $(VENV)/bin/activate; \ $(PYTHON) -m pytest -v --order-scope=module $(UNIT); \ +command-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) -m pytest -v $(TEST)/integration/commands/; \ + + +workflow-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) -m pytest -v $(TEST)/integration/workflows/; \ + + +integration-tests: command-tests workflow-tests + # run CLI tests - these require an active install of merlin in a venv e2e-tests: @@ -111,18 +124,8 @@ e2e-tests-local-diagnostic: $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose -e2e-tests-distributed: - . $(VENV)/bin/activate; \ - $(PYTHON) $(TEST)/integration/run_tests.py --distributed; \ - - -e2e-tests-distributed-diagnostic: - . $(VENV)/bin/activate; \ - $(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose - - # run unit and CLI tests -tests: unit-tests e2e-tests +tests: unit-tests integration-tests e2e-tests check-flake8: diff --git a/merlin/celery.py b/merlin/celery.py index 81a17c5b2..7db1891ed 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -114,9 +114,11 @@ def route_for_task(name, args, kwargs, options, task=None, **kw): # pylint: dis BROKER_URI = None RESULTS_BACKEND_URI = None +app_name = "merlin_test_app" if os.getenv("CELERY_ENV") == "test" else "merlin" + # initialize app with essential properties app: Celery = patch_celery().Celery( - "merlin", + app_name, broker=BROKER_URI, backend=RESULTS_BACKEND_URI, broker_use_ssl=BROKER_SSL, diff --git a/merlin/examples/dev_workflows/multiple_workers.yaml b/merlin/examples/dev_workflows/multiple_workers.yaml index 8785d9e9a..967582a53 100644 --- a/merlin/examples/dev_workflows/multiple_workers.yaml +++ b/merlin/examples/dev_workflows/multiple_workers.yaml @@ -46,11 +46,11 @@ merlin: resources: workers: step_1_merlin_test_worker: - args: -l INFO + args: -l INFO --concurrency 1 steps: [step_1] step_2_merlin_test_worker: - args: -l INFO + args: -l INFO --concurrency 1 steps: [step_2] other_merlin_test_worker: - args: -l INFO + args: -l INFO --concurrency 1 steps: [step_3, step_4] diff --git a/tests/README.md b/tests/README.md index 9b2f7ba1f..e2fa22cbf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,16 +4,20 @@ This directory utilizes pytest to create and run our test suite. This directory is organized like so: - `conftest.py` - The script containing common fixtures for our tests +- `constants.py` - Constant values to be used throughout the test suite. +- `fixture_data_classes.py` - Dataclasses to help group pytest fixtures together, reducing the required number of imports. +- `fixture_types.py` - Aliases for type hinting fixtures. - `context_managers/` - The directory containing context managers used for testing - `celery_workers_manager.py` - A context manager used to manage celery workers for integration testing - `server_manager.py` - A context manager used to manage the redis server used for integration testing - `fixtures/` - The directory containing specific test module fixtures - `.py` - Fixtures for specific test modules - `integration/` - The directory containing integration tests - - `definitions.py` - The test definitions - `run_tests.py` - The script to run the tests defined in `definitions.py` - `conditions.py` - The conditions to test against + - `commands/` - The directory containing tests for commands of the Merlin library. + - `workflows/` The directory containing tests for entire workflow runs. - `unit/` - The directory containing unit tests - `test_*.py` - The actual test scripts to run diff --git a/tests/conftest.py b/tests/conftest.py index eb68542bf..46fad196b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,18 +35,28 @@ from copy import copy from glob import glob from time import sleep -from typing import Dict import pytest import yaml from _pytest.tmpdir import TempPathFactory from celery import Celery -from celery.canvas import Signature +from redis import Redis from merlin.config.configfile import CONFIG from tests.constants import CERT_FILES, SERVER_PASS from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import ( + FixtureBytes, + FixtureCallable, + FixtureCelery, + FixtureDict, + FixtureModification, + FixtureRedis, + FixtureSignature, + FixtureStr, +) from tests.utils import create_cert_files, create_pass_file @@ -57,9 +67,11 @@ # Loading in Module Specific Fixtures # ####################################### - +fixture_glob = os.path.join("tests", "fixtures", "**", "*.py") pytest_plugins = [ - fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) + fixture_file.replace(os.sep, ".").replace(".py", "") + for fixture_file in glob(fixture_glob, recursive=True) + if not fixture_file.endswith("__init__.py") ] @@ -94,13 +106,98 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi yaml.dump(app_yaml, app_yaml_file) +def setup_redis_config(config_type: str, merlin_server_dir: str): + """ + Sets up the Redis configuration for either broker or results backend. + + Args: + config_type: The type of configuration to set up ('broker' or 'results_backend'). + merlin_server_dir: The directory to the merlin test server configuration. + """ + port = 6379 + name = "redis" + pass_file = os.path.join(merlin_server_dir, "redis.pass") + create_pass_file(pass_file) + + if config_type == "broker": + CONFIG.broker.password = pass_file + CONFIG.broker.port = port + CONFIG.broker.name = name + elif config_type == "results_backend": + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.port = port + CONFIG.results_backend.name = name + else: + raise ValueError("Invalid config_type. Must be 'broker' or 'results_backend'.") + + ####################################### ######### Fixture Definitions ######### ####################################### @pytest.fixture(scope="session") -def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: +def path_to_test_specs() -> FixtureStr: + """ + Fixture to provide the path to the directory containing test specifications. + + This fixture returns the absolute path to the 'test_specs' directory + within the 'integration' folder of the test directory. It expands + environment variables and user home directory as necessary. + + Returns: + The absolute path to the 'test_specs' directory. + """ + path_to_test_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))) + return os.path.join(path_to_test_dir, "integration", "test_specs") + + +@pytest.fixture(scope="session") +def path_to_merlin_codebase() -> FixtureStr: + """ + Fixture to provide the path to the directory containing the Merlin code. + + This fixture returns the absolute path to the 'merlin' directory at the + top level of this repository. It expands environment variables and user + home directory as necessary. + + Returns: + The absolute path to the 'merlin' directory. + """ + path_to_test_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))) + return os.path.join(path_to_test_dir, "..", "merlin") + + +@pytest.fixture(scope="session") +def create_testing_dir() -> FixtureCallable: + """ + Fixture to create a temporary testing directory. + + Returns: + A function that creates the testing directory. + """ + + def _create_testing_dir(base_dir: str, sub_dir: str) -> str: + """ + Helper function to create a temporary testing directory. + + Args: + base_dir: The base directory where the testing directory will be created. + sub_dir: The name of the subdirectory to create. + + Returns: + The path to the created testing directory. + """ + testing_dir = os.path.join(base_dir, sub_dir) + if not os.path.exists(testing_dir): + os.makedirs(testing_dir) # Use makedirs to create intermediate directories if needed + return testing_dir + + return _create_testing_dir + + +@pytest.fixture(scope="session") +def temp_output_dir(tmp_path_factory: TempPathFactory) -> FixtureStr: """ This fixture will create a temporary directory to store output files of integration tests. The temporary directory will be stored at /tmp/`whoami`/pytest-of-`whoami`/. There can be at most @@ -123,21 +220,21 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str) -> str: +def merlin_server_dir(temp_output_dir: FixtureStr) -> FixtureStr: """ The path to the merlin_server directory that will be created by the `redis_server` fixture. :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture """ - server_dir = f"{temp_output_dir}/merlin_server" + server_dir = os.path.join(temp_output_dir, "merlin_server") if not os.path.exists(server_dir): os.mkdir(server_dir) return server_dir @pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: +def redis_server(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureStr: """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. @@ -146,11 +243,14 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ + os.environ["CELERY_ENV"] = "test" with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() create_encryption_file( - f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml" + os.path.join(merlin_server_dir, "encrypt_data_key"), + test_encryption_key, + app_yaml_filepath=os.path.join(merlin_server_dir, "app.yaml"), ) # Yield the redis_server uri to any fixtures/tests that may need it yield redis_server_manager.redis_server_uri @@ -158,18 +258,36 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: @pytest.fixture(scope="session") -def celery_app(redis_server: str) -> Celery: +def redis_client(redis_server: FixtureStr) -> FixtureRedis: + """ + Fixture that provides a Redis client instance for the test session. + It connects to this client using the url created from the `redis_server` + fixture. + + Args: + redis_server: The redis server uri we'll use to connect to redis + + Returns: + An instance of the Redis client that can be used to interact + with the Redis server. + """ + return Redis.from_url(url=redis_server) + + +@pytest.fixture(scope="session") +def celery_app(redis_server: FixtureStr) -> FixtureCelery: """ Create the celery app to be used throughout our integration tests. :param redis_server: The redis server uri we'll use to connect to redis :returns: The celery app object we'll use for testing """ + os.environ["CELERY_ENV"] = "test" return Celery("merlin_test_app", broker=redis_server, backend=redis_server) @pytest.fixture(scope="session") -def sleep_sig(celery_app: Celery) -> Signature: +def sleep_sig(celery_app: FixtureCelery) -> FixtureSignature: """ Create a task registered to our celery app and return a signature for it. Once requested by a test, you can set the queue you'd like to send this to @@ -191,7 +309,7 @@ def sleep_task(): @pytest.fixture(scope="session") -def worker_queue_map() -> Dict[str, str]: +def worker_queue_map() -> FixtureDict[str, str]: """ Worker and queue names to be used throughout tests @@ -201,7 +319,7 @@ def worker_queue_map() -> Dict[str, str]: @pytest.fixture(scope="class") -def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): +def launch_workers(celery_app: FixtureCelery, worker_queue_map: FixtureDict[str, str]): """ Launch the workers on the celery app fixture using the worker and queue names defined in the worker_queue_map fixture. @@ -219,7 +337,7 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): @pytest.fixture(scope="session") -def test_encryption_key() -> bytes: +def test_encryption_key() -> FixtureBytes: """ An encryption key to be used for tests that need it. @@ -243,28 +361,33 @@ def test_encryption_key() -> bytes: ####################################### -@pytest.fixture(scope="function") -def config(merlin_server_dir: str, test_encryption_key: bytes): +def _config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes): """ - DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. - This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` - fixtures. It sets up the CONFIG object but leaves certain broker settings unset. + Sets up the configuration for testing purposes by modifying the global CONFIG object. - :param merlin_server_dir: The directory to the merlin test server configuration - :param test_encryption_key: An encryption key to be used for testing - """ + This helper function prepares the broker and results backend configurations for testing + by creating necessary encryption key files and resetting the CONFIG object to its + original state after the tests are executed. + Args: + merlin_server_dir: The directory to the merlin test server configuration + test_encryption_key: An encryption key to be used for testing + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ # Create a copy of the CONFIG option so we can reset it after the test orig_config = copy(CONFIG) # Create an encryption key file (if it doesn't already exist) - key_file = f"{merlin_server_dir}/encrypt_data_key" + key_file = os.path.join(merlin_server_dir, "encrypt_data_key") create_encryption_file(key_file, test_encryption_key) # Set the broker configuration for testing - CONFIG.broker.password = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` - CONFIG.broker.port = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` - CONFIG.broker.name = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.password = None # This will be updated in `redis_broker_config_*` or `rabbit_broker_config` + CONFIG.broker.port = None # This will be updated in `redis_broker_config_*` or `rabbit_broker_config` + CONFIG.broker.name = None # This will be updated in `redis_broker_config_*` or `rabbit_broker_config` CONFIG.broker.server = "127.0.0.1" CONFIG.broker.username = "default" CONFIG.broker.vhost = "host4testing" @@ -272,11 +395,11 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # Set the results_backend configuration for testing CONFIG.results_backend.password = ( - None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + None # This will be updated in `redis_results_backend_config_function` or `mysql_results_backend_config` ) - CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` + CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config_function` CONFIG.results_backend.name = ( - None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + None # This will be updated in `redis_results_backend_config_function` or `mysql_results_backend_config` ) CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` CONFIG.results_backend.server = "127.0.0.1" @@ -295,51 +418,149 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): @pytest.fixture(scope="function") -def redis_broker_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): +def config_function(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: """ - This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a Redis broker and results_backend. + Sets up the configuration for testing with a function scope. - :param merlin_server_dir: The directory to the merlin test server configuration - :param config: The fixture that sets up most of the CONFIG object for testing + Warning: + DO NOT USE THIS FIXTURE IN A TEST, USE ONE OF THE SERVER SPECIFIC CONFIGURATIONS + (LIKE `redis_broker_config_function`, `rabbit_broker_config`, etc.) INSTEAD. + + This fixture modifies the global CONFIG object to prepare the broker and results backend + configurations for testing. It creates necessary encryption key files and ensures that + the original configuration is restored after the tests are executed. + + Args: + merlin_server_dir: The directory to the merlin test server configuration + test_encryption_key: An encryption key to be used for testing + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. """ - pass_file = f"{merlin_server_dir}/redis.pass" - create_pass_file(pass_file) + yield from _config(merlin_server_dir, test_encryption_key) - CONFIG.broker.password = pass_file - CONFIG.broker.port = 6379 - CONFIG.broker.name = "redis" +@pytest.fixture(scope="class") +def config_class(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: + """ + Sets up the configuration for testing with a class scope. + + Warning: + DO NOT USE THIS FIXTURE IN A TEST, USE ONE OF THE SERVER SPECIFIC CONFIGURATIONS + (LIKE `redis_broker_config_class`, `rabbit_broker_config`, etc.) INSTEAD. + + This fixture modifies the global CONFIG object to prepare the broker and results backend + configurations for testing. It creates necessary encryption key files and ensures that + the original configuration is restored after the tests are executed. + + Args: + merlin_server_dir: The directory to the merlin test server configuration + test_encryption_key: An encryption key to be used for testing + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + yield from _config(merlin_server_dir, test_encryption_key) + + +@pytest.fixture(scope="function") +def redis_broker_config_function( + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: + """ + Fixture for configuring the Redis broker for testing with a function scope. + + This fixture sets up the CONFIG object to use a Redis broker for testing any functionality + in the codebase that interacts with the broker. It modifies the configuration to point + to the specified Redis broker settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + setup_redis_config("broker", merlin_server_dir) + yield + + +@pytest.fixture(scope="class") +def redis_broker_config_class( + merlin_server_dir: FixtureStr, config_class: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: + """ + Fixture for configuring the Redis broker for testing with a class scope. + + This fixture sets up the CONFIG object to use a Redis broker for testing any functionality + in the codebase that interacts with the broker. It modifies the configuration to point + to the specified Redis broker settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + setup_redis_config("broker", merlin_server_dir) yield @pytest.fixture(scope="function") -def redis_results_backend_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): +def redis_results_backend_config_function( + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ - This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a Redis results_backend. + Fixture for configuring the Redis results backend for testing with a function scope. - :param merlin_server_dir: The directory to the merlin test server configuration - :param config: The fixture that sets up most of the CONFIG object for testing + This fixture sets up the CONFIG object to use a Redis results backend for testing any + functionality in the codebase that interacts with the results backend. It modifies the + configuration to point to the specified Redis results backend settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. """ - pass_file = f"{merlin_server_dir}/redis.pass" - create_pass_file(pass_file) + setup_redis_config("results_backend", merlin_server_dir) + yield - CONFIG.results_backend.password = pass_file - CONFIG.results_backend.port = 6379 - CONFIG.results_backend.name = "redis" +@pytest.fixture(scope="class") +def redis_results_backend_config_class( + merlin_server_dir: FixtureStr, config_class: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: + """ + Fixture for configuring the Redis results backend for testing with a class scope. + + This fixture sets up the CONFIG object to use a Redis results backend for testing any + functionality in the codebase that interacts with the results backend. It modifies the + configuration to point to the specified Redis results backend settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + setup_redis_config("results_backend", merlin_server_dir) yield @pytest.fixture(scope="function") def rabbit_broker_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a RabbitMQ broker. @@ -347,7 +568,7 @@ def rabbit_broker_config( :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - pass_file = f"{merlin_server_dir}/rabbit.pass" + pass_file = os.path.join(merlin_server_dir, "rabbit.pass") create_pass_file(pass_file) CONFIG.broker.password = pass_file @@ -359,8 +580,8 @@ def rabbit_broker_config( @pytest.fixture(scope="function") def mysql_results_backend_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a MySQL results_backend. @@ -368,7 +589,7 @@ def mysql_results_backend_config( :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - pass_file = f"{merlin_server_dir}/mysql.pass" + pass_file = os.path.join(merlin_server_dir, "mysql.pass") create_pass_file(pass_file) create_cert_files(merlin_server_dir, CERT_FILES) @@ -381,3 +602,81 @@ def mysql_results_backend_config( CONFIG.results_backend.ca_certs = CERT_FILES["ssl_ca"] yield + + +@pytest.fixture(scope="function") +def redis_broker_and_backend_function( + redis_client: FixtureRedis, + redis_server: FixtureStr, + redis_broker_config_function: FixtureModification, + redis_results_backend_config_function: FixtureModification, +): + """ + Fixture for setting up Redis broker and backend for function-scoped tests. + + This fixture creates an instance of `RedisBrokerAndBackend`, which + encapsulates all necessary Redis-related fixtures required for + establishing connections to Redis as both a broker and a backend + during function-scoped tests. + + Args: + redis_client: A fixture that provides a client for interacting with the + Redis server. + redis_server: A fixture providing the connection string to the Redis + server instance. + redis_broker_config_function: A fixture that modifies the configuration + to point to the Redis server used as the message broker for + function-scoped tests. + redis_results_backend_config_function: A fixture that modifies the + configuration to point to the Redis server used for storing results + in function-scoped tests. + + Returns: + An instance containing the Redis client, server connection string, and + configuration modifications for both the broker and backend. + """ + return RedisBrokerAndBackend( + client=redis_client, + server=redis_server, + broker_config=redis_broker_config_function, + results_backend_config=redis_results_backend_config_function, + ) + + +@pytest.fixture(scope="class") +def redis_broker_and_backend_class( + redis_client: FixtureRedis, + redis_server: FixtureStr, + redis_broker_config_class: FixtureModification, + redis_results_backend_config_class: FixtureModification, +) -> RedisBrokerAndBackend: + """ + Fixture for setting up Redis broker and backend for class-scoped tests. + + This fixture creates an instance of `RedisBrokerAndBackend`, which + encapsulates all necessary Redis-related fixtures required for + establishing connections to Redis as both a broker and a backend + during class-scoped tests. + + Args: + redis_client: A fixture that provides a client for interacting with the + Redis server. + redis_server: A fixture providing the connection string to the Redis + server instance. + redis_broker_config_function: A fixture that modifies the configuration + to point to the Redis server used as the message broker for + class-scoped tests. + redis_results_backend_config_function: A fixture that modifies the + configuration to point to the Redis server used for storing results + in class-scoped tests. + + Returns: + An instance containing the Redis client, server connection string, and + configuration modifications for both the broker and backend. + """ + return RedisBrokerAndBackend( + client=redis_client, + server=redis_server, + broker_config=redis_broker_config_class, + results_backend_config=redis_results_backend_config_class, + ) diff --git a/tests/context_managers/celery_task_manager.py b/tests/context_managers/celery_task_manager.py new file mode 100644 index 000000000..b71dcd8a4 --- /dev/null +++ b/tests/context_managers/celery_task_manager.py @@ -0,0 +1,155 @@ +""" +Module to define functionality for sending tasks to the server +and ensuring they're cleared from the server when the test finishes. +""" + +from types import TracebackType +from typing import List, Type + +from celery import Celery +from celery.result import AsyncResult +from redis import Redis + + +class CeleryTaskManager: + """ + A context manager for managing Celery tasks. + + This class provides a way to send tasks to a Celery server and clean up + any tasks that were sent during its lifetime. It is designed to be used + as a context manager, ensuring that tasks are properly removed from the + server when the context is exited. + + Attributes: + celery_app: The Celery application instance. + redis_server: The Redis server connection string. + """ + + def __init__(self, app: Celery, redis_client: Redis): + self.celery_app: Celery = app + self.redis_client = redis_client + + def __enter__(self) -> "CeleryTaskManager": + """ + Enters the runtime context related to this object. + + Returns: + The current instance of the manager. + """ + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + Exits the runtime context and performs cleanup. + + This method removes any tasks currently in the server. + + Args: + exc_type: The exception type raised, if any. + exc_value: The exception instance raised, if any. + traceback: The traceback object, if an exception was raised. + """ + self.remove_tasks() + + def send_task(self, task_name: str, *args, **kwargs) -> AsyncResult: + """ + Sends a task to the Celery server. + + This method will be used for tests that don't call + `merlin run`, allowing for isolated test functionality. + + Args: + task_name: The name of the task to send to the server. + *args: Additional positional arguments to pass to the task. + **kwargs: Additional keyword arguments to pass to the task. + + Returns: + A Celery AsyncResult object containing information about the + task that was sent to the server. + """ + valid_kwargs = [ + "add_to_parent", + "chain", + "chord", + "compression", + "connection", + "countdown", + "eta", + "exchange", + "expires", + "group_id", + "group_index", + "headers", + "ignore_result", + "link", + "link_error", + "parent_id", + "priority", + "producer", + "publisher", + "queue", + "replaced_task_nesting", + "reply_to", + "result_cls", + "retries", + "retry", + "retry_policy", + "root_id", + "route_name", + "router", + "routing_key", + "serializer", + "shadow", + "soft_time_limit", + "task_id", + "task_type", + "time_limit", + ] + send_task_kwargs = {key: kwargs.pop(key) for key in valid_kwargs if key in kwargs} + + return self.celery_app.send_task(task_name, args=args, kwargs=kwargs, **send_task_kwargs) + + def remove_tasks(self): + """ + Removes tasks from the Celery server. + + Tasks are removed in two ways: + 1. By purging the Celery app queues, which will only purge tasks + sent with `send_task`. + 2. By deleting the remaining queues in the Redis server, which will + purge any tasks that weren't sent with `send_task` (e.g., tasks + sent with `merlin run`). + """ + # Purge the tasks + self.celery_app.control.purge() + + # Purge any remaining tasks directly through redis that may have been missed + queues = self.get_queue_list() + for queue in queues: + self.redis_client.delete(queue) + + def get_queue_list(self) -> List[str]: + """ + Builds a list of Celery queues that exist on the Redis server. + + Queries the Redis server for its keys and returns the keys + that represent the Celery queues. + + Returns: + A list of Celery queue names. + """ + cursor = 0 + queues = [] + while True: + # Get the 'merlin' queue if it exists + cursor, matching_queues = self.redis_client.scan(cursor=cursor, match="merlin") + queues.extend(matching_queues) + + # Get any queues that start with '[merlin]' + cursor, matching_queues = self.redis_client.scan(cursor=cursor, match="\\[merlin\\]*") + queues.extend(matching_queues) + + if cursor == 0: + break + + return queues diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 5e750b6ef..279eab325 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -52,7 +52,8 @@ class CeleryWorkersManager: def __init__(self, app: Celery): self.app = app - self.running_workers = [] + self.running_workers = set() + self.run_worker_processes = set() self.worker_processes = {} self.echo_processes = {} @@ -80,8 +81,9 @@ def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: T try: if str(pid) in ps_proc.stdout: os.kill(pid, signal.SIGKILL) - except ProcessLookupError as exc: - raise ProcessLookupError(f"PID {pid} not found. Output of 'ps ux':\n{ps_proc.stdout}") from exc + # If the process can't be found then it doesn't exist anymore + except ProcessLookupError: + pass def _is_worker_ready(self, worker_name: str, verbose: bool = False) -> bool: """ @@ -144,7 +146,7 @@ def start_worker(self, worker_launch_cmd: List[str]): """ self.app.worker_main(worker_launch_cmd) - def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = 1): + def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = 1, prefetch: int = 1): """ Launch a single worker. We'll add the process that the worker is running in to the list of worker processes. We'll also create an echo process to simulate a celery worker command that will show up with 'ps ux'. @@ -158,6 +160,8 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = self.stop_all_workers() raise ValueError(f"The worker {worker_name} is already running. Choose a different name.") + queues = [f"[merlin]_{queue}" for queue in queues] + # Create the launch command for this worker worker_launch_cmd = [ "worker", @@ -167,6 +171,8 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = ",".join(queues), "--concurrency", str(concurrency), + "--prefetch-multiplier", + str(prefetch), f"--logfile={worker_name}.log", "--loglevel=DEBUG", ] @@ -174,9 +180,9 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = # Create an echo command to simulate a running celery worker since our celery worker will be spun up in # a different process and we won't be able to see it with 'ps ux' like we normally would echo_process = subprocess.Popen( # pylint: disable=consider-using-with - f"echo 'celery merlin_test_app {' '.join(worker_launch_cmd)}'; sleep inf", + f"echo 'celery -A merlin_test_app {' '.join(worker_launch_cmd)}'; sleep inf", shell=True, - preexec_fn=os.setpgrp, # Make this the parent of the group so we can kill the 'sleep inf' that's spun up + start_new_session=True, # Make this the parent of the group so we can kill the 'sleep inf' that's spun up ) self.echo_processes[worker_name] = echo_process.pid @@ -184,7 +190,7 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = worker_process = multiprocessing.Process(target=self.start_worker, args=(worker_launch_cmd,)) worker_process.start() self.worker_processes[worker_name] = worker_process - self.running_workers.append(worker_name) + self.running_workers.add(worker_name) # Wait for the worker to launch properly try: @@ -204,6 +210,24 @@ def launch_workers(self, worker_info: Dict[str, Dict]): for worker_name, worker_settings in worker_info.items(): self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) + def add_run_workers_process(self, pid: int): + """ + Add a process ID for a `merlin run-workers` process to the + set that tracks all `merlin run-workers` processes that are + currently running. + + Warning: + The process that's added here must utilize the + `start_new_session=True` setting of subprocess.Popen. This + is necessary for us to be able to terminate all the workers + that are started with it safely since they will be seen as + child processes of the `merlin run-workers` process. + + Args: + pid: The process ID running `merlin run-workers`. + """ + self.run_worker_processes.add(pid) + def stop_worker(self, worker_name: str): """ Stop a single running worker and its associated processes. @@ -223,12 +247,16 @@ def stop_worker(self, worker_name: str): self.worker_processes[worker_name].kill() # Terminate the echo process and its sleep inf subprocess - os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGTERM) - sleep(2) + if self.echo_processes[worker_name] is not None: + os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGKILL) + sleep(2) def stop_all_workers(self): """ Stop all of the running workers and the processes associated with them. """ + for run_worker_pid in self.run_worker_processes: + os.killpg(os.getpgid(run_worker_pid), signal.SIGKILL) + for worker_name in self.running_workers: self.stop_worker(worker_name) diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index b99afb2c6..bb5d86036 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -91,7 +91,7 @@ def stop_server(self): if "Merlin server terminated." not in kill_process.stderr: # If it wasn't, try to kill the process by using the pid stored in a file created by `merlin server` try: - with open(f"{self.server_dir}/merlin_server.pf", "r") as process_file: + with open(os.path.join(self.server_dir, "merlin_server.pf"), "r") as process_file: server_process_info = yaml.load(process_file, yaml.Loader) os.kill(int(server_process_info["image_pid"]), signal.SIGKILL) # If the file can't be found then let's make sure there's even a redis-server process running diff --git a/tests/fixture_data_classes.py b/tests/fixture_data_classes.py new file mode 100644 index 000000000..2ff7bf877 --- /dev/null +++ b/tests/fixture_data_classes.py @@ -0,0 +1,80 @@ +""" +This module houses dataclasses to be used with pytest fixtures. +""" + +from dataclasses import dataclass + +from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr + + +@dataclass +class RedisBrokerAndBackend: + """ + Data class to encapsulate all Redis-related fixtures required for + establishing connections to Redis for both the broker and backend. + + This class simplifies the management of Redis fixtures by grouping + them into a single object, reducing the number of individual fixture + imports needed in tests that require Redis functionality. + + Attributes: + client: A fixture that provides a client for interacting + with the Redis server. + server: A fixture providing the connection string to the + Redis server instance. + broker_config: A fixture that modifies the configuration + to point to the Redis server used as the message broker. + results_backend_config: A fixture that modifies the + configuration to point to the Redis server used for storing + results. + """ + + client: FixtureRedis + server: FixtureStr + results_backend_config: FixtureModification + broker_config: FixtureModification + + +@dataclass +class FeatureDemoSetup: + """ + Data class to encapsulate all feature-demo-related fixtures required + for testing the feature demo workflow. + + This class simplifies the management of feature demo setup fixtures + by grouping them into a single object, reducing the number of individual + fixture imports needed in tests that require feature demo setup. + + Attributes: + testing_dir: The path to the temp output directory for feature_demo workflow tests. + num_samples: An integer representing the number of samples to use in the feature_demo + workflow. + name: A string representing the name to use for the feature_demo workflow. + path: The path to the feature demo YAML file. + """ + + testing_dir: FixtureStr + num_samples: FixtureInt + name: FixtureStr + path: FixtureStr + + +@dataclass +class ChordErrorSetup: + """ + Data class to encapsulate all chord-error-related fixtures required + for testing the chord error workflow. + + This class simplifies the management of chord error setup fixtures + by grouping them into a single object, reducing the number of individual + fixture imports needed in tests that require chord error setup. + + Attributes: + testing_dir: The path to the temp output directory for chord_err workflow tests. + name: A string representing the name to use for the chord_err workflow. + path: The path to the chord error YAML file. + """ + + testing_dir: FixtureStr + name: FixtureStr + path: FixtureStr diff --git a/tests/fixture_types.py b/tests/fixture_types.py new file mode 100644 index 000000000..ac633d4dd --- /dev/null +++ b/tests/fixture_types.py @@ -0,0 +1,74 @@ +""" +It's hard to type hint pytest fixtures in a way that makes it clear +that the variable being used is a fixture. This module will created +aliases for these fixtures in order to make it easier to track what's +happening. + +The types here will be defined as such: +- `FixtureBytes`: A fixture that returns bytes +- `FixtureCelery`: A fixture that returns a Celery app object +- `FixtureDict`: A fixture that returns a dictionary +- `FixtureInt`: A fixture that returns an integer +- `FixtureModification`: A fixture that modifies something but never actually + returns/yields a value to be used in the test. +- `FixtureRedis`: A fixture that returns a Redis client +- `FixtureSignature`: A fixture that returns a Celery Signature object +- `FixtureStr`: A fixture that returns a string +""" + +import sys +from argparse import Namespace +from collections.abc import Callable +from typing import Any, Dict, Generic, Tuple, TypeVar + +import pytest +from celery import Celery +from celery.canvas import Signature +from redis import Redis + + +# TODO convert unit test type hinting to use these +# - likely will do this when I work on API docs for test library + +K = TypeVar("K") +V = TypeVar("V") + +# TODO when we drop support for Python 3.8, remove this if/else statement +# Check Python version +if sys.version_info >= (3, 9): + from typing import Annotated + + FixtureBytes = Annotated[bytes, pytest.fixture] + FixtureCallable = Annotated[Callable, pytest.fixture] + FixtureCelery = Annotated[Celery, pytest.fixture] + FixtureDict = Annotated[Dict[K, V], pytest.fixture] + FixtureInt = Annotated[int, pytest.fixture] + FixtureModification = Annotated[Any, pytest.fixture] + FixtureNamespace = Annotated[Namespace, pytest.fixture] + FixtureRedis = Annotated[Redis, pytest.fixture] + FixtureSignature = Annotated[Signature, pytest.fixture] + FixtureStr = Annotated[str, pytest.fixture] + FixtureTuple = Annotated[Tuple[K, V], pytest.fixture] +else: + # Fallback for Python 3.7 and 3.8 + class FixtureDict(Generic[K, V], Dict[K, V]): + """ + This class is necessary to allow FixtureDict to be subscriptable + when using it to type hint. + """ + + class FixtureTuple(Generic[K, V], Tuple[K, V]): + """ + This class is necessary to allow FixtureTuple to be subscriptable + when using it to type hint. + """ + + FixtureBytes = pytest.fixture + FixtureCallable = pytest.fixture + FixtureCelery = pytest.fixture + FixtureInt = pytest.fixture + FixtureModification = pytest.fixture + FixtureNamespace = pytest.fixture + FixtureRedis = pytest.fixture + FixtureSignature = pytest.fixture + FixtureStr = pytest.fixture diff --git a/tests/fixtures/chord_err.py b/tests/fixtures/chord_err.py new file mode 100644 index 000000000..54c639072 --- /dev/null +++ b/tests/fixtures/chord_err.py @@ -0,0 +1,114 @@ +""" +Fixtures specifically for help testing the chord_err workflow. +""" + +import os +import subprocess + +import pytest + +from tests.fixture_data_classes import ChordErrorSetup, RedisBrokerAndBackend +from tests.fixture_types import FixtureCallable, FixtureStr +from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow + + +# pylint: disable=redefined-outer-name + + +@pytest.fixture(scope="session") +def chord_err_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: + """ + Fixture to create a temporary output directory for tests related to testing the + chord_err workflow. + + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. + + Returns: + The path to the temporary testing directory for chord_err workflow tests. + """ + return create_testing_dir(temp_output_dir, "chord_err_testing") + + +@pytest.fixture(scope="session") +def chord_err_name() -> FixtureStr: + """ + Defines a specific name to use for the chord_err workflow. This helps ensure + that even if changes were made to the chord_err workflow, tests using this fixture + should still run the same thing. + + Returns: + A string representing the name to use for the chord_err workflow. + """ + return "chord_err_test" + + +@pytest.fixture(scope="session") +def chord_err_setup( + chord_err_testing_dir: FixtureStr, + chord_err_name: FixtureStr, + path_to_test_specs: FixtureStr, +) -> ChordErrorSetup: + """ + Fixture for setting up the environment required for testing the chord error workflow. + + This fixture prepares the necessary configuration and paths for executing tests related + to the chord error workflow. It aggregates the required parameters into a single + [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] data class instance, which + simplifies the management of these parameters in tests. + + Args: + chord_err_testing_dir: The path to the temporary output directory where chord error + workflow tests will store their results. + chord_err_name: A string representing the name to use for the chord error workflow. + path_to_test_specs: The base path to the Merlin test specs directory, which is + used to locate the chord error YAML file. + + Returns: + A [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] instance containing + the testing directory, name, and path to the chord error YAML file, which can + be used in tests that require this setup. + """ + chord_err_path = os.path.join(path_to_test_specs, "chord_err.yaml") + return ChordErrorSetup( + testing_dir=chord_err_testing_dir, + name=chord_err_name, + path=chord_err_path, + ) + + +@pytest.fixture(scope="class") +def chord_err_run_workflow( + redis_broker_and_backend_class: RedisBrokerAndBackend, + chord_err_setup: ChordErrorSetup, + merlin_server_dir: FixtureStr, +) -> subprocess.CompletedProcess: + """ + Run the chord error workflow. + + This fixture sets up and executes the chord error workflow using the specified configurations + and parameters. It prepares the environment by modifying the CONFIG object to connect to a + Redis server and runs the workflow with the provided name and output path. + + Args: + redis_broker_and_backend_class: Fixture for setting up Redis broker and + backend for class-scoped tests. + chord_err_setup: A fixture that returns a [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] + instance. + merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be + created by the [`redis_server`][conftest.redis_server] fixture. + + Returns: + The completed process object containing information about the execution of the workflow, including + return code, stdout, and stderr. + """ + # Setup the test + copy_app_yaml_to_cwd(merlin_server_dir) + # chord_err_path = os.path.join(path_to_test_specs, "chord_err.yaml") + + # Create the variables to pass in to the workflow + vars_to_substitute = [f"NAME={chord_err_setup.name}", f"OUTPUT_PATH={chord_err_setup.testing_dir}"] + + # Run the workflow + return run_workflow(redis_broker_and_backend_class.client, chord_err_setup.path, vars_to_substitute) diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py index 7c4626e3e..4096b0e76 100644 --- a/tests/fixtures/examples.py +++ b/tests/fixtures/examples.py @@ -2,21 +2,21 @@ Fixtures specifically for help testing the modules in the examples/ directory. """ -import os - import pytest +from tests.fixture_types import FixtureCallable, FixtureStr + @pytest.fixture(scope="session") -def examples_testing_dir(temp_output_dir: str) -> str: +def examples_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to the examples functionality. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: The path to the temporary testing directory for examples tests - """ - testing_dir = f"{temp_output_dir}/examples_testing" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary output directory we'll be using for this test run. - return testing_dir + Returns: + The path to the temporary testing directory for examples tests. + """ + return create_testing_dir(temp_output_dir, "examples_testing") diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py new file mode 100644 index 000000000..ca77f23fb --- /dev/null +++ b/tests/fixtures/feature_demo.py @@ -0,0 +1,136 @@ +""" +Fixtures specifically for help testing the feature_demo workflow. +""" + +import os +import subprocess + +import pytest + +from tests.fixture_data_classes import FeatureDemoSetup, RedisBrokerAndBackend +from tests.fixture_types import FixtureCallable, FixtureInt, FixtureStr +from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow + + +# pylint: disable=redefined-outer-name + + +@pytest.fixture(scope="session") +def feature_demo_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: + """ + Fixture to create a temporary output directory for tests related to testing the + feature_demo workflow. + + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. + + Returns: + The path to the temporary testing directory for feature_demo workflow tests. + """ + return create_testing_dir(temp_output_dir, "feature_demo_testing") + + +@pytest.fixture(scope="session") +def feature_demo_num_samples() -> FixtureInt: + """ + Defines a specific number of samples to use for the feature_demo workflow. + This helps ensure that even if changes were made to the feature_demo workflow, + tests using this fixture should still run the same thing. + + Returns: + An integer representing the number of samples to use in the feature_demo workflow. + """ + return 8 + + +@pytest.fixture(scope="session") +def feature_demo_name() -> FixtureStr: + """ + Defines a specific name to use for the feature_demo workflow. This helps ensure + that even if changes were made to the feature_demo workflow, tests using this fixture + should still run the same thing. + + Returns: + A string representing the name to use for the feature_demo workflow. + """ + return "feature_demo_test" + + +@pytest.fixture(scope="session") +def feature_demo_setup( + feature_demo_testing_dir: FixtureStr, + feature_demo_num_samples: FixtureInt, + feature_demo_name: FixtureStr, + path_to_merlin_codebase: FixtureStr, +) -> FeatureDemoSetup: + """ + Fixture for setting up the environment required for testing the feature demo workflow. + + This fixture prepares the necessary configuration and paths for executing tests related + to the feature demo workflow. It aggregates the required parameters into a single + [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] data class instance, which + simplifies the management of these parameters in tests. + + Args: + feature_demo_testing_dir: The path to the temporary output directory where + feature demo workflow tests will store their results. + feature_demo_num_samples: An integer representing the number of samples + to use in the feature demo workflow. + feature_demo_name: A string representing the name to use for the feature + demo workflow. + path_to_merlin_codebase: The base path to the Merlin codebase, which is + used to locate the feature demo YAML file. + + Returns: + A [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] instance containing + the testing directory, number of samples, name, and path to the feature demo + YAML file, which can be used in tests that require this setup. + """ + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + feature_demo_path = os.path.join(path_to_merlin_codebase, demo_workflow) + return FeatureDemoSetup( + testing_dir=feature_demo_testing_dir, + num_samples=feature_demo_num_samples, + name=feature_demo_name, + path=feature_demo_path, + ) + + +@pytest.fixture(scope="class") +def feature_demo_run_workflow( + redis_broker_and_backend_class: RedisBrokerAndBackend, + feature_demo_setup: FeatureDemoSetup, + merlin_server_dir: FixtureStr, +) -> subprocess.CompletedProcess: + """ + Run the feature demo workflow. + + This fixture sets up and executes the feature demo workflow using the specified configurations + and parameters. It prepares the environment by modifying the CONFIG object to connect to a + Redis server and runs the demo workflow with the provided sample size and name. + + Args: + redis_broker_and_backend_class: Fixture for setting up Redis broker and + backend for class-scoped tests. + feature_demo_setup: A fixture that returns a [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] + instance. + merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be + created by the [`redis_server`][conftest.redis_server] fixture. + + Returns: + The completed process object containing information about the execution of the workflow, including + return code, stdout, and stderr. + """ + # Setup the test + copy_app_yaml_to_cwd(merlin_server_dir) + + # Create the variables to pass in to the workflow + vars_to_substitute = [ + f"N_SAMPLES={feature_demo_setup.num_samples}", + f"NAME={feature_demo_setup.name}", + f"OUTPUT_PATH={feature_demo_setup.testing_dir}", + ] + + # Run the workflow + return run_workflow(redis_broker_and_backend_class.client, feature_demo_setup.path, vars_to_substitute) diff --git a/tests/fixtures/run_command.py b/tests/fixtures/run_command.py new file mode 100644 index 000000000..5a666209b --- /dev/null +++ b/tests/fixtures/run_command.py @@ -0,0 +1,23 @@ +""" +Fixtures specifically for help testing the `merlin run` command. +""" + +import pytest + +from tests.fixture_types import FixtureCallable, FixtureStr + + +@pytest.fixture(scope="session") +def run_command_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: + """ + Fixture to create a temporary output directory for tests related to testing the + `merlin run` functionality. + + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. + + Returns: + The path to the temporary testing directory for `merlin run` tests. + """ + return create_testing_dir(temp_output_dir, "run_command_testing") diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 4f4a07a2c..c2bcdc762 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -9,27 +9,29 @@ import pytest import yaml +from tests.fixture_types import FixtureCallable, FixtureDict, FixtureNamespace, FixtureStr + # pylint: disable=redefined-outer-name @pytest.fixture(scope="session") -def server_testing_dir(temp_output_dir: str) -> str: +def server_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to the server functionality. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: The path to the temporary testing directory for server tests - """ - testing_dir = f"{temp_output_dir}/server_testing" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - return testing_dir + Returns: + The path to the temporary testing directory for server tests. + """ + return create_testing_dir(temp_output_dir, "server_testing") @pytest.fixture(scope="session") -def server_redis_conf_file(server_testing_dir: str) -> str: +def server_redis_conf_file(server_testing_dir: FixtureStr) -> FixtureStr: """ Fixture to write a redis.conf file to the temporary output directory. @@ -77,7 +79,7 @@ def server_redis_conf_file(server_testing_dir: str) -> str: @pytest.fixture(scope="session") -def server_redis_pass_file(server_testing_dir: str) -> str: +def server_redis_pass_file(server_testing_dir: FixtureStr) -> FixtureStr: """ Fixture to create a redis password file in the temporary output directory. @@ -96,7 +98,7 @@ def server_redis_pass_file(server_testing_dir: str) -> str: @pytest.fixture(scope="session") -def server_users() -> Dict[str, Dict[str, str]]: +def server_users() -> FixtureDict[str, Dict[str, str]]: """ Create a dictionary of two test users with identical configuration settings. @@ -122,7 +124,7 @@ def server_users() -> Dict[str, Dict[str, str]]: @pytest.fixture(scope="session") -def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: +def server_redis_users_file(server_testing_dir: FixtureStr, server_users: FixtureDict[str, Dict[str, str]]) -> FixtureStr: """ Fixture to write a redis.users file to the temporary output directory. @@ -143,11 +145,11 @@ def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: @pytest.fixture(scope="class") def server_container_config_data( - server_testing_dir: str, - server_redis_conf_file: str, - server_redis_pass_file: str, - server_redis_users_file: str, -) -> Dict[str, str]: + server_testing_dir: FixtureStr, + server_redis_conf_file: FixtureStr, + server_redis_pass_file: FixtureStr, + server_redis_users_file: FixtureStr, +) -> FixtureDict[str, str]: """ Fixture to provide sample data for ContainerConfig tests. @@ -172,7 +174,7 @@ def server_container_config_data( @pytest.fixture(scope="class") -def server_container_format_config_data() -> Dict[str, str]: +def server_container_format_config_data() -> FixtureDict[str, str]: """ Fixture to provide sample data for ContainerFormatConfig tests @@ -187,7 +189,7 @@ def server_container_format_config_data() -> Dict[str, str]: @pytest.fixture(scope="class") -def server_process_config_data() -> Dict[str, str]: +def server_process_config_data() -> FixtureDict[str, str]: """ Fixture to provide sample data for ProcessConfig tests @@ -201,10 +203,10 @@ def server_process_config_data() -> Dict[str, str]: @pytest.fixture(scope="class") def server_server_config( - server_container_config_data: Dict[str, str], - server_process_config_data: Dict[str, str], - server_container_format_config_data: Dict[str, str], -) -> Dict[str, Dict[str, str]]: + server_container_config_data: FixtureDict[str, str], + server_process_config_data: FixtureDict[str, str], + server_container_format_config_data: FixtureDict[str, str], +) -> FixtureDict[str, FixtureDict[str, str]]: """ Fixture to provide sample data for ServerConfig tests @@ -222,10 +224,10 @@ def server_server_config( @pytest.fixture(scope="function") def server_app_yaml_contents( - server_redis_pass_file: str, - server_container_config_data: Dict[str, str], - server_process_config_data: Dict[str, str], -) -> Dict[str, Union[str, int]]: + server_redis_pass_file: FixtureStr, + server_container_config_data: FixtureDict[str, str], + server_process_config_data: FixtureDict[str, str], +) -> FixtureDict[str, Union[str, int]]: """ Fixture to create the contents of an app.yaml file. @@ -260,7 +262,7 @@ def server_app_yaml_contents( @pytest.fixture(scope="function") -def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> str: +def server_app_yaml(server_testing_dir: FixtureStr, server_app_yaml_contents: FixtureDict[str, Union[str, int]]) -> FixtureStr: """ Fixture to create an app.yaml file in the temporary output directory. @@ -280,13 +282,13 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> @pytest.fixture(scope="function") -def server_process_file_contents() -> str: +def server_process_file_contents() -> FixtureDict[str, Union[str, int]]: """Fixture to represent process file contents.""" return {"parent_pid": 123, "image_pid": 456, "port": 6379, "hostname": "dummy_server"} @pytest.fixture(scope="function") -def server_config_server_args() -> Namespace: +def server_config_server_args() -> FixtureNamespace: """ Setup an argparse Namespace with all args that the `config_server` function will need. These can be modified on a test-by-test basis. diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index 39a36f9bf..57ff16bac 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -11,6 +11,7 @@ import pytest import yaml +from tests.fixture_types import FixtureCallable, FixtureNamespace, FixtureStr from tests.unit.study.status_test_files import status_test_variables @@ -18,22 +19,22 @@ @pytest.fixture(scope="session") -def status_testing_dir(temp_output_dir: str) -> str: +def status_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ A pytest fixture to set up a temporary directory to write files to for testing status. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: The path to the temporary testing directory for status testing - """ - testing_dir = f"{temp_output_dir}/status_testing/" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - return testing_dir + Returns: + The path to the temporary testing directory for status tests. + """ + return create_testing_dir(temp_output_dir, "status_testing") @pytest.fixture(scope="class") -def status_empty_file(status_testing_dir: str) -> str: +def status_empty_file(status_testing_dir: FixtureStr) -> FixtureStr: """ A pytest fixture to create an empty status file. @@ -49,7 +50,7 @@ def status_empty_file(status_testing_dir: str) -> str: @pytest.fixture(scope="session") -def status_spec_path(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_spec_path(status_testing_dir: FixtureStr) -> FixtureStr: # pylint: disable=W0621 """ Copy the test spec to the temp directory and modify the OUTPUT_PATH in the spec to point to the temp location. @@ -94,7 +95,7 @@ def set_sample_path(output_workspace: str): @pytest.fixture(scope="session") -def status_output_workspace(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_output_workspace(status_testing_dir: FixtureStr) -> FixtureStr: # pylint: disable=W0621 """ A pytest fixture to copy the test output workspace for status to the temporary status testing directory. @@ -110,7 +111,7 @@ def status_output_workspace(status_testing_dir: str) -> str: # pylint: disable= @pytest.fixture(scope="function") -def status_args(): +def status_args() -> FixtureNamespace: """ A pytest fixture to set up a namespace with all the arguments necessary for the Status object. @@ -130,7 +131,7 @@ def status_args(): @pytest.fixture(scope="session") -def status_nested_workspace(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_nested_workspace(status_testing_dir: FixtureStr) -> FixtureStr: # pylint: disable=W0621 """ Create an output workspace that contains another output workspace within one of its steps. In this case it will copy the status test workspace then within the 'just_samples' diff --git a/tests/integration/commands/__init__.py b/tests/integration/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/commands/pgen.py b/tests/integration/commands/pgen.py new file mode 100644 index 000000000..6973317d6 --- /dev/null +++ b/tests/integration/commands/pgen.py @@ -0,0 +1,36 @@ +""" +This file contains pgen functionality for testing purposes. +It's specifically set up to work with the feature demo example. +""" + +import random + +from maestrowf.datastructures.core import ParameterGenerator + + +# pylint complains about unused argument `env` but it's necessary for Maestro +def get_custom_generator(env, **kwargs): # pylint: disable=unused-argument + """ + Custom parameter generator that's used for testing the `--pgen` flag + of the `merlin run` command. + """ + p_gen = ParameterGenerator() + + # Unpack any pargs passed in + x2_min = int(kwargs.get("X2_MIN", "0")) + x2_max = int(kwargs.get("X2_MAX", "1")) + n_name_min = int(kwargs.get("N_NAME_MIN", "0")) + n_name_max = int(kwargs.get("N_NAME_MAX", "10")) + + # We'll only have two parameter entries each just for testing + num_points = 2 + + params = { + "X2": {"values": [random.uniform(x2_min, x2_max) for _ in range(num_points)], "label": "X2.%%"}, + "N_NEW": {"values": [random.randint(n_name_min, n_name_max) for _ in range(num_points)], "label": "N_NEW.%%"}, + } + + for key, value in params.items(): + p_gen.add_parameter(key, value["values"], value["label"]) + + return p_gen diff --git a/tests/integration/commands/test_purge.py b/tests/integration/commands/test_purge.py new file mode 100644 index 000000000..91340e946 --- /dev/null +++ b/tests/integration/commands/test_purge.py @@ -0,0 +1,363 @@ +""" +This module will contain the testing logic +for the `merlin purge` command. +""" + +import os +import subprocess +from typing import Dict, List, Tuple, Union + +from merlin.spec.expansion import get_spec_with_expansion +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import FixtureRedis, FixtureStr +from tests.integration.conditions import HasRegex, HasReturnCode +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd + + +class TestPurgeCommand: + """ + Tests for the `merlin purge` command. + """ + + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + + def setup_test(self, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr) -> str: + """ + Setup the test environment for these tests by: + 1. Copying the app.yaml file created by the `redis_server` fixture to the cwd so that + Merlin can connect to the test server. + 2. Obtaining the path to the feature_demo spec that we'll use for these tests. + + Args: + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + + Returns: + The path to the feature_demo spec file. + """ + copy_app_yaml_to_cwd(merlin_server_dir) + return os.path.join(path_to_merlin_codebase, self.demo_workflow) + + def setup_tasks(self, celery_task_manager: CeleryTaskManager, spec_file: str) -> Tuple[Dict[str, str], int]: + """ + Helper method to setup tasks in the specified queues. + + This method sends tasks named 'task_for_{queue}' to each queue defined in the + provided spec file and returns the total number of queues that received tasks. + + Args: + celery_task_manager: + A context manager for managing Celery tasks, used to send tasks to the server. + spec_file: + The path to the spec file from which queues will be extracted. + + Returns: + A tuple with: + - A dictionary where the keys are step names and values are their associated queues. + - The number of queues that received tasks + """ + spec = get_spec_with_expansion(spec_file) + queues_in_spec = spec.get_task_queues() + + for queue in queues_in_spec.values(): + celery_task_manager.send_task(f"task_for_{queue}", queue=queue) + + return queues_in_spec, len(queues_in_spec.values()) + + def run_purge( + self, + spec_file: str, + input_value: str = None, + force: bool = False, + steps_to_purge: List[str] = None, + ) -> Dict[str, Union[str, int]]: + """ + Helper method to run the purge command. + + Args: + spec_file: The path to the spec file from which queues will be purged. + input_value: Any input we need to send to the subprocess. + force: If True, add the `-f` option to the purge command. + steps_to_purge: An optional list of steps to send to the purge command. + + Returns: + The result from executing the command in a subprocess. + """ + purge_cmd = ( + "merlin purge" + + (" -f" if force else "") + + f" {spec_file}" + + (f" --steps {' '.join(steps_to_purge)}" if steps_to_purge is not None else "") + ) + result = subprocess.run(purge_cmd, shell=True, capture_output=True, text=True, input=input_value) + return { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + def check_queues( + self, + redis_client: FixtureRedis, + queues_in_spec: Dict[str, str], + expected_task_count: int, + steps_to_purge: List[str] = None, + ): + """ + Check the state of queues in Redis against expected task counts. + + When `steps_to_purge` is set, the `expected_task_count` will represent the + number of expected tasks in the queues that _are not_ associated with the + steps in the `steps_to_purge` list. + + Args: + redis_client: The Redis client instance. + queues_in_spec: A dictionary of queues to check. + expected_task_count: The expected number of tasks in the queues (0 or 1). + steps_to_purge: Optional list of steps to determine which queues should be purged. + """ + for queue in queues_in_spec.values(): + # Brackets are special chars in regex so we have to add \ to make them literal + queue = queue.replace("[", "\\[").replace("]", "\\]") + matching_queues_on_server = redis_client.keys(pattern=f"{queue}*") + + for matching_queue in matching_queues_on_server: + tasks = redis_client.lrange(matching_queue, 0, -1) + if steps_to_purge and matching_queue in [queues_in_spec[step] for step in steps_to_purge]: + assert len(tasks) == 0, f"Expected 0 tasks in {matching_queue}, found {len(tasks)}." + else: + assert ( + len(tasks) == expected_task_count + ), f"Expected {expected_task_count} tasks in {matching_queue}, found {len(tasks)}." + + def test_no_options_tasks_exist_y( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with no options added and + tasks sent to the server. This should come up with a y/N + prompt in which we type 'y'. This should then purge the + tasks from the server. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) + + # Run the purge test + test_info = self.run_purge(feature_demo, input_value="y") + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"Purged {num_queues} messages from {num_queues} known task queues."), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were purged + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) + + def test_no_options_no_tasks_y( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with no options added and + no tasks sent to the server. This should come up with a y/N + prompt in which we type 'y'. This should then give us a "No + messages purged" log. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): + # Get the queues from the spec file + spec = get_spec_with_expansion(feature_demo) + queues_in_spec = spec.get_task_queues() + num_queues = len(queues_in_spec.values()) + + # Check that there are no tasks in the queues before we run the purge command + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) + + # Run the purge test + test_info = self.run_purge(feature_demo, input_value="y") + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"No messages purged from {num_queues} queues."), + ] + check_test_conditions(conditions, test_info) + + # Check that the Redis server still has no tasks + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) + + def test_no_options_n( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with no options added and + tasks sent to the server. This should come up with a y/N + prompt in which we type 'N'. This should take us out of the + command without purging the tasks. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) + + # Run the purge test + test_info = self.run_purge(feature_demo, input_value="N") + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"Purged {num_queues} messages from {num_queues} known task queues.", negate=True), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were not purged + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=1) + + def test_force_option( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with the `--force` option + enabled. This should not bring up a y/N prompt and should + immediately purge all tasks. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) + + # Run the purge test + test_info = self.run_purge(feature_demo, force=True) + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?", negate=True), + HasRegex(f"Purged {num_queues} messages from {num_queues} known task queues."), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were purged + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) + + def test_steps_option( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with the `--steps` option + enabled. This should only purge the tasks in the task queues + associated with the steps provided. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: + # Send tasks to the server for every queue in the spec + queues_in_spec, _ = self.setup_tasks(celery_task_manager, feature_demo) + + # Run the purge test + steps_to_purge = ["hello", "collect"] + test_info = self.run_purge(feature_demo, input_value="y", steps_to_purge=steps_to_purge) + + # Make sure the subprocess ran and the correct output messages are given + num_steps_to_purge = len(steps_to_purge) + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"Purged {num_steps_to_purge} messages from {num_steps_to_purge} known task queues."), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were not purged + self.check_queues( + redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=1, steps_to_purge=steps_to_purge + ) diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py new file mode 100644 index 000000000..c7b27b3bb --- /dev/null +++ b/tests/integration/commands/test_run.py @@ -0,0 +1,468 @@ +""" +This module will contain the testing logic +for the `merlin run` command. +""" + +import csv +import os +import re +import shutil +import subprocess +from typing import Dict, Union + +from merlin.spec.expansion import get_spec_with_expansion +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import FixtureStr +from tests.integration.conditions import HasReturnCode, PathExists, StepFinishedFilesCount +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd + + +# pylint: disable=import-outside-toplevel,unused-argument + + +class TestRunCommand: + """ + Base class for testing the `merlin run` command. + """ + + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + + def setup_test_environment( + self, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr + ) -> str: + """ + Setup the test environment for these tests by: + 1. Moving into the temporary output directory created specifically for these tests. + 2. Copying the app.yaml file created by the `redis_server` fixture to the cwd so that + Merlin can connect to the test server. + 3. Obtaining the path to the feature_demo spec that we'll use for these tests. + + Args: + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + + Returns: + The path to the feature_demo spec file. + """ + os.chdir(run_command_testing_dir) + copy_app_yaml_to_cwd(merlin_server_dir) + return os.path.join(path_to_merlin_codebase, self.demo_workflow) + + def run_merlin_command(self, command: str) -> Dict[str, Union[str, int]]: + """ + Open a subprocess and run the command specified by the `command` parameter. + Ensure this command runs successfully and return the process results. + + Args: + command: The command to execute in a subprocess. + + Returns: + The results from executing the command in a subprocess. + + Raises: + AssertionError: If the command fails (non-zero return code). + """ + result = subprocess.run(command, shell=True, capture_output=True, text=True) + return { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + def get_output_workspace_from_logs(self, test_info: Dict[str, Union[str, int]]) -> str: + """ + Extracts the workspace path from the provided standard output and error logs. + + This method searches for a specific message indicating the study workspace + in the combined logs (both stdout and stderr). The expected message format + is: "Study workspace is ''". If the message is found, + the method returns the extracted workspace path. If the message is not + found, an assertion error is raised. + + Args: + test_info: The results from executing our test. + + Returns: + The extracted workspace path from the logs. + + Raises: + AssertionError: If the expected message is not found in the combined logs. + """ + workspace_pattern = re.compile(r"Study workspace is '(\S+)'") + combined_output = test_info["stdout"] + test_info["stderr"] + match = workspace_pattern.search(combined_output) + assert match, "No 'Study workspace is...' message found in command output." + return match.group(1) + + +class TestRunCommandDistributed(TestRunCommand): + """ + Tests for the `merlin run` command that are run in a distributed manner + rather than being run locally. + """ + + def test_distributed_run( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, + ): + """ + This test verifies that tasks can be successfully sent to a Redis server + using the `merlin run` command with no flags. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + from merlin.celery import app as celery_app + + # Setup the testing environment + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): + # Send tasks to the server + test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_distributed_run") + + # Check that the test ran properly + check_test_conditions([HasReturnCode()], test_info) + + # Get the queues we need to query + spec = get_spec_with_expansion(feature_demo) + queues_in_spec = spec.get_task_queues() + + for queue in queues_in_spec.values(): + # Brackets are special chars in regex so we have to add \ to make them literal + queue = queue.replace("[", "\\[").replace("]", "\\]") + matching_queues_on_server = redis_broker_and_backend_function.client.keys(pattern=f"{queue}*") + + # Make sure any queues that exist on the server have tasks in them + for matching_queue in matching_queues_on_server: + tasks = redis_broker_and_backend_function.client.lrange(matching_queue, 0, -1) + assert len(tasks) > 0 + + def test_samplesfile_option( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, + ): + """ + This test verifies that passing in a samples filepath from the command line will + substitute in the file properly. It should copy the samples file that's passed + in to the merlin_info subdirectory. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + from merlin.celery import app as celery_app + + # Setup the testing environment + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) + + # Create a new samples file to pass into our test workflow + data = [ + ["X1, Value 1", "X2, Value 1"], + ["X1, Value 2", "X2, Value 2"], + ["X1, Value 3", "X2, Value 3"], + ] + sample_filename = "test_samplesfile.csv" + new_samples_file = os.path.join(run_command_testing_dir, sample_filename) + with open(new_samples_file, mode="w", newline="") as file: + writer = csv.writer(file) + writer.writerows(data) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): + # Send tasks to the server + test_info = self.run_merlin_command( + f"merlin run {feature_demo} --vars NAME=run_command_test_samplesfile_option --samplesfile {new_samples_file}" + ) + + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + PathExists(os.path.join(expected_workspace_path, "merlin_info", sample_filename)), + ] + check_test_conditions(conditions, test_info) + + def test_pgen_and_pargs_options( # pylint: disable=too-many-locals + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, + ): + """ + Test the `--pgen` and `--pargs` options with the `merlin run` command. + This should update the parameter block of the expanded yaml file to have + 2 entries for both `X2` and `N_NEW`. The `X2` parameter should be between + `X2_MIN` and `X2_MAX`, and the `N_NEW` parameter should be between `N_NEW_MIN` + and `N_NEW_MAX`. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + from merlin.celery import app as celery_app + + # Setup test vars and the testing environment + bounds = {"X2": (1, 2), "N_NEW": (5, 15)} + pgen_filepath = os.path.join( + os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))), "pgen.py" + ) + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): + # Send tasks to the server + test_info = self.run_merlin_command( + f"merlin run {feature_demo} " + "--vars NAME=run_command_test_pgen_and_pargs_options " + f"--pgen {pgen_filepath} " + f'--parg "X2_MIN:{bounds["X2"][0]}" ' + f'--parg "X2_MAX:{bounds["X2"][1]}" ' + f'--parg "N_NAME_MIN:{bounds["N_NEW"][0]}" ' + f'--parg "N_NAME_MAX:{bounds["N_NEW"][1]}"' + ) + + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + expanded_yaml = os.path.join(expected_workspace_path, "merlin_info", "feature_demo.expanded.yaml") + conditions = [HasReturnCode(), PathExists(expected_workspace_path), PathExists(os.path.join(expanded_yaml))] + check_test_conditions(conditions, test_info) + + # Read in the parameters from the expanded yaml and ensure they're within the new bounds we provided + params = get_spec_with_expansion(expanded_yaml).get_parameters() + for param_name, (min_val, max_val) in bounds.items(): + for param in params.parameters[param_name]: + assert min_val <= param <= max_val + + +class TestRunCommandLocal(TestRunCommand): + """ + Tests for the `merlin run` command that are run in a locally rather + than in a distributed manner. + """ + + def test_dry_run( # pylint: disable=too-many-locals + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, + ): + """ + Test the `merlin run` command's `--dry` option. This should create all the output + subdirectories for each step but it shouldn't execute anything for the steps. In + other words, the only file in each step subdirectory should be the .sh file. + + Note: + This test will run locally so that we don't have to worry about starting + & stopping workers. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + # Setup the test environment + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) + + # Run the test and grab the output workspace generated from it + test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_dry_run --local --dry") + + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + check_test_conditions([HasReturnCode(), PathExists(expected_workspace_path)], test_info) + + # Check that every step was ran by looking for an existing output workspace + for step in get_spec_with_expansion(feature_demo).get_study_steps(): + step_directory = os.path.join(expected_workspace_path, step.name) + assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" + + allowed_dry_run_files = {"MERLIN_STATUS.json", "status.lock"} + for dirpath, dirnames, filenames in os.walk(step_directory): + # Check if the current directory has no subdirectories (leaf directory) + if not dirnames: + # Check for unexpected files + unexpected_files = [ + file for file in filenames if file not in allowed_dry_run_files and not file.endswith(".sh") + ] + assert not unexpected_files, ( + f"Unexpected files found in {dirpath}: {unexpected_files}. " + f"Expected only .sh files or {allowed_dry_run_files}." + ) + + # Check that there is exactly one .sh file + sh_file_count = sum(1 for file in filenames if file.endswith(".sh")) + assert ( + sh_file_count == 1 + ), f"Expected exactly one .sh file in {dirpath} but found {sh_file_count} .sh files." + + def test_local_run( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, + ): + """ + This test verifies that tasks can be successfully executed locally using + the `merlin run` command with the `--local` flag. + + Args: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + # Setup the test environment + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) + + # Run the test and grab the output workspace generated from it + study_name = "run_command_test_local_run" + num_samples = 8 + vars_dict = {"NAME": study_name, "OUTPUT_PATH": run_command_testing_dir, "N_SAMPLES": num_samples} + vars_str = " ".join(f"{key}={value}" for key, value in vars_dict.items()) + command = f"merlin run {feature_demo} --vars {vars_str} --local" + test_info = self.run_merlin_command(command) + + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + StepFinishedFilesCount( # The rest of the conditions will ensure every step ran to completion + step="hello", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=num_samples, + ), + StepFinishedFilesCount( + step="python3_hello", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="collect", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="translate", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="learn", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="make_new_samples", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="predict", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="verify", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + ] + + # GitHub actions doesn't have a python2 path so we'll conditionally add this check + if shutil.which("python2"): + conditions.append( + StepFinishedFilesCount( + step="python2_hello", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ) + ) + + check_test_conditions(conditions, test_info) + + # # Check that every step was ran by looking for an existing output workspace and MERLIN_FINISHED files + # for step in get_spec_with_expansion(feature_demo).get_study_steps(): + # step_directory = os.path.join(expected_workspace_path, step.name) + # assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" + # for dirpath, dirnames, filenames in os.walk(step_directory): + # # Check if the current directory has no subdirectories (leaf directory) + # if not dirnames: + # # Check for the existence of the MERLIN_FINISHED file + # assert ( + # "MERLIN_FINISHED" in filenames + # ), f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" + + +# pylint: enable=import-outside-toplevel,unused-argument diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py new file mode 100644 index 000000000..beed6599b --- /dev/null +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -0,0 +1,380 @@ +""" +This module will contain the testing logic for +the `stop-workers` and `query-workers` commands. +""" + +import os +import subprocess +from contextlib import contextmanager +from enum import Enum +from typing import List + +import pytest + +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import FixtureStr +from tests.integration.conditions import Condition, HasRegex +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec + + +# pylint: disable=unused-argument,import-outside-toplevel + + +class WorkerMessages(Enum): + """ + Enumerated strings to help keep track of the messages + that we're expecting (or not expecting) to see from the + tests in this module. + """ + + NO_WORKERS_MSG_STOP = "No workers found to stop" + NO_WORKERS_MSG_QUERY = "No workers found!" + STEP_1_WORKER = "step_1_merlin_test_worker" + STEP_2_WORKER = "step_2_merlin_test_worker" + OTHER_WORKER = "other_merlin_test_worker" + + +class TestStopAndQueryWorkersCommands: + """ + Tests for the `merlin stop-workers` and `merlin query-workers` commands. + Most of these tests will: + 1. Start workers from a spec file used for testing + - Use CeleryWorkerManager for this to ensure safe stoppage of workers + if something goes wrong + 2. Run the test command from a subprocess + """ + + @contextmanager + def run_test_with_workers( # pylint: disable=too-many-arguments + self, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, + conditions: List[Condition], + command: str, + flag: str = None, + ): + """ + Helper method to run common testing logic for tests with workers started. + This method must also be a context manager so we can check the status of the + workers prior to the CeleryWorkersManager running it's exit code that shuts down + all active workers. + + This method will: + 0. Read in the necessary fixtures as parameters. These fixtures grab paths to + our test specs and the merlin server directory created from starting the + containerized redis server. + 1. Load in the worker specifications from the `multiple_workers.yaml` file. + 2. Use a context manager to start up the workers on the celery app connected to + the containerized redis server + 3. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 4. Run the test command that's provided and check that the conditions given are + passing. + 5. Yield control back to the calling method. + 6. Safely terminate workers that may have not been stopped once the calling method + completes. + + Parameters: + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + conditions: + A list of `Condition` instances that need to pass in order for this test to + be successful. + command: + The command that we're testing. E.g. "merlin stop-workers" + flag: + An optional flag to add to the command that we're testing so we can test + different functionality for the command. + """ + from merlin.celery import app as celery_app + + # Grab worker configurations from the spec file + multiple_worker_spec = os.path.join(path_to_test_specs, "multiple_workers.yaml") + workers_from_spec = load_workers_from_spec(multiple_worker_spec) + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as workers_manager: + workers_manager.launch_workers(workers_from_spec) + + # Copy the app.yaml to the cwd so merlin will connect to the testing server + copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + cmd_to_test = f"{command} {flag}" if flag else command + result = subprocess.run(cmd_to_test, capture_output=True, text=True, shell=True) + + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + check_test_conditions(conditions, info) + + yield + + def get_no_workers_msg(self, command_to_test: str) -> WorkerMessages: + """ + Retrieve the appropriate "no workers" found message. + + This method checks the command to test and returns a corresponding + message based on whether the command is to stop workers or query for them. + + Returns: + The message indicating that no workers are available, depending on the + command being tested. + """ + no_workers_msg = None + if command_to_test == "merlin stop-workers": + no_workers_msg = WorkerMessages.NO_WORKERS_MSG_STOP.value + else: + no_workers_msg = WorkerMessages.NO_WORKERS_MSG_QUERY.value + return no_workers_msg + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_no_workers( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + merlin_server_dir: FixtureStr, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with no workers + started in the first place. + + This test will: + 0. Setup the pytest fixtures which include: + - starting a containerized Redis server + - updating the CONFIG object to point to the containerized Redis server + - obtaining the path to the merlin server directory created from starting + the containerized Redis server + 1. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 2. Run the test command that's provided and check that the conditions given are + passing. + + Parameters: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test)), + HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), + ] + + # Copy the app.yaml to the cwd so merlin will connect to the testing server + copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + result = subprocess.run(command_to_test, capture_output=True, text=True, shell=True) + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + check_test_conditions(conditions, info) + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_no_flags( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with no flags. + + Run the commands referenced above and ensure the text output from Merlin is correct. + For the `stop-workers` command, we check if all workers are stopped as well. + To see more information on exactly what this test is doing, see the + `run_test_with_workers()` method. + + Parameters: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value), + HasRegex(WorkerMessages.OTHER_WORKER.value), + ] + with self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, command_to_test): + if command_to_test == "merlin stop-workers": + # After the test runs and before the CeleryWorkersManager exits, ensure there are no workers on the app + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + assert active_queues is None + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_spec_flag( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with the `--spec` + flag. + + Run the commands referenced above with the `--spec` flag and ensure the text output + from Merlin is correct. For the `stop-workers` command, we check if all workers defined + in the spec file are stopped as well. To see more information on exactly what this test + is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value), + HasRegex(WorkerMessages.OTHER_WORKER.value), + ] + with self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + command_to_test, + flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", + ): + if command_to_test == "merlin stop-workers": + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + assert active_queues is None + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_workers_flag( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with the `--workers` + flag. + + Run the commands referenced above with the `--workers` flag and ensure the text output + from Merlin is correct. For the `stop-workers` command, we check to make sure that all + workers given with this flag are stopped. To see more information on exactly what this + test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value), + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), + ] + with self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + command_to_test, + flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", + ): + if command_to_test == "merlin stop-workers": + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + worker_name = f"celery@{WorkerMessages.OTHER_WORKER.value}" + assert worker_name in active_queues + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_queues_flag( + self, + redis_broker_and_backend_function: RedisBrokerAndBackend, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with the `--queues` + flag. + + Run the commands referenced above with the `--queues` flag and ensure the text output + from Merlin is correct. For the `stop-workers` command, we check that only the workers + attached to the given queues are stopped. To see more information on exactly what this + test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), + ] + with self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + command_to_test, + flag="--queues hello_queue", + ): + if command_to_test == "merlin stop-workers": + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + workers_that_should_be_alive = [ + f"celery@{WorkerMessages.OTHER_WORKER.value}", + f"celery@{WorkerMessages.STEP_2_WORKER.value}", + ] + for worker_name in workers_that_should_be_alive: + assert worker_name in active_queues + + +# pylint: enable=unused-argument,import-outside-toplevel diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 1c852b064..2ae0aa02a 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -34,7 +34,6 @@ from re import search -# TODO when moving command line tests to pytest, change Condition boolean returns to assertions class Condition(ABC): """Abstract Condition class that other conditions will inherit from""" @@ -131,7 +130,7 @@ def __init__(self, study_name, output_path): """ self.study_name = study_name self.output_path = output_path - self.dirpath_glob = f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*" + self.dirpath_glob = os.path.join(self.output_path, f"{self.study_name}_[0-9]*-[0-9]*") def glob(self, glob_string): """ @@ -154,7 +153,7 @@ class StepFileExists(StudyOutputAware): A StudyOutputAware that checks for a particular file's existence. """ - def __init__(self, step, filename, study_name, output_path, params=False): # pylint: disable=R0913 + def __init__(self, step, filename, study_name, output_path, params=False, samples=False): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -165,6 +164,7 @@ def __init__(self, step, filename, study_name, output_path, params=False): # py self.step = step self.filename = filename self.params = params + self.samples = samples def __str__(self): return f"{__class__.__name__} expected to find file '{self.glob_string}', but file did not exist" @@ -174,10 +174,9 @@ def glob_string(self): """ Returns a regex string for the glob library to recursively find files with. """ - param_glob = "" - if self.params: - param_glob = "*/" - return f"{self.dirpath_glob}/{self.step}/{param_glob}{self.filename}" + param_glob = "*" if self.params else "" + samples_glob = "**" if self.samples else "" + return os.path.join(self.dirpath_glob, self.step, param_glob, samples_glob, self.filename) def file_exists(self): """Check if the file path created by glob_string exists""" @@ -229,7 +228,7 @@ def contains(self): with open(filename, "r") as textfile: filetext = textfile.read() return self.is_within(filetext) - except Exception: # pylint: disable=W0718 + except Exception: # pylint: disable=broad-except return False def is_within(self, text): @@ -243,6 +242,108 @@ def passes(self): return self.contains() +# TODO when writing API docs for tests make sure this looks correct and has functioning links +# - Do we want to list expected_count, glob_string, and passes as methods since they're already attributes? +class StepFinishedFilesCount(StudyOutputAware): + """ + A [`StudyOutputAware`][integration.conditions.StudyOutputAware] that checks for the + exact number of `MERLIN_FINISHED` files in a specified step's output directory based + on the number of parameters and samples. + + Attributes: + step: The name of the step to check. + study_name: The name of the study. + output_path: The output path of the study. + num_parameters: The number of parameters for the step. + num_samples: The number of samples for the step. + expected_count: The expected number of `MERLIN_FINISHED` files based on parameters and samples or explicitly set. + glob_string: The glob pattern to find `MERLIN_FINISHED` files in the specified step's output directory. + passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. + + Methods: + expected_count: Calculates the expected number of `MERLIN_FINISHED` files. + glob_string: Constructs the glob pattern for searching `MERLIN_FINISHED` files. + count_finished_files: Counts the number of `MERLIN_FINISHED` files found. + passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. + """ + + # All of these parameters are necessary for this Condition so we'll ignore pylint + def __init__( + self, + step: str, + study_name: str, + output_path: str, + num_parameters: int = 0, + num_samples: int = 0, + expected_count: int = None, + ): # pylint: disable=too-many-arguments + super().__init__(study_name, output_path) + self.step = step + self.num_parameters = num_parameters + self.num_samples = num_samples + self._expected_count = expected_count + + @property + def expected_count(self) -> int: + """ + Calculate the expected number of `MERLIN_FINISHED` files. + + Returns: + The expected number of `MERLIN_FINISHED` files. + """ + # Return the explicitly set expected count if given + if self._expected_count is not None: + return self._expected_count + + # Otherwise calculate the correct number of MERLIN_FINISHED files to expect + if self.num_parameters > 0 and self.num_samples > 0: + return self.num_parameters * self.num_samples + if self.num_parameters > 0: + return self.num_parameters + if self.num_samples > 0: + return self.num_samples + + return 1 # Default case when there are no parameters or samples + + @property + def glob_string(self) -> str: + """ + Glob pattern to find `MERLIN_FINISHED` files in the specified step's output directory. + + Returns: + A glob pattern to find `MERLIN_FINISHED` files. + """ + param_glob = "*" if self.num_parameters > 0 else "" + samples_glob = "**" if self.num_samples > 0 else "" + return os.path.join(self.dirpath_glob, self.step, param_glob, samples_glob, "MERLIN_FINISHED") + + def count_finished_files(self) -> int: + """ + Count the number of `MERLIN_FINISHED` files found. + + Returns: + The actual number of `MERLIN_FINISHED` files that exist in the step's output directory. + """ + finished_files = glob(self.glob_string) # Adjust the glob pattern as needed + return len(finished_files) + + @property + def passes(self) -> bool: + """ + Check if the count of `MERLIN_FINISHED` files matches the expected count. + + Returns: + True if the expected count matches the actual count. False otherwise. + """ + return self.count_finished_files() == self.expected_count + + def __str__(self) -> str: + return ( + f"{__class__.__name__} expected {self.expected_count} `MERLIN_FINISHED` " + f"files, but found {self.count_finished_files()}" + ) + + class ProvenanceYAMLFileHasRegex(HasRegex): """ A condition that a Merlin provenance yaml spec in the 'merlin_info' directory @@ -339,7 +440,7 @@ def contains(self) -> bool: with open(self.filename, "r") as f: # pylint: disable=C0103 filetext = f.read() return self.is_within(filetext) - except Exception: # pylint: disable=W0718 + except Exception: # pylint: disable=broad-except return False def is_within(self, text): diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 18435d461..4714275f5 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -108,16 +108,12 @@ def define_tests(): # pylint: disable=R0914,R0915 workers_lsf = get_worker_by_cmd("jsrun", workers) run = f"merlin {err_lvl} run" restart = f"merlin {err_lvl} restart" - purge = "merlin purge" - stop = "merlin stop-workers" - query = "merlin query-workers" # Shortcuts for example workflow paths examples = "merlin/examples/workflows" dev_examples = "merlin/examples/dev_workflows" test_specs = "tests/integration/test_specs" demo = f"{examples}/feature_demo/feature_demo.yaml" - remote_demo = f"{examples}/remote_feature_demo/remote_feature_demo.yaml" demo_pgen = f"{examples}/feature_demo/scripts/pgen.py" simple = f"{examples}/simple_chain/simple_chain.yaml" slurm = f"{test_specs}/slurm_test.yaml" @@ -126,9 +122,7 @@ def define_tests(): # pylint: disable=R0914,R0915 flux_restart = f"{examples}/flux/flux_par_restart.yaml" flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" - mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" cli_substitution_wf = f"{test_specs}/cli_substitution_test.yaml" - chord_err_wf = f"{test_specs}/chord_err.yaml" # Other shortcuts black = "black --check --target-version py36" @@ -650,213 +644,6 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } - stop_workers_tests = { - "stop workers no workers": { - "cmds": f"{stop}", - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop"), - HasRegex("step_1_merlin_test_worker", negate=True), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - }, - "stop workers no flags": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "stop workers with spec flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop} --spec {mul_workers_demo}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "stop workers with workers flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop} --workers step_1_merlin_test_worker step_2_merlin_test_worker", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "stop workers with queues flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop} --queues hello_queue", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - } - query_workers_tests = { - "query workers no workers": { - "cmds": f"{query}", - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!"), - HasRegex("step_1_merlin_test_worker", negate=True), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - }, - "query workers no flags": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "query workers with spec flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query} --spec {mul_workers_demo}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "query workers with workers flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query} --workers step_1_merlin_test_worker step_2_merlin_test_worker", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "query workers with queues flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query} --queues hello_queue", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - } - distributed_tests = { # noqa: F841 - "run and purge feature_demo": { - "cmds": f"{run} {demo}; {purge} {demo} -f", - "conditions": HasReturnCode(), - "run type": "distributed", - }, - "remote feature_demo": { - "cmds": f"""{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers; - {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers""", - "conditions": [ - HasReturnCode(), - ProvenanceYAMLFileHasRegex( - regex="cli_test_demo_workers:", - spec_file_name="remote_feature_demo", - study_name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - ), - StepFileExists( - "verify", - "MERLIN_FINISHED", - "feature_demo", - OUTPUT_DIR, - params=True, - ), - ], - "run type": "distributed", - }, - } - distributed_error_checks = { - "check chord error continues wf": { - "cmds": [ - f"{workers} {chord_err_wf} --vars OUTPUT_PATH=./{OUTPUT_DIR}", - f"{run} {chord_err_wf} --vars OUTPUT_PATH=./{OUTPUT_DIR}; sleep 40; tree {OUTPUT_DIR}", - ], - "conditions": [ - HasReturnCode(), - PathExists( # Check that the sample that's supposed to raise an error actually raises an error - f"{OUTPUT_DIR}/process_samples/01/MERLIN_FINISHED", - negate=True, - ), - StepFileExists( # Check that step 3 is actually started and completes - "step_3", - "MERLIN_FINISHED", - "chord_err", - OUTPUT_DIR, - ), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - } - } # combine and return test dictionaries all_tests = {} @@ -876,10 +663,6 @@ def define_tests(): # pylint: disable=R0914,R0915 # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, - stop_workers_tests, - query_workers_tests, - distributed_tests, - distributed_error_checks, ]: all_tests.update(test_dict) diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py new file mode 100644 index 000000000..4837b516b --- /dev/null +++ b/tests/integration/helper_funcs.py @@ -0,0 +1,164 @@ +""" +This module contains helper functions for the integration +test suite. +""" + +import os +import re +import shutil +import subprocess +from time import sleep +from typing import Dict, List + +from merlin.spec.expansion import get_spec_with_expansion +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.fixture_types import FixtureRedis +from tests.integration.conditions import Condition + + +def load_workers_from_spec(spec_filepath: str) -> dict: + """ + Load worker specifications from a YAML file. + + This function reads a YAML file containing study specifications and + extracts the worker information under the "merlin" section. It + constructs a dictionary in the form that + [`CeleryWorkersManager.launch_workers`][context_managers.celery_workers_manager.CeleryWorkersManager.launch_workers] + requires. + + Args: + spec_filepath: The file path to the YAML specification file. + + Returns: + A dictionary containing the worker specifications from the + "merlin" section of the YAML file. + """ + worker_info = {} + spec = get_spec_with_expansion(spec_filepath) + steps_and_queues = spec.get_task_queues(omit_tag=True) + + for worker_name, worker_settings in spec.merlin["resources"]["workers"].items(): + match = re.search(r"--concurrency\s+(\d+)", worker_settings["args"]) + concurrency = int(match.group(1)) if match else 1 + worker_info[worker_name] = {"concurrency": concurrency} + if worker_settings["steps"] == ["all"]: + worker_info[worker_name]["queues"] = list(steps_and_queues.values()) + else: + worker_info[worker_name]["queues"] = [steps_and_queues[step] for step in worker_settings["steps"]] + + return worker_info + + +def copy_app_yaml_to_cwd(merlin_server_dir: str): + """ + Copy the app.yaml file from the directory provided to the current working + directory. + + Grab the app.yaml file from `merlin_server_dir` and copy it to the current + working directory so that Merlin will read this in as the server configuration + for whatever test is calling this. + + Args: + merlin_server_dir: The path to the `merlin_server` directory that should be created by the + [`redis_server`][conftest.redis_server] fixture. + """ + copied_app_yaml = os.path.join(os.getcwd(), "app.yaml") + if not os.path.exists(copied_app_yaml): + server_app_yaml = os.path.join(merlin_server_dir, "app.yaml") + shutil.copy(server_app_yaml, copied_app_yaml) + + +def check_test_conditions(conditions: List[Condition], info: Dict[str, str]): + """ + Ensure all specified test conditions are satisfied based on the output + from a subprocess. + + This function iterates through a list of [`Condition`][integration.conditions.Condition] + instances, ingests the provided information (stdout, stderr, and return + code) for each condition, and checks if each condition passes. If any + condition fails, an AssertionError is raised with a detailed message that + includes the condition that failed, along with the captured output and + return code. + + Args: + conditions: A list of Condition instances that define the expectations for the test. + info: A dictionary containing the output from the subprocess, which should + include the following keys:\n + - 'stdout': The standard output captured from the subprocess. + - 'stderr': The standard error output captured from the subprocess. + - 'return_code': The return code of the subprocess, indicating success + or failure of the command executed. + + Raises: + AssertionError: If any of the conditions do not pass, an AssertionError is raised with + a detailed message including the failed condition and the subprocess + output. + """ + for condition in conditions: + condition.ingest_info(info) + try: + assert condition.passes + except AssertionError as exc: + error_message = ( + f"Condition failed: {condition}\n" + f"Captured stdout: {info['stdout']}\n" + f"Captured stderr: {info['stderr']}\n" + f"Return code: {info['return_code']}\n" + ) + raise AssertionError(error_message) from exc + + +def run_workflow(redis_client: FixtureRedis, workflow_path: str, vars_to_substitute: List[str]) -> subprocess.CompletedProcess: + """ + Run a Merlin workflow using the `merlin run` and `merlin run-workers` commands. + + This function executes a Merlin workflow using a specified path to a study and variables to + configure the study with. It utilizes context managers to safely send tasks to the server + and start up workers. The tasks are given 15 seconds to be sent to the server. Once tasks + exist on the server, the workflow is given 30 seconds to run to completion, which should be + plenty of time. + + Args: + redis_client: A fixture that connects us to a redis client that we can interact with. + workflow_path: The path to the study that we're going to run here + vars_to_substitute: A list of variables in the form ["VAR_NAME=var_value"] to be modified + in the workflow. + + Returns: + The completed process object containing information about the execution of the workflow, including + return code, stdout, and stderr. + """ + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel + + run_workers_proc = None + + with CeleryTaskManager(celery_app, redis_client): + # Send the tasks to the server + try: + subprocess.run( + f"merlin run {workflow_path} --vars {' '.join(vars_to_substitute)}", + shell=True, + capture_output=True, + text=True, + timeout=15, + ) + except subprocess.TimeoutExpired as exc: + raise TimeoutError("Could not send tasks to the server within the allotted time.") from exc + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as celery_worker_manager: + # Start the workers then add them to the context manager so they can be stopped safely later + run_workers_proc = subprocess.Popen( # pylint: disable=consider-using-with + f"merlin run-workers {workflow_path}".split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + celery_worker_manager.add_run_workers_process(run_workers_proc.pid) + + # Let the workflow try to run for 30 seconds + sleep(30) + + return run_workers_proc diff --git a/tests/integration/test_celeryadapter.py b/tests/integration/test_celeryadapter.py index a40f1054b..89241088b 100644 --- a/tests/integration/test_celeryadapter.py +++ b/tests/integration/test_celeryadapter.py @@ -34,12 +34,9 @@ import json import os from datetime import datetime -from time import sleep from typing import Dict -import pytest from celery import Celery -from celery.canvas import Signature from deepdiff import DeepDiff from merlin.config import Config @@ -49,150 +46,153 @@ from tests.unit.study.status_test_files.status_test_variables import SPEC_PATH -@pytest.mark.order(before="TestInactive") -class TestActive: - """ - This class will test functions in the celeryadapter.py module. - It will run tests where we need active queues/workers to interact with. - - NOTE: The tests in this class must be ran before the TestInactive class or else the - Celery workers needed for this class don't start - - TODO: fix the bug noted above and then check if we still need pytest-order - """ - - def test_query_celery_queues( - self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 - ): - """ - Test the query_celery_queues function by providing it with a list of active queues. - This should return a dict where keys are queue names and values are more dicts containing - the number of jobs and consumers in that queue. - - :param `celery_app`: A pytest fixture for the test Celery app - :param launch_workers: A pytest fixture that launches celery workers for us to interact with - :param worker_queue_map: A pytest fixture that returns a dict of workers and queues - """ - # Set up a dummy configuration to use in the test - dummy_config = Config({"broker": {"name": "redis"}}) - - # Get the actual output - queues_to_query = list(worker_queue_map.values()) - actual_queue_info = celeryadapter.query_celery_queues(queues_to_query, app=celery_app, config=dummy_config) - - # Ensure all 3 queues in worker_queue_map were queried before looping - assert len(actual_queue_info) == 3 - - # Ensure each queue has a worker attached - for queue_name, queue_info in actual_queue_info.items(): - assert queue_name in worker_queue_map.values() - assert queue_info == {"consumers": 1, "jobs": 0} - - def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 - """ - Test the get_running_queues function with queues active. - This should return a list of active queues. - - :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with - :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues - """ - result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) - assert sorted(result) == sorted(list(worker_queue_map.values())) - - def test_get_active_celery_queues( - self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 - ): - """ - Test the get_active_celery_queues function with queues active. - This should return a tuple where the first entry is a dict of queue info - and the second entry is a list of worker names. - - :param `celery_app`: A pytest fixture for the test Celery app - :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with - :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues - """ - # Start the queues and run the test - queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) - - # Ensure we got output before looping - assert len(queue_result) == len(worker_result) == 3 - - for worker, queue in worker_queue_map.items(): - # Check that the entry in the queue_result dict for this queue is correct - assert queue in queue_result - assert len(queue_result[queue]) == 1 - assert worker in queue_result[queue][0] - - # Remove this entry from the queue_result dict - del queue_result[queue] - - # Check that this worker was added to the worker_result list - worker_found = False - for worker_name in worker_result[:]: - if worker in worker_name: - worker_found = True - worker_result.remove(worker_name) - break - assert worker_found - - # Ensure there was no extra output that we weren't expecting - assert queue_result == {} - assert worker_result == [] - - def test_build_set_of_queues( - self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 - ): - """ - Test the build_set_of_queues function with queues active. - This should return a set of queues (the queues defined in setUp). - """ - # Run the test - result = celeryadapter.build_set_of_queues( - steps=["all"], spec=None, specific_queues=None, verbose=False, app=celery_app - ) - assert result == set(worker_queue_map.values()) - - @pytest.mark.order(index=1) - def test_check_celery_workers_processing_tasks( - self, - celery_app: Celery, - sleep_sig: Signature, - launch_workers: "Fixture", # noqa: F821 - ): - """ - Test the check_celery_workers_processing function with workers active and a task in a queue. - This function will query workers for any tasks they're still processing. We'll send a - a task that sleeps for 3 seconds to our workers before we run this test so that there should be - a task for this function to find. - - NOTE: the celery app fixture shows strange behavior when using app.control.inspect() calls (which - check_celery_workers_processing uses) so we have to run this test first in this class in order to - have it run properly. - - :param celery_app: A pytest fixture for the test Celery app - :param sleep_sig: A pytest fixture for a celery signature of a task that sleeps for 3 sec - :param launch_workers: A pytest fixture that launches celery workers for us to interact with - """ - # Our active workers/queues are test_worker_[0-2]/test_queue_[0-2] so we're - # sending this to test_queue_0 for test_worker_0 to process - queue_for_signature = "test_queue_0" - sleep_sig.set(queue=queue_for_signature) - result = sleep_sig.delay() - - # We need to give the task we just sent to the server a second to get picked up by the worker - sleep(1) - - # Run the test now that the task should be getting processed - active_queue_test = celeryadapter.check_celery_workers_processing([queue_for_signature], celery_app) - assert active_queue_test is True - - # Now test that a queue without any tasks returns false - # We sent the signature to task_queue_0 so task_queue_1 shouldn't have any tasks to find - non_active_queue_test = celeryadapter.check_celery_workers_processing(["test_queue_1"], celery_app) - assert non_active_queue_test is False - - # Wait for the worker to finish running the task - result.get() +# from time import sleep +# import pytest +# from celery.canvas import Signature +# @pytest.mark.order(before="TestInactive") +# class TestActive: +# """ +# This class will test functions in the celeryadapter.py module. +# It will run tests where we need active queues/workers to interact with. + +# NOTE: The tests in this class must be ran before the TestInactive class or else the +# Celery workers needed for this class don't start + +# TODO: fix the bug noted above and then check if we still need pytest-order +# """ + +# def test_query_celery_queues( +# self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 +# ): +# """ +# Test the query_celery_queues function by providing it with a list of active queues. +# This should return a dict where keys are queue names and values are more dicts containing +# the number of jobs and consumers in that queue. + +# :param `celery_app`: A pytest fixture for the test Celery app +# :param launch_workers: A pytest fixture that launches celery workers for us to interact with +# :param worker_queue_map: A pytest fixture that returns a dict of workers and queues +# """ +# # Set up a dummy configuration to use in the test +# dummy_config = Config({"broker": {"name": "redis"}}) + +# # Get the actual output +# queues_to_query = list(worker_queue_map.values()) +# actual_queue_info = celeryadapter.query_celery_queues(queues_to_query, app=celery_app, config=dummy_config) + +# # Ensure all 3 queues in worker_queue_map were queried before looping +# assert len(actual_queue_info) == 3 + +# # Ensure each queue has a worker attached +# for queue_name, queue_info in actual_queue_info.items(): +# assert queue_name in worker_queue_map.values() +# assert queue_info == {"consumers": 1, "jobs": 0} + +# def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 +# """ +# Test the get_running_queues function with queues active. +# This should return a list of active queues. + +# :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with +# :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues +# """ +# result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) +# assert sorted(result) == sorted(list(worker_queue_map.values())) + +# def test_get_active_celery_queues( +# self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 +# ): +# """ +# Test the get_active_celery_queues function with queues active. +# This should return a tuple where the first entry is a dict of queue info +# and the second entry is a list of worker names. + +# :param `celery_app`: A pytest fixture for the test Celery app +# :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with +# :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues +# """ +# # Start the queues and run the test +# queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) + +# # Ensure we got output before looping +# assert len(queue_result) == len(worker_result) == 3 + +# for worker, queue in worker_queue_map.items(): +# # Check that the entry in the queue_result dict for this queue is correct +# assert queue in queue_result +# assert len(queue_result[queue]) == 1 +# assert worker in queue_result[queue][0] + +# # Remove this entry from the queue_result dict +# del queue_result[queue] + +# # Check that this worker was added to the worker_result list +# worker_found = False +# for worker_name in worker_result[:]: +# if worker in worker_name: +# worker_found = True +# worker_result.remove(worker_name) +# break +# assert worker_found + +# # Ensure there was no extra output that we weren't expecting +# assert queue_result == {} +# assert worker_result == [] + +# def test_build_set_of_queues( +# self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 +# ): +# """ +# Test the build_set_of_queues function with queues active. +# This should return a set of queues (the queues defined in setUp). +# """ +# # Run the test +# result = celeryadapter.build_set_of_queues( +# steps=["all"], spec=None, specific_queues=None, verbose=False, app=celery_app +# ) +# assert result == set(worker_queue_map.values()) + +# @pytest.mark.order(index=1) +# def test_check_celery_workers_processing_tasks( +# self, +# celery_app: Celery, +# sleep_sig: Signature, +# launch_workers: "Fixture", # noqa: F821 +# ): +# """ +# Test the check_celery_workers_processing function with workers active and a task in a queue. +# This function will query workers for any tasks they're still processing. We'll send a +# a task that sleeps for 3 seconds to our workers before we run this test so that there should be +# a task for this function to find. + +# NOTE: the celery app fixture shows strange behavior when using app.control.inspect() calls (which +# check_celery_workers_processing uses) so we have to run this test first in this class in order to +# have it run properly. + +# :param celery_app: A pytest fixture for the test Celery app +# :param sleep_sig: A pytest fixture for a celery signature of a task that sleeps for 3 sec +# :param launch_workers: A pytest fixture that launches celery workers for us to interact with +# """ +# # Our active workers/queues are test_worker_[0-2]/test_queue_[0-2] so we're +# # sending this to test_queue_0 for test_worker_0 to process +# queue_for_signature = "test_queue_0" +# sleep_sig.set(queue=queue_for_signature) +# result = sleep_sig.delay() + +# # We need to give the task we just sent to the server a second to get picked up by the worker +# sleep(1) + +# # Run the test now that the task should be getting processed +# active_queue_test = celeryadapter.check_celery_workers_processing([queue_for_signature], celery_app) +# assert active_queue_test is True + +# # Now test that a queue without any tasks returns false +# # We sent the signature to task_queue_0 so task_queue_1 shouldn't have any tasks to find +# non_active_queue_test = celeryadapter.check_celery_workers_processing(["test_queue_1"], celery_app) +# assert non_active_queue_test is False + +# # Wait for the worker to finish running the task +# result.get() class TestInactive: @@ -250,7 +250,7 @@ def test_get_running_queues(self): This should return an empty list. """ result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) - assert result == [] + assert not result def test_get_active_celery_queues(self, celery_app: Celery): """ @@ -261,8 +261,8 @@ def test_get_active_celery_queues(self, celery_app: Celery): :param `celery_app`: A pytest fixture for the test Celery app """ queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) - assert queue_result == {} - assert worker_result == [] + assert not queue_result + assert not worker_result def test_check_celery_workers_processing_tasks(self, celery_app: Celery, worker_queue_map: Dict[str, str]): """ @@ -476,7 +476,7 @@ def test_dump_celery_queue_info_csv(self, worker_queue_map: Dict[str, str]): # Make sure the rest of the csv file was created as expected dump_diff = DeepDiff(csv_dump_output, expected_output) - assert dump_diff == {} + assert not dump_diff finally: try: os.remove(outfile) @@ -513,7 +513,7 @@ def test_dump_celery_queue_info_json(self, worker_queue_map: Dict[str, str]): # There should only be one entry in the json dump file so this will only 'loop' once for dump_entry in json_df_contents.values(): json_dump_diff = DeepDiff(dump_entry, expected_output) - assert json_dump_diff == {} + assert not json_dump_diff finally: try: os.remove(outfile) diff --git a/tests/integration/test_specs/chord_err.yaml b/tests/integration/test_specs/chord_err.yaml index 3da99ae03..9fe7d55ea 100644 --- a/tests/integration/test_specs/chord_err.yaml +++ b/tests/integration/test_specs/chord_err.yaml @@ -1,10 +1,11 @@ description: - name: chord_err + name: $(NAME) description: test the chord err problem env: variables: OUTPUT_PATH: ./studies + NAME: chord_err global.parameters: TEST_PARAM: diff --git a/tests/integration/test_specs/multiple_workers.yaml b/tests/integration/test_specs/multiple_workers.yaml new file mode 100644 index 000000000..967582a53 --- /dev/null +++ b/tests/integration/test_specs/multiple_workers.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: other_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO --concurrency 1 + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO --concurrency 1 + steps: [step_2] + other_merlin_test_worker: + args: -l INFO --concurrency 1 + steps: [step_3, step_4] diff --git a/tests/integration/workflows/test_chord_error.py b/tests/integration/workflows/test_chord_error.py new file mode 100644 index 000000000..e1f9fad0b --- /dev/null +++ b/tests/integration/workflows/test_chord_error.py @@ -0,0 +1,63 @@ +""" +This module contains tests for the feature_demo workflow. +""" + +import subprocess + +from tests.fixture_data_classes import ChordErrorSetup +from tests.integration.conditions import HasRegex, StepFinishedFilesCount +from tests.integration.helper_funcs import check_test_conditions + + +class TestChordError: + """ + Tests for the chord error workflow. + """ + + def test_chord_error_continues( + self, + chord_err_setup: ChordErrorSetup, + chord_err_run_workflow: subprocess.CompletedProcess, + ): + """ + Test that this workflow continues through to the end of its execution, even + though a ChordError will be raised. + + Args: + chord_err_setup: A fixture that returns a [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] + instance. + chord_err_run_workflow: A fixture to run the chord error study. + """ + + conditions = [ + HasRegex("Exception raised by request from the user"), + StepFinishedFilesCount( # Check that the `process_samples` step has only 2 MERLIN_FINISHED files + step="process_samples", + study_name=chord_err_setup.name, + output_path=chord_err_setup.testing_dir, + expected_count=2, + num_samples=3, + ), + StepFinishedFilesCount( # Check that the `samples_and_params` step has all of its MERLIN_FINISHED files + step="samples_and_params", + study_name=chord_err_setup.name, + output_path=chord_err_setup.testing_dir, + num_parameters=2, + num_samples=3, + ), + StepFinishedFilesCount( # Check that the final step has a MERLIN_FINISHED file + step="step_3", + study_name=chord_err_setup.name, + output_path=chord_err_setup.testing_dir, + num_parameters=0, + num_samples=0, + ), + ] + + info = { + "return_code": chord_err_run_workflow.returncode, + "stdout": chord_err_run_workflow.stdout.read(), + "stderr": chord_err_run_workflow.stderr.read(), + } + + check_test_conditions(conditions, info) diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py new file mode 100644 index 000000000..7974ebecc --- /dev/null +++ b/tests/integration/workflows/test_feature_demo.py @@ -0,0 +1,132 @@ +""" +This module contains tests for the feature_demo workflow. +""" + +import shutil +import subprocess + +from tests.fixture_data_classes import FeatureDemoSetup +from tests.integration.conditions import ProvenanceYAMLFileHasRegex, StepFinishedFilesCount + + +class TestFeatureDemo: + """ + Tests for the feature_demo workflow. + """ + + def test_end_to_end_run( + self, feature_demo_setup: FeatureDemoSetup, feature_demo_run_workflow: subprocess.CompletedProcess + ): + """ + Test that the workflow runs from start to finish with no problems. + + This will check that each step has the proper amount of `MERLIN_FINISHED` files. + The workflow will be run via the + [`feature_demo_run_workflow`][fixtures.feature_demo.feature_demo_run_workflow] + fixture. + + Args: + feature_demo_setup: A fixture that returns a + [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] instance. + feature_demo_run_workflow: A fixture to run the feature demo study. + """ + conditions = [ + ProvenanceYAMLFileHasRegex( # This condition will check that variable substitution worked + regex=f"N_SAMPLES: {feature_demo_setup.num_samples}", + spec_file_name="feature_demo", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + provenance_type="expanded", + ), + StepFinishedFilesCount( # The rest of the conditions will ensure every step ran to completion + step="hello", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=feature_demo_setup.num_samples, + ), + StepFinishedFilesCount( + step="python3_hello", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="collect", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="translate", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="learn", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="make_new_samples", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="predict", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="verify", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ), + ] + + # GitHub actions doesn't have a python2 path so we'll conditionally add this check + if shutil.which("python2"): + conditions.append( + StepFinishedFilesCount( + step="python2_hello", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ) + ) + + for condition in conditions: + assert condition.passes + + # TODO implement the below tests + # def test_step_execution_order(self): + # """ + # Test that steps are executed in the correct order. + # """ + # # TODO build a list with the correct order that steps should be ran + # # TODO compare the list against the logs from the worker + + # def test_workflow_error_handling(self): + # """ + # Test the behavior when errors arise during the worfklow. + + # TODO should this test both soft and hard fails? should this test all return codes? + # """ + + # def test_data_passing(self): + # """ + # Test that data can be successfully passed between steps using built-in Merlin variables. + # """ diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 3e37cef84..3a06c0ab9 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -17,34 +17,34 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_encrypt(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_decrypt(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test that our decryption function is decrypting the bytes that we're passing to it. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_key_path(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) @@ -89,14 +89,17 @@ def test_gen_key(self, temp_output_dir: str): assert key_gen_contents != "" def test_get_key( - self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture" # noqa: F821 + self, + merlin_server_dir: str, + test_encryption_key: bytes, + redis_results_backend_config_function: "fixture", # noqa: F821 ): """ Test the `_get_key` function. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 581b19488..3b44c5261 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -36,37 +36,37 @@ def test_read_file(merlin_server_dir: str): assert actual == SERVER_PASS -def test_get_connection_string_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_connection_string_invalid_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "invalid_broker" with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_no_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_connection_string_no_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function without a broker name value in the CONFIG object. This should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: F821 +def test_get_connection_string_simple(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function in the simplest way that we can. This function will automatically check for a broker url and if it finds one in the CONFIG object it will just return the value it finds. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_url = "test_url" CONFIG.broker.url = test_url @@ -74,11 +74,11 @@ def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: assert actual == test_url -def test_get_ssl_config_no_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_ssl_config_no_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function without a broker. This should return False. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name assert not get_ssl_config() @@ -274,11 +274,11 @@ def run_get_redissock_connection(self, expected_vals: Dict[str, str]): actual = get_redissock_connection() assert actual == expected - def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with both a db_num and a broker path set. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path and db_num for testing test_path = "/fake/path/to/broker" @@ -290,12 +290,12 @@ def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa expected_vals = {"db_num": test_db_num, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_db(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a broker path set but no db num. This should default the db_num to 0. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path for testing test_path = "/fake/path/to/broker" @@ -305,25 +305,25 @@ def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): expected_vals = {"db_num": 0, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_path(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a db num set but no broker path. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.db_num = "45" with pytest.raises(AttributeError): get_redissock_connection() - def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with neither a broker path nor a db num set. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ with pytest.raises(AttributeError): get_redissock_connection() @@ -341,11 +341,11 @@ def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_passwo actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) assert expected == actual - def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -356,13 +356,13 @@ def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F8 } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_port(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the port setting from the CONFIG object. This should still run and give us port = 6379. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port expected_vals = { @@ -374,12 +374,12 @@ def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_with_db(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after adding the db_num setting to the CONFIG object. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_db_num = "45" CONFIG.broker.db_num = test_db_num @@ -392,25 +392,25 @@ def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_username(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_username(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the username setting from the CONFIG object. This should still run and give us username = ''. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.username expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_invalid_pass_file(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after changing the permissions of the password file so it can't be opened. This should still run and give us password = CONFIG.broker.password. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Capture the initial permissions of the password file so we can reset them orig_file_permissions = os.stat(CONFIG.broker.password).st_mode @@ -435,21 +435,21 @@ def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixt os.chmod(CONFIG.broker.password, orig_file_permissions) - def test_get_redis_connection_dont_include_password(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_dont_include_password(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function without including the password. This should place 6 *s where the password would normally be placed in spass. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) - def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_use_ssl(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "rediss", @@ -460,24 +460,24 @@ def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) - def test_get_redis_connection_no_password(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_password(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the password setting from the CONFIG object. This should still run and give us spass = ''. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.password expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis as the broker (this is what our CONFIG - is set to by default with the redis_broker_config fixture). + is set to by default with the redis_broker_config_function fixture). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -490,11 +490,11 @@ def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # n actual = get_connection_string() assert expected == actual - def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_connection_string_rediss(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rediss (with two 's') as the broker. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected_vals = { @@ -508,11 +508,11 @@ def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # actual = get_connection_string() assert expected == actual - def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis_socket(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis+socket as the broker. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Change our broker CONFIG.broker.name = "redis+socket" @@ -529,21 +529,21 @@ def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture" actual = get_connection_string() assert actual == expected - def test_get_ssl_config_redis(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_ssl_config_redis(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). This should return False. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert not get_ssl_config() - def test_get_ssl_config_rediss(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss (with two 's') as the broker. This should return a dict of cert reqs with ssl.CERT_NONE as the value. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected = {"ssl_cert_reqs": CERT_NONE} diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index f49e3e897..55459f3ec 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -102,7 +102,7 @@ def test_get_backend_password_using_certs_path(temp_output_dir: str): assert get_backend_password(pass_filename, certs_path=test_dir) == SERVER_PASS -def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 +def test_get_ssl_config_no_results_backend(config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with no results_backend set. This should return False. NOTE: we're using the config fixture here to make sure values are reset after this test finishes. @@ -114,7 +114,7 @@ def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 assert get_ssl_config() is False -def test_get_connection_string_no_results_backend(config: "fixture"): # noqa: F821 +def test_get_connection_string_no_results_backend(config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with no results_backend set. This should raise a ValueError. @@ -160,11 +160,11 @@ def run_get_redis( actual = get_redis(certs_path=certs_path, include_password=include_password, ssl=ssl) assert actual == expected - def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with default functionality. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -175,11 +175,11 @@ def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_dont_include_password(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_dont_include_password(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with the password hidden. This should * out the password. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -190,11 +190,11 @@ def test_get_redis_dont_include_password(self, redis_results_backend_config: "fi } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=False, ssl=False) - def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_using_ssl(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with ssl enabled. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "rediss", @@ -205,11 +205,11 @@ def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=True) - def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_port(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no port in our CONFIG object. This should default to port=6379. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.port expected_vals = { @@ -221,11 +221,11 @@ def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # no } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_db_num(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no db_num in our CONFIG object. This should default to db_num=0. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.db_num expected_vals = { @@ -237,11 +237,11 @@ def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_username(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no username in our CONFIG object. This should default to username=''. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.username expected_vals = { @@ -253,11 +253,11 @@ def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_password_file(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no password filepath in our CONFIG object. This should default to spass=''. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.password expected_vals = { @@ -269,12 +269,12 @@ def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_invalid_pass_file(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function. We'll run this after changing the permissions of the password file so it can't be opened. This should still run and give us password=CONFIG.results_backend.password. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Capture the initial permissions of the password file so we can reset them @@ -299,41 +299,41 @@ def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixtur os.chmod(CONFIG.results_backend.password, orig_file_permissions) raise AssertionError from exc - def test_get_ssl_config_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_redis(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with redis as the results_backend. This should return False since ssl requires using rediss (with two 's'). - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_ssl_config() is False - def test_get_ssl_config_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss as the results_backend. This should return a dict of cert reqs with ssl.CERT_NONE as the value. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.results_backend.name = "rediss" assert get_ssl_config() == {"ssl_cert_reqs": CERT_NONE} - def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss as the results_backend and no cert_reqs set. This should return True. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.cert_reqs CONFIG.results_backend.name = "rediss" assert get_ssl_config() is True - def test_get_connection_string_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis as the results_backend. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -346,11 +346,11 @@ def test_get_connection_string_redis(self, redis_results_backend_config: "fixtur actual = get_connection_string() assert actual == expected - def test_get_connection_string_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_connection_string_rediss(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rediss as the results_backend. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.results_backend.name = "rediss" expected_vals = { diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py index 9d64c10c7..b4f7c52fa 100644 --- a/tests/unit/config/test_utils.py +++ b/tests/unit/config/test_utils.py @@ -47,12 +47,12 @@ def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F assert get_priority(Priority.RETRY) == 10 -def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_priority_redis_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_priority` function with redis as the broker. Low priority for redis is 10 and high is 2. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_priority(Priority.LOW) == 10 assert get_priority(Priority.MID) == 5 @@ -60,12 +60,12 @@ def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F82 assert get_priority(Priority.RETRY) == 1 -def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_priority_invalid_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_priority` function with an invalid broker. This should raise a ValueError. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "invalid" with pytest.raises(ValueError) as excinfo: @@ -73,12 +73,12 @@ def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F assert "Unsupported broker name: invalid" in str(excinfo.value) -def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 +def test_get_priority_invalid_priority(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_priority` function with an invalid priority. This should raise a TypeError. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ with pytest.raises(ValueError) as excinfo: get_priority("invalid_priority")