From 29573d43b17f97f90d93ce44773770754d7359de Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 20 Nov 2023 14:42:08 -0800 Subject: [PATCH 001/201] remove a merge conflict statement that was missed --- merlin/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/merlin/__init__.py b/merlin/__init__.py index dda10809c..c1ad21b22 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -38,11 +38,7 @@ import sys -<<<<<<< HEAD -__version__ = "1.10.2" -======= __version__ = "1.11.1" ->>>>>>> 38651f2650e8aba97552c4575e97d66be3205545 VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") From f10c896d9f67397fc3e8e6111742a2ba5e3257d1 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 11 Dec 2023 15:30:05 -0800 Subject: [PATCH 002/201] add pytest coverage library and add sample_index coverage --- .gitignore | 3 +- merlin/common/sample_index.py | 2 +- requirements/dev.txt | 1 + tests/unit/common/test_sample_index.py | 672 ++++++++++++++++++------- 4 files changed, 508 insertions(+), 170 deletions(-) diff --git a/.gitignore b/.gitignore index c22521934..cec577a85 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,9 @@ flux.out slurm*.out docs/build/ -# Tox files +# Test files .tox/* +.coverage # Jupyter jupyter/.ipynb_checkpoints diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 4e3ac3a52..00295dab0 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -225,8 +225,8 @@ def __setitem__(self, full_address, sub_tree): # Replace if we already have something at this address. if delete_me is not None: - self.children.__delitem__(full_address) SampleIndex.check_valid_addresses_for_insertion(full_address, sub_tree) + self.children.__delitem__(full_address) self.children[full_address] = sub_tree return raise KeyError diff --git a/requirements/dev.txt b/requirements/dev.txt index 895a89249..ccf00e112 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,6 +5,7 @@ dep-license flake8 isort pytest +pytest-cov pylint twine sphinx>=2.0.0 diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index c693827f0..1237c52a1 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,178 +1,514 @@ import os +import pytest import shutil from contextlib import suppress +from merlin.common.sample_index import SampleIndex, uniform_directories, new_dir from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy -TEST_DIR = "UNIT_TEST_SPACE" - - -def clear_test_tree(): - with suppress(FileNotFoundError): - shutil.rmtree(TEST_DIR) - - -def clear(func): - def wrapper(): - clear_test_tree() - func() - clear_test_tree() - - return wrapper - - -@clear -def test_index_file_writing(): - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) - indx.write_directories() - indx.write_multiple_sample_index_files() - indx2 = read_hierarchy(TEST_DIR) - assert indx2.get_path_to_sample(123000123) == indx.get_path_to_sample(123000123) - - -def test_bundle_retrieval(): - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) - expected = f"{TEST_DIR}/0/0/0/samples0-10000.ext" - result = indx.get_path_to_sample(123) - assert expected == result - - expected = f"{TEST_DIR}/0/0/0/samples10000-20000.ext" - result = indx.get_path_to_sample(10000) - assert expected == result - - expected = f"{TEST_DIR}/1/2/3/samples123000000-123010000.ext" - result = indx.get_path_to_sample(123000123) - assert expected == result - - -def test_start_sample_id(): - expected = """: DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10 - 0: BUNDLE 0 MIN 203 MAX 213 - 1: BUNDLE 1 MIN 213 MAX 223 - 2: BUNDLE 2 MIN 223 MAX 233 - 3: BUNDLE 3 MIN 233 MAX 243 - 4: BUNDLE 4 MIN 243 MAX 253 - 5: BUNDLE 5 MIN 253 MAX 263 - 6: BUNDLE 6 MIN 263 MAX 273 - 7: BUNDLE 7 MIN 273 MAX 283 - 8: BUNDLE 8 MIN 283 MAX 293 - 9: BUNDLE 9 MIN 293 MAX 303 -""" - idx203 = create_hierarchy(100, 10, start_sample_id=203) - assert expected == str(idx203) - - -@clear -def test_directory_writing(): - path = os.path.join(TEST_DIR) - indx = create_hierarchy(2, 1, [1], root=path) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE 0 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: BUNDLE 1 MIN 1 MAX 2 -""" - assert expected == str(indx) - indx.write_directories() - assert os.path.isdir(f"{TEST_DIR}/0") - assert os.path.isdir(f"{TEST_DIR}/1") - indx.write_multiple_sample_index_files() - - clear_test_tree() - - path = os.path.join(TEST_DIR) - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000], root=path) - indx.write_directories() - path = indx.get_path_to_sample(123000123) - assert os.path.exists(os.path.dirname(path)) - assert path != TEST_DIR - path = indx.get_path_to_sample(10000000000) - assert path == TEST_DIR - - clear_test_tree() - - path = os.path.join(TEST_DIR) - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=path) - indx.write_directories() - - -def test_directory_path(): - indx = create_hierarchy(20, 1, [20, 5, 1], root="") - leaves = indx.make_directory_string() - expected_leaves = "0/0/0 0/0/1 0/0/2 0/0/3 0/0/4 0/1/0 0/1/1 0/1/2 0/1/3 0/1/4 0/2/0 0/2/1 0/2/2 0/2/3 0/2/4 0/3/0 0/3/1 0/3/2 0/3/3 0/3/4" - assert leaves == expected_leaves - all_dirs = indx.make_directory_string(just_leaf_directories=False) - expected_all_dirs = " 0 0/0 0/0/0 0/0/1 0/0/2 0/0/3 0/0/4 0/1 0/1/0 0/1/1 0/1/2 0/1/3 0/1/4 0/2 0/2/0 0/2/1 0/2/2 0/2/3 0/2/4 0/3 0/3/0 0/3/1 0/3/2 0/3/3 0/3/4" - assert all_dirs == expected_all_dirs - - -@clear -def test_subhierarchy_insertion(): - indx = create_hierarchy(2, 1, [1], root=TEST_DIR) - print("Writing directories") - indx.write_directories() - indx.write_multiple_sample_index_files() - print("reading heirarchy") - top = read_hierarchy(os.path.abspath(TEST_DIR)) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE -1 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: BUNDLE -1 MIN 1 MAX 2 -""" - assert str(top) == expected - print("creating sub_heirarchy") - sub_h = create_hierarchy(100, 10, address="1.0") - print("inserting sub_heirarchy") - top["1.0"] = sub_h - print(str(indx)) - print("after insertion") - print(str(top)) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE -1 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10 - 1.0.0: BUNDLE 0 MIN 0 MAX 10 - 1.0.1: BUNDLE 1 MIN 10 MAX 20 - 1.0.2: BUNDLE 2 MIN 20 MAX 30 - 1.0.3: BUNDLE 3 MIN 30 MAX 40 - 1.0.4: BUNDLE 4 MIN 40 MAX 50 - 1.0.5: BUNDLE 5 MIN 50 MAX 60 - 1.0.6: BUNDLE 6 MIN 60 MAX 70 - 1.0.7: BUNDLE 7 MIN 70 MAX 80 - 1.0.8: BUNDLE 8 MIN 80 MAX 90 - 1.0.9: BUNDLE 9 MIN 90 MAX 100 -""" - assert str(top) == expected - - -def test_sample_index(): - """Run through some basic testing of the SampleIndex class.""" +def test_uniform_directories(): + """ + Test the `uniform_directories` function with different inputs. + """ + # Create the tests and the expected outputs tests = [ - (10, 1, []), - (10, 3, []), - (11, 2, [5]), - (10, 3, [3]), - (10, 3, [1]), - (10, 1, [3]), - (10, 3, [1, 3]), - (10, 1, [2]), - (1000, 100, [500]), - (1000, 50, [500, 100]), - (1000000000, 100000132, []), + # SMALL SAMPLE SIZE + (10, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (10, 1, 2), + (10, 2, 100), + (10, 2, 2), + # MEDIUM SAMPLE SIZE + (10000, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (10000, 1, 5), + (10000, 5, 100), + (10000, 5, 10), + # LARGE SAMPLE SIZE + (1000000000, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (1000000000, 1, 5), + (1000000000, 5, 100), + (1000000000, 5, 10), ] + expected_outputs = [ + # SMALL SAMPLE SIZE + [1], + [8, 4, 2, 1], + [2], + [8, 4, 2], + # MEDIUM SAMPLE SIZE + [100, 1], + [3125, 625, 125, 25, 5, 1], + [500, 5], + [5000, 500, 50, 5], + # LARGE SAMPLE SIZE + [100000000, 1000000, 10000, 100, 1], + [244140625, 48828125, 9765625, 1953125, 390625, 78125, 15625, 3125, 625, 125, 25, 5, 1], + [500000000, 5000000, 50000, 500, 5], + [500000000, 50000000, 5000000, 500000, 50000, 5000, 500, 50, 5], + ] + + # Run the tests and compare outputs + for i, test in enumerate(tests): + actual = uniform_directories(num_samples=test[0], bundle_size=test[1], level_max_dirs=test[2]) + assert actual == expected_outputs[i] + + +def test_new_dir(temp_output_dir: str): + """ + Test the `new_dir` function. This will test a valid path and also raising an OSError during + creation. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Test basic functionality + test_path = f"{os.getcwd()}/test_new_dir" + new_dir(test_path) + assert os.path.exists(test_path) + + # Test OSError functionality + new_dir(test_path) + + + +class TestSampleIndex: + """ + These tests focus on testing the SampleIndex class used for creating the + sample hierarchy. + + NOTE to see output of creating any hierarchy, change `write_all_hierarchies` to True. + The results of each hierarchy will be written to: + /tmp/`whoami`/pytest/pytest-of-`whoami`/pytest-current/integration_outfiles_current/test_sample_index/ + """ + + write_all_hierarchies = False + + def get_working_dir(self, test_workspace: str): + """ + This method is called for every test to get a unique workspace in the temporary + directory for the test output. + + :param test_workspace: The unique name for this workspace + (all tests use their unique test name for this value usually) + """ + return f"{os.getcwd()}/test_sample_index/{test_workspace}" + + def write_hierarchy_for_debug(self, indx: SampleIndex): + """ + This method is for debugging purposes. It will cause all tests that don't write + hierarchies to write them so the output can be investigated. + + :param indx: The `SampleIndex` object to write the hierarchy for + """ + if self.write_all_hierarchies: + indx.write_directories() + indx.write_multiple_sample_index_files() + + def test_invalid_children(self): + """ + This will test that an invalid type for the `children` argument will raise + an error. + """ + tests = [ + ["a", "b", "c"], + True, + "a b c", + ] + for test in tests: + with pytest.raises(TypeError): + SampleIndex(0, 10, test, "name") + + def test_is_parent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_parent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_parent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if parent of leaf is recognized + assert indx.is_parent_of_leaf is False + assert indx.children["0"].is_parent_of_leaf is True + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"] + assert leaf_node.is_parent_of_leaf is False + + def test_is_grandparent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_grandparent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if grandparent of leaf is recognized + assert indx.is_grandparent_of_leaf is True + assert indx.children["0"].is_grandparent_of_leaf is False + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"] + assert leaf_node.is_grandparent_of_leaf is False + + def test_is_great_grandparent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_great_grandparent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_great_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [5, 1], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if great grandparent of leaf is recognized + assert indx.is_great_grandparent_of_leaf is True + assert indx.children["0"].is_great_grandparent_of_leaf is False + assert indx.children["0"].children["0.0"].is_great_grandparent_of_leaf is False + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"].children["0.0.0"] + assert leaf_node.is_great_grandparent_of_leaf is False + + def test_traverse_bundle(self, temp_output_dir: str): + """ + Test the `traverse_bundle` method to make sure it's just returning leaves. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Ensure all nodes in the traversal are leaves + for _, node in indx.traverse_bundles(): + assert node.is_leaf + + def test_getitem(self, temp_output_dir: str): + """ + Test the `__getitem__` magic method. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test getting that requesting the root returns itself + assert indx[""] == indx + + # Test a valid address + assert indx["0"] == indx.children["0"] + + # Test an invalid address + with pytest.raises(KeyError): + indx["10"] + + def test_setitem(self, temp_output_dir: str): + """ + Test the `__setitem__` magic method. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + invalid_indx = SampleIndex(1, 3, {}, "invalid_indx") + + # Ensure that trying to change the root raises an error + with pytest.raises(KeyError): + indx[""] = invalid_indx + + # Ensure we can't just add a new subtree to a level + with pytest.raises(KeyError): + indx["10"] = invalid_indx + + # Test that invalid subtrees are caught + with pytest.raises(TypeError): + indx["0"] = invalid_indx + + # Test a valid set operation + dummy_indx = SampleIndex(0, 1, {}, "dummy_indx", leafid=0, address="0.0") + indx["0"]["0.0"] = dummy_indx + + + def test_index_file_writing(self, temp_output_dir: str): + """ + Test the functionality of writing multiple index files. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_index_file_writing") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + indx.write_directories() + indx.write_multiple_sample_index_files() + indx2 = read_hierarchy(working_dir) + assert indx2.get_path_to_sample(123000123) == indx.get_path_to_sample(123000123) + + def test_directory_writing_small(self, temp_output_dir: str): + """ + Test that writing a small directory functions properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the directory and ensure it has the correct format + working_dir = self.get_working_dir("test_directory_writing_small/") + indx = create_hierarchy(2, 1, [1], root=working_dir) + expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ + " 0.0: BUNDLE 0 MIN 0 MAX 1\n" \ + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ + " 1.0: BUNDLE 1 MIN 1 MAX 2\n" \ + + assert expected == str(indx) + + # Write the directories and ensure the paths are actually written + indx.write_directories() + assert os.path.isdir(f"{working_dir}/0") + assert os.path.isdir(f"{working_dir}/1") + indx.write_multiple_sample_index_files() + + def test_directory_writing_large(self, temp_output_dir: str): + """ + Test that writing a large directory functions properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_directory_writing_large") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + indx.write_directories() + path = indx.get_path_to_sample(123000123) + assert os.path.exists(os.path.dirname(path)) + assert path != working_dir + path = indx.get_path_to_sample(10000000000) + assert path == working_dir + + def test_bundle_retrieval(self, temp_output_dir: str): + """ + Test the functionality to get a bundle of samples when providing a sample id to find. + This will test a large sample hierarchy to ensure this scales properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the hierarchy + working_dir = self.get_working_dir("test_bundle_retrieval") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test for a small sample id + expected = f"{working_dir}/0/0/0/samples0-10000.ext" + result = indx.get_path_to_sample(123) + assert expected == result + + # Test for a mid size sample id + expected = f"{working_dir}/0/0/0/samples10000-20000.ext" + result = indx.get_path_to_sample(10000) + assert expected == result + + # Test for a large sample id + expected = f"{working_dir}/1/2/3/samples123000000-123010000.ext" + result = indx.get_path_to_sample(123000123) + assert expected == result + + def test_start_sample_id(self, temp_output_dir: str): + """ + Test creating a hierarchy using a starting sample id. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_start_sample_id") + expected = ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" \ + " 0: BUNDLE 0 MIN 203 MAX 213\n" \ + " 1: BUNDLE 1 MIN 213 MAX 223\n" \ + " 2: BUNDLE 2 MIN 223 MAX 233\n" \ + " 3: BUNDLE 3 MIN 233 MAX 243\n" \ + " 4: BUNDLE 4 MIN 243 MAX 253\n" \ + " 5: BUNDLE 5 MIN 253 MAX 263\n" \ + " 6: BUNDLE 6 MIN 263 MAX 273\n" \ + " 7: BUNDLE 7 MIN 273 MAX 283\n" \ + " 8: BUNDLE 8 MIN 283 MAX 293\n" \ + " 9: BUNDLE 9 MIN 293 MAX 303\n" \ + + idx203 = create_hierarchy(100, 10, start_sample_id=203, root=working_dir) + self.write_hierarchy_for_debug(idx203) + + assert expected == str(idx203) + + def test_make_directory_string(self, temp_output_dir: str): + """ + Test the `make_directory_string` method of `SampleIndex`. This will check + both the normal functionality where we just request paths to the leaves and + also the inverse functionality where we request all paths that are not leaves. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Creating the hierarchy + working_dir = self.get_working_dir("test_make_directory_string") + indx = create_hierarchy(20, 1, [20, 5, 1], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Testing normal functionality (just leaf directories) + leaves = indx.make_directory_string() + expected_leaves_list = [ + f"{working_dir}/0/0/0", + f"{working_dir}/0/0/1", + f"{working_dir}/0/0/2", + f"{working_dir}/0/0/3", + f"{working_dir}/0/0/4", + f"{working_dir}/0/1/0", + f"{working_dir}/0/1/1", + f"{working_dir}/0/1/2", + f"{working_dir}/0/1/3", + f"{working_dir}/0/1/4", + f"{working_dir}/0/2/0", + f"{working_dir}/0/2/1", + f"{working_dir}/0/2/2", + f"{working_dir}/0/2/3", + f"{working_dir}/0/2/4", + f"{working_dir}/0/3/0", + f"{working_dir}/0/3/1", + f"{working_dir}/0/3/2", + f"{working_dir}/0/3/3", + f"{working_dir}/0/3/4", + ] + expected_leaves = " ".join(expected_leaves_list) + assert leaves == expected_leaves + + # Testing no leaf functionality + all_dirs = indx.make_directory_string(just_leaf_directories=False) + expected_all_dirs_list = [ + working_dir, + f"{working_dir}/0", + f"{working_dir}/0/0", + f"{working_dir}/0/0/0", + f"{working_dir}/0/0/1", + f"{working_dir}/0/0/2", + f"{working_dir}/0/0/3", + f"{working_dir}/0/0/4", + f"{working_dir}/0/1", + f"{working_dir}/0/1/0", + f"{working_dir}/0/1/1", + f"{working_dir}/0/1/2", + f"{working_dir}/0/1/3", + f"{working_dir}/0/1/4", + f"{working_dir}/0/2", + f"{working_dir}/0/2/0", + f"{working_dir}/0/2/1", + f"{working_dir}/0/2/2", + f"{working_dir}/0/2/3", + f"{working_dir}/0/2/4", + f"{working_dir}/0/3", + f"{working_dir}/0/3/0", + f"{working_dir}/0/3/1", + f"{working_dir}/0/3/2", + f"{working_dir}/0/3/3", + f"{working_dir}/0/3/4" + ] + expected_all_dirs = " ".join(expected_all_dirs_list) + assert all_dirs == expected_all_dirs + + def test_subhierarchy_insertion(self, temp_output_dir: str): + """ + Test that a subhierarchy can be inserted into our `SampleIndex` properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the hierarchy and read it + working_dir = self.get_working_dir("test_subhierarchy_insertion") + indx = create_hierarchy(2, 1, [1], root=working_dir) + indx.write_directories() + indx.write_multiple_sample_index_files() + top = read_hierarchy(os.path.abspath(working_dir)) + + # Compare results + expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ + " 1.0: BUNDLE -1 MIN 1 MAX 2\n" \ + + assert str(top) == expected + + # Create and insert the sub hierarchy + sub_h = create_hierarchy(100, 10, address="1.0") + top["1.0"] = sub_h + + # Compare results + expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ + " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" \ + " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" \ + " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" \ + " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" \ + " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" \ + " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" \ + " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" \ + " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" \ + " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" \ + " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" \ + " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" \ + + assert str(top) == expected + + def test_sample_index_creation_and_insertion(self, temp_output_dir: str): + """ + Run through some basic testing of the SampleIndex class. This will try + creating hierarchies of different sizes and inserting subhierarchies of + different sizes as well. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Define the tests for hierarchies of varying sizes + tests = [ + (10, 1, []), + (10, 3, []), + (11, 2, [5]), + (10, 3, [3]), + (10, 3, [1]), + (10, 1, [3]), + (10, 3, [1, 3]), + (10, 1, [2]), + (1000, 100, [500]), + (1000, 50, [500, 100]), + (1000000000, 100000132, []), + ] + + # Run all the tests we defined above + for i, args in enumerate(tests): + working_dir = self.get_working_dir(f"test_sample_index_creation_and_insertion/{i}") + + # Put at root address of "0" to guarantee insertion at "0.1" later is valid + idx = create_hierarchy(args[0], args[1], args[2], address="0", root=working_dir) + self.write_hierarchy_for_debug(idx) - for args in tests: - print(f"############ TEST {args[0]} {args[1]} {args[2]} ###########") - # put at root address of "0" to guarantee insertion at "0.1" later is valid - idx = create_hierarchy(args[0], args[1], args[2], address="0") - print(str(idx)) - try: - idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") - print("successful set") - print(str(idx)) - except KeyError as error: - print(error) - assert False + # Inserting hierarchy at 0.1 + try: + idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") + except KeyError as error: + assert False From 362478ef084c44876d54571616e0e78e346b7bc5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 09:31:22 -0800 Subject: [PATCH 003/201] run fix style and add module header --- tests/unit/common/test_sample_index.py | 100 +++++++++++++------------ 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index 1237c52a1..296783273 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,9 +1,11 @@ +""" +Tests for the `sample_index.py` and `sample_index_factory.py` files. +""" import os + import pytest -import shutil -from contextlib import suppress -from merlin.common.sample_index import SampleIndex, uniform_directories, new_dir +from merlin.common.sample_index import SampleIndex, new_dir, uniform_directories from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy @@ -70,7 +72,6 @@ def test_new_dir(temp_output_dir: str): new_dir(test_path) - class TestSampleIndex: """ These tests focus on testing the SampleIndex class used for creating the @@ -228,7 +229,7 @@ def test_setitem(self, temp_output_dir: str): working_dir = self.get_working_dir("test_is_grandparent_of_leaf") indx = create_hierarchy(10, 1, [2], root=working_dir) self.write_hierarchy_for_debug(indx) - + invalid_indx = SampleIndex(1, 3, {}, "invalid_indx") # Ensure that trying to change the root raises an error @@ -247,7 +248,6 @@ def test_setitem(self, temp_output_dir: str): dummy_indx = SampleIndex(0, 1, {}, "dummy_indx", leafid=0, address="0.0") indx["0"]["0.0"] = dummy_indx - def test_index_file_writing(self, temp_output_dir: str): """ Test the functionality of writing multiple index files. @@ -272,12 +272,13 @@ def test_directory_writing_small(self, temp_output_dir: str): # Create the directory and ensure it has the correct format working_dir = self.get_working_dir("test_directory_writing_small/") indx = create_hierarchy(2, 1, [1], root=working_dir) - expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ - " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ - " 0.0: BUNDLE 0 MIN 0 MAX 1\n" \ - " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ - " 1.0: BUNDLE 1 MIN 1 MAX 2\n" \ - + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE 0 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: BUNDLE 1 MIN 1 MAX 2\n" + ) assert expected == str(indx) # Write the directories and ensure the paths are actually written @@ -338,18 +339,19 @@ def test_start_sample_id(self, temp_output_dir: str): temporary output path for our tests """ working_dir = self.get_working_dir("test_start_sample_id") - expected = ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" \ - " 0: BUNDLE 0 MIN 203 MAX 213\n" \ - " 1: BUNDLE 1 MIN 213 MAX 223\n" \ - " 2: BUNDLE 2 MIN 223 MAX 233\n" \ - " 3: BUNDLE 3 MIN 233 MAX 243\n" \ - " 4: BUNDLE 4 MIN 243 MAX 253\n" \ - " 5: BUNDLE 5 MIN 253 MAX 263\n" \ - " 6: BUNDLE 6 MIN 263 MAX 273\n" \ - " 7: BUNDLE 7 MIN 273 MAX 283\n" \ - " 8: BUNDLE 8 MIN 283 MAX 293\n" \ - " 9: BUNDLE 9 MIN 293 MAX 303\n" \ - + expected = ( + ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" + " 0: BUNDLE 0 MIN 203 MAX 213\n" + " 1: BUNDLE 1 MIN 213 MAX 223\n" + " 2: BUNDLE 2 MIN 223 MAX 233\n" + " 3: BUNDLE 3 MIN 233 MAX 243\n" + " 4: BUNDLE 4 MIN 243 MAX 253\n" + " 5: BUNDLE 5 MIN 253 MAX 263\n" + " 6: BUNDLE 6 MIN 263 MAX 273\n" + " 7: BUNDLE 7 MIN 273 MAX 283\n" + " 8: BUNDLE 8 MIN 283 MAX 293\n" + " 9: BUNDLE 9 MIN 293 MAX 303\n" + ) idx203 = create_hierarchy(100, 10, start_sample_id=203, root=working_dir) self.write_hierarchy_for_debug(idx203) @@ -424,7 +426,7 @@ def test_make_directory_string(self, temp_output_dir: str): f"{working_dir}/0/3/1", f"{working_dir}/0/3/2", f"{working_dir}/0/3/3", - f"{working_dir}/0/3/4" + f"{working_dir}/0/3/4", ] expected_all_dirs = " ".join(expected_all_dirs_list) assert all_dirs == expected_all_dirs @@ -444,12 +446,13 @@ def test_subhierarchy_insertion(self, temp_output_dir: str): top = read_hierarchy(os.path.abspath(working_dir)) # Compare results - expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ - " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ - " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ - " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ - " 1.0: BUNDLE -1 MIN 1 MAX 2\n" \ - + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: BUNDLE -1 MIN 1 MAX 2\n" + ) assert str(top) == expected # Create and insert the sub hierarchy @@ -457,22 +460,23 @@ def test_subhierarchy_insertion(self, temp_output_dir: str): top["1.0"] = sub_h # Compare results - expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ - " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ - " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ - " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ - " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" \ - " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" \ - " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" \ - " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" \ - " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" \ - " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" \ - " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" \ - " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" \ - " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" \ - " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" \ - " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" \ - + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" + " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" + " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" + " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" + " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" + " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" + " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" + " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" + " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" + " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" + " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" + ) assert str(top) == expected def test_sample_index_creation_and_insertion(self, temp_output_dir: str): @@ -510,5 +514,5 @@ def test_sample_index_creation_and_insertion(self, temp_output_dir: str): # Inserting hierarchy at 0.1 try: idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") - except KeyError as error: + except KeyError: assert False From 9339b6bd439ba08ff6094eda7ae207e88d69c589 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 09:31:43 -0800 Subject: [PATCH 004/201] add tests for encryption modules --- merlin/common/security/encrypt.py | 9 +- tests/conftest.py | 37 ++++++++ tests/encryption_manager.py | 49 ++++++++++ tests/unit/common/test_encryption.py | 129 +++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 tests/encryption_manager.py create mode 100644 tests/unit/common/test_encryption.py diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 806d42e0c..a9f4a7107 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -52,11 +52,10 @@ def _get_key_path(): except AttributeError: key_filepath = "~/.merlin/encrypt_data_key" - try: - key_filepath = os.path.abspath(os.path.expanduser(key_filepath)) - except KeyError as e: - raise ValueError("Error! No password provided for RabbitMQ") from e - return key_filepath + if key_filepath is None: + raise ValueError("Error! No password provided for RabbitMQ") + + return os.path.abspath(os.path.expanduser(key_filepath)) def _gen_key(key_path): diff --git a/tests/conftest.py b/tests/conftest.py index 88932c5db..ca4237319 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,8 @@ from _pytest.tmpdir import TempPathFactory from celery import Celery +from tests.encryption_manager import EncryptionManager + class RedisServerError(Exception): """ @@ -300,3 +302,38 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl # Shut down the workers and terminate the processes celery_app.control.broadcast("shutdown", destination=list(worker_queue_map.keys())) shutdown_processes(worker_processes, echo_processes) + + +@pytest.fixture(scope="session") +def encryption_output_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name + """ + Get a temporary output directory for our encryption tests. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + encryption_dir = f"{temp_output_dir}/encryption_tests" + os.mkdir(encryption_dir) + return encryption_dir + + +@pytest.fixture(scope="session") +def test_encryption_key() -> bytes: + """An encryption key to be used for tests that need it""" + return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" + + +@pytest.fixture(scope="class") +def use_fake_encrypt_data_key(encryption_output_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name + """ + Create a fake encrypt data key to use for these tests. This will save the + current data key so we can set it back to what it was prior to running + the tests. + + :param encryption_output_dir: The path to the temporary output directory we'll be using for this test run + """ + # Use a context manager to ensure cleanup runs even if an error occurs + with EncryptionManager(encryption_output_dir, test_encryption_key) as encrypt_manager: + # Set the fake encryption key + encrypt_manager.set_fake_key() + # Yield control to the tests + yield diff --git a/tests/encryption_manager.py b/tests/encryption_manager.py new file mode 100644 index 000000000..883b1a184 --- /dev/null +++ b/tests/encryption_manager.py @@ -0,0 +1,49 @@ +""" +Module to define functionality for managing encryption settings +while running our test suite. +""" +import os +from types import TracebackType +from typing import Type + +from merlin.config.configfile import CONFIG + + +class EncryptionManager: + """ + A class to handle safe setup and teardown of encryption tests. + """ + + def __init__(self, temp_output_dir: str, test_encryption_key: bytes): + self.temp_output_dir = temp_output_dir + self.key_path = os.path.abspath(os.path.expanduser(f"{self.temp_output_dir}/encrypt_data_key")) + self.test_encryption_key = test_encryption_key + self.orig_results_backend = CONFIG.results_backend + + def __enter__(self): + """This magic method is necessary for allowing this class to be sued as a context manager""" + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + This will always run at the end of a context with statement, even if an error is raised. + It's a safe way to ensure all of our encryption settings at the start of the tests are reset. + """ + self.reset_encryption_settings() + + def set_fake_key(self): + """ + Create a fake encrypt data key to use for tests. This will save the fake encryption key to + our temporary output directory located at: + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/encryption_tests/ + """ + with open(self.key_path, "w") as key_file: + key_file.write(self.test_encryption_key.decode("utf-8")) + + CONFIG.results_backend.encryption_key = self.key_path + + def reset_encryption_settings(self): + """ + Reset the encryption settings to what they were prior to running our encryption tests. + """ + CONFIG.results_backend = self.orig_results_backend diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py new file mode 100644 index 000000000..6daa53817 --- /dev/null +++ b/tests/unit/common/test_encryption.py @@ -0,0 +1,129 @@ +""" +Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. +""" +import os + +import celery +import pytest + +from merlin.common.security.encrypt import _gen_key, _get_key, _get_key_path, decrypt, encrypt +from merlin.common.security.encrypt_backend_traffic import _decrypt_decode, _encrypt_encode, set_backend_funcs +from merlin.config.configfile import CONFIG + + +class TestEncryption: + """ + This class will house all tests necessary for our encryption modules. + """ + + def test_encrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + """ + Test that our encryption function is encrypting the bytes that we're + passing to it. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + """ + str_to_encrypt = b"super secret string shhh" + encrypted_str = encrypt(str_to_encrypt) + for word in str_to_encrypt.decode("utf-8").split(" "): + assert word not in encrypted_str.decode("utf-8") + + def test_decrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + """ + Test that our decryption function is decrypting the bytes that we're + passing to it. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + """ + # This is the output of the bytes from the encrypt test + str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" + decrypted_str = decrypt(str_to_decrypt) + assert decrypted_str == b"super secret string shhh" + + def test_get_key_path(self, use_fake_encrypt_data_key: "fixture"): # noqa F821 + """ + Test the `_get_key_path` function. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + """ + # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which + # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) + user = os.getlogin() + actual_default = _get_key_path() + assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encryption_tests/encrypt_data_key") + + # Test with having the encryption key set to None + temp = CONFIG.results_backend.encryption_key + CONFIG.results_backend.encryption_key = None + with pytest.raises(ValueError) as excinfo: + _get_key_path() + assert "Error! No password provided for RabbitMQ" in str(excinfo.value) + CONFIG.results_backend.encryption_key = temp + + # Test with having the entire results_backend wiped from CONFIG + orig_results_backend = CONFIG.results_backend + CONFIG.results_backend = None + actual_no_results_backend = _get_key_path() + assert actual_no_results_backend == os.path.abspath(os.path.expanduser("~/.merlin/encrypt_data_key")) + CONFIG.results_backend = orig_results_backend + + def test_gen_key(self, encryption_output_dir: str): + """ + Test the `_gen_key` function. + + :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + """ + # Create the file but don't put anything in it + key_gen_test_file = f"{encryption_output_dir}/key_gen_test" + with open(key_gen_test_file, "w"): + pass + + # Ensure nothing is in the file + with open(key_gen_test_file, "r") as key_gen_file: + key_gen_contents = key_gen_file.read() + assert key_gen_contents == "" + + # Run the test and then check to make sure the file is now populated + _gen_key(key_gen_test_file) + with open(key_gen_test_file, "r") as key_gen_file: + key_gen_contents = key_gen_file.read() + assert key_gen_contents != "" + + def test_get_key(self, use_fake_encrypt_data_key: str, encryption_output_dir: str, test_encryption_key: bytes): + """ + Test the `_get_key` function. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + :param test_encryption_key: A fixture to establish a fixed encryption key for testing + """ + # Test the default functionality + actual_default = _get_key() + assert actual_default == test_encryption_key + + # Modify the permission of the key file so that it can't be read by anyone + # (we're purposefully trying to raise an IOError) + key_path = f"{encryption_output_dir}/encrypt_data_key" + orig_file_permissions = os.stat(key_path).st_mode + os.chmod(key_path, 0o222) + with pytest.raises(IOError): + _get_key() + os.chmod(key_path, orig_file_permissions) + + # Reset the key value to our test value since the IOError test will rewrite the key + with open(key_path, "w") as key_file: + key_file.write(test_encryption_key.decode("utf-8")) + + def test_set_backend_funcs(self): + """ + Test the `set_backend_funcs` function. + """ + # Make sure these values haven't been set yet + assert celery.backends.base.Backend.encode != _encrypt_encode + assert celery.backends.base.Backend.decode != _decrypt_decode + + set_backend_funcs() + + # Ensure the new functions have been set + assert celery.backends.base.Backend.encode == _encrypt_encode + assert celery.backends.base.Backend.decode == _decrypt_decode From 54a31bce2bf5a5bad5dd9ff25b7bfd32476e0aa8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 10:52:34 -0800 Subject: [PATCH 005/201] add unit tests for util_sampling --- merlin/common/util_sampling.py | 1 + tests/unit/common/test_util_sampling.py | 44 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/unit/common/test_util_sampling.py diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 134d0b66c..1309448ef 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -35,6 +35,7 @@ import numpy as np +# TODO should we move this to merlin-spellbook? def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False): """Scale samples to new limits, either log10 or linearly. diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py new file mode 100644 index 000000000..0fd77739f --- /dev/null +++ b/tests/unit/common/test_util_sampling.py @@ -0,0 +1,44 @@ +""" +Tests for the `util_sampling.py` file. +""" +import numpy as np +import pytest + +from merlin.common.util_sampling import scale_samples + + +class TestUtilSampling: + """ + This class will hold all of the tests for the `util_sampling.py` file. + """ + + def test_scale_samples_basic(self): + """Test basic functionality without logging""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(-1, 1), (2, 6)] + result = scale_samples(samples_norm, limits) + expected_result = np.array([[-0.6, 3.6], [0.2, 5.2]]) + np.testing.assert_array_almost_equal(result, expected_result) + + def test_scale_samples_logarithmic(self): + """Test functionality with log enabled""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(1, 5), (1, 100)] + result = scale_samples(samples_norm, limits, do_log=[False, True]) + expected_result = np.array([[1.8, 6.309573], [3.4, 39.810717]]) + np.testing.assert_array_almost_equal(result, expected_result) + + def test_scale_samples_invalid_input(self): + """Test that function raises ValueError for invalid input""" + with pytest.raises(ValueError): + # Invalid input: samples_norm should be a 2D array + scale_samples([0.2, 0.4, 0.6], [(1, 5), (2, 6)]) + + def test_scale_samples_with_custom_limits_norm(self): + """Test functionality with custom limits_norm""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(1, 5), (2, 6)] + limits_norm = (-1, 1) + result = scale_samples(samples_norm, limits, limits_norm=limits_norm) + expected_result = np.array([[3.4, 4.8], [4.2, 5.6]]) + np.testing.assert_array_almost_equal(result, expected_result) \ No newline at end of file From be02611b4a436cc1973e8b83d52c30451254453d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 10:54:33 -0800 Subject: [PATCH 006/201] run fix-style and fix typo --- tests/unit/common/test_util_sampling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py index 0fd77739f..c957ac105 100644 --- a/tests/unit/common/test_util_sampling.py +++ b/tests/unit/common/test_util_sampling.py @@ -13,7 +13,7 @@ class TestUtilSampling: """ def test_scale_samples_basic(self): - """Test basic functionality without logging""" + """Test basic functionality""" samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) limits = [(-1, 1), (2, 6)] result = scale_samples(samples_norm, limits) @@ -41,4 +41,4 @@ def test_scale_samples_with_custom_limits_norm(self): limits_norm = (-1, 1) result = scale_samples(samples_norm, limits, limits_norm=limits_norm) expected_result = np.array([[3.4, 4.8], [4.2, 5.6]]) - np.testing.assert_array_almost_equal(result, expected_result) \ No newline at end of file + np.testing.assert_array_almost_equal(result, expected_result) From 63d22f063a755a5ac99095a160550bc32c51008c Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 11:52:48 -0800 Subject: [PATCH 007/201] create directory for context managers and fix issue with an encryption test --- tests/conftest.py | 7 +++---- tests/context_managers/__init__.py | 0 .../celery_workers_manager.py} | 5 +++-- tests/{ => context_managers}/encryption_manager.py | 0 tests/unit/common/test_encryption.py | 6 ++++++ 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 tests/context_managers/__init__.py rename tests/{celery_test_workers.py => context_managers/celery_workers_manager.py} (98%) rename tests/{ => context_managers}/encryption_manager.py (100%) diff --git a/tests/conftest.py b/tests/conftest.py index 5aec494e8..33fea4ce1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,9 +42,8 @@ from celery import Celery from celery.canvas import Signature -from tests.celery_test_workers import CeleryTestWorkersManager - -from tests.encryption_manager import EncryptionManager +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.context_managers.encryption_manager import EncryptionManager class RedisServerError(Exception): @@ -206,7 +205,7 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl # (basically just add in concurrency value to worker_queue_map) worker_info = {worker_name: {"concurrency": 1, "queues": [queue]} for worker_name, queue in worker_queue_map.items()} - with CeleryTestWorkersManager(celery_app) as workers_manager: + with CeleryWorkersManager(celery_app) as workers_manager: workers_manager.launch_workers(worker_info) yield diff --git a/tests/context_managers/__init__.py b/tests/context_managers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/celery_test_workers.py b/tests/context_managers/celery_workers_manager.py similarity index 98% rename from tests/celery_test_workers.py rename to tests/context_managers/celery_workers_manager.py index 39eb2a39b..70a01c8a0 100644 --- a/tests/celery_test_workers.py +++ b/tests/context_managers/celery_workers_manager.py @@ -40,9 +40,9 @@ from typing import Dict, List, Type from celery import Celery +from merlin.config.configfile import CONFIG - -class CeleryTestWorkersManager: +class CeleryWorkersManager: """ A class to handle the setup and teardown of celery workers. This should be treated as a context and used with python's @@ -198,6 +198,7 @@ def launch_workers(self, worker_info: Dict[str, Dict]): :param worker_info: A dict of worker info with the form {"worker_name": {"concurrency": , "queues": }} """ + # CONFIG.results_backend.encryption_key = "~/.merlin/encrypt_data_key" for worker_name, worker_settings in worker_info.items(): self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) diff --git a/tests/encryption_manager.py b/tests/context_managers/encryption_manager.py similarity index 100% rename from tests/encryption_manager.py rename to tests/context_managers/encryption_manager.py diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 6daa53817..6f978ddfe 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -118,6 +118,9 @@ def test_set_backend_funcs(self): """ Test the `set_backend_funcs` function. """ + orig_encode = celery.backends.base.Backend.encode + orig_decode = celery.backends.base.Backend.decode + # Make sure these values haven't been set yet assert celery.backends.base.Backend.encode != _encrypt_encode assert celery.backends.base.Backend.decode != _decrypt_decode @@ -127,3 +130,6 @@ def test_set_backend_funcs(self): # Ensure the new functions have been set assert celery.backends.base.Backend.encode == _encrypt_encode assert celery.backends.base.Backend.decode == _decrypt_decode + + celery.backends.base.Backend.encode = orig_encode + celery.backends.base.Backend.decode = orig_decode From b2a997628066665b72166722b5b4ce7af40b8f0f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 17:31:41 -0800 Subject: [PATCH 008/201] add a context manager for spinning up/down the redis server --- tests/conftest.py | 85 ++------------ .../celery_workers_manager.py | 3 +- tests/context_managers/encryption_manager.py | 2 +- tests/context_managers/server_manager.py | 105 ++++++++++++++++++ 4 files changed, 117 insertions(+), 78 deletions(-) create mode 100644 tests/context_managers/server_manager.py diff --git a/tests/conftest.py b/tests/conftest.py index 33fea4ce1..037d5868f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,30 +32,17 @@ integration test suite. """ import os -import subprocess from time import sleep from typing import Dict import pytest -import redis from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.encryption_manager import EncryptionManager - - -class RedisServerError(Exception): - """ - Exception to signal that the server wasn't pinged properly. - """ - - -class ServerInitError(Exception): - """ - Exception to signal that there was an error initializing the server. - """ +from tests.context_managers.server_manager import RedisServerManager @pytest.fixture(scope="session") @@ -80,73 +67,20 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def redis_pass() -> str: - """ - This fixture represents the password to the merlin test server. - - :returns: The redis password for our test server - """ - return "merlin-test-server" - - -@pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name - """ - This fixture will initialize the merlin server (i.e. create all the files we'll - need to start up a local redis server). It will return the path to the directory - containing the files needed for the server to start up. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :param redis_pass: The password to the test redis server that we'll create here - :returns: The path to the merlin_server directory with the server configurations - """ - # Initialize the setup for the local redis server - # We'll also set the password to 'merlin-test-server' so it'll be easy to shutdown if there's an issue - subprocess.run(f"merlin server init; merlin server config -pwd {redis_pass}", shell=True, capture_output=True, text=True) - - # Check that the merlin server was initialized properly - server_dir = f"{temp_output_dir}/merlin_server" - if not os.path.exists(server_dir): - raise ServerInitError("The merlin server was not initialized properly.") - - return server_dir - - -@pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name,unused-argument +def redis_server(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. - :param merlin_server_dir: The directory to the merlin test server configuration. - This will not be used here but we need the server configurations before we can - start the server. - :param redis_pass: The raw redis password stored in the redis.pass file + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :yields: The local redis server uri """ - # Start the local redis server - try: - # Need to set LC_ALL='C' before starting the server or else redis causes a failure - subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) - except subprocess.TimeoutExpired: - pass - - # Ensure the server started properly - host = "localhost" - port = 6379 - database = 0 - username = "default" - redis_client = redis.Redis(host=host, port=port, db=database, password=redis_pass, username=username) - if not redis_client.ping(): - raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") - - # Hand over the redis server url to any other fixtures/tests that need it - redis_server_uri = f"redis://{username}:{redis_pass}@{host}:{port}/{database}" - yield redis_server_uri - - # Kill the server; don't run this until all tests are done (accomplished with 'yield' above) - kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) - assert "Merlin server terminated." in kill_process.stderr + with RedisServerManager(temp_output_dir) as redis_server_manager: + redis_server_manager.initialize_server() + redis_server_manager.start_server() + # Yield the redis_server uri to any fixtures/tests that may need it + yield redis_server_manager.redis_server_uri + # The server will be stopped once this context reaches the end of it's execution here @pytest.fixture(scope="session") @@ -242,3 +176,4 @@ def use_fake_encrypt_data_key(encryption_output_dir: str, test_encryption_key: b # Set the fake encryption key encrypt_manager.set_fake_key() # Yield control to the tests + yield diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 70a01c8a0..38526bc1b 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -40,7 +40,7 @@ from typing import Dict, List, Type from celery import Celery -from merlin.config.configfile import CONFIG + class CeleryWorkersManager: """ @@ -198,7 +198,6 @@ def launch_workers(self, worker_info: Dict[str, Dict]): :param worker_info: A dict of worker info with the form {"worker_name": {"concurrency": , "queues": }} """ - # CONFIG.results_backend.encryption_key = "~/.merlin/encrypt_data_key" for worker_name, worker_settings in worker_info.items(): self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) diff --git a/tests/context_managers/encryption_manager.py b/tests/context_managers/encryption_manager.py index 883b1a184..84b2e4a1e 100644 --- a/tests/context_managers/encryption_manager.py +++ b/tests/context_managers/encryption_manager.py @@ -21,7 +21,7 @@ def __init__(self, temp_output_dir: str, test_encryption_key: bytes): self.orig_results_backend = CONFIG.results_backend def __enter__(self): - """This magic method is necessary for allowing this class to be sued as a context manager""" + """This magic method is necessary for allowing this class to be used as a context manager""" return self def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py new file mode 100644 index 000000000..d373c1f1c --- /dev/null +++ b/tests/context_managers/server_manager.py @@ -0,0 +1,105 @@ +""" +Module to define functionality for managing the containerized +server used for testing. +""" +import os +import signal +import subprocess +from types import TracebackType +from typing import Type + +import redis +import yaml + + +class RedisServerError(Exception): + """ + Exception to signal that the server wasn't pinged properly. + """ + + +class ServerInitError(Exception): + """ + Exception to signal that there was an error initializing the server. + """ + + +class RedisServerManager: + """ + A class to handle the setup and teardown of a containerized redis server. + This should be treated as a context and used with python's built-in 'with' + statement. If you use it without this statement, beware that the processes + spun up here may never be stopped. + """ + + def __init__(self, temp_output_dir: str): + self._redis_pass = "merlin-test-server" + self.server_dir = f"{temp_output_dir}/merlin_server" + self.host = "localhost" + self.port = 6379 + self.database = 0 + self.username = "default" + self.redis_server_uri = f"redis://{self.username}:{self._redis_pass}@{self.host}:{self.port}/{self.database}" + + def __enter__(self): + """This magic method is necessary for allowing this class to be used as a context manager""" + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + This will always run at the end of a context with statement, even if an error is raised. + It's a safe way to ensure all of our server gets stopped no matter what. + """ + self.stop_server() + + def initialize_server(self): + """ + Initialize the setup for the local redis server. We'll write the folder to: + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ + We'll set the password to be 'merlin-test-server' so it'll be easy to shutdown if necessary + """ + subprocess.run( + f"merlin server init; merlin server config -pwd {self._redis_pass}", shell=True, capture_output=True, text=True + ) + + # Check that the merlin server was initialized properly + if not os.path.exists(self.server_dir): + raise ServerInitError("The merlin server was not initialized properly.") + + def start_server(self): + """Attempt to start the local redis server.""" + try: + # Need to set LC_ALL='C' before starting the server or else redis causes a failure + subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) + except subprocess.TimeoutExpired: + pass + + # Ensure the server started properly + redis_client = redis.Redis( + host=self.host, port=self.port, db=self.database, password=self._redis_pass, username=self.username + ) + if not redis_client.ping(): + raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + + def stop_server(self): + """Stop the server.""" + # Attempt to stop the server gracefully with `merlin server` + kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) + + # Check that the server was terminated + if "Merlin server terminated." not in kill_process.stderr: + # If it wasn't, try to kill the process by using the pid stored in a file created by `merlin server` + try: + with open(f"{self.server_dir}/merlin_server.pf", "r") as process_file: + server_process_info = yaml.load(process_file, yaml.Loader) + os.kill(int(server_process_info["image_pid"]), signal.SIGKILL) + # If the file can't be found then let's make sure there's even a redis-server process running + except FileNotFoundError as exc: + process_query = subprocess.run("ps ux", shell=True, text=True, capture_output=True) + # If there is a file running we didn't start it in this test run so we can't kill it + if "redis-server" in process_query.stdout: + raise RedisServerError( + "Found an active redis server but cannot stop it since there is no process file (merlin_server.pf). " + "Did you start this server before running tests?" + ) from exc + # No else here. If there's no redis-server process found then there's nothing to stop From 35c614a4b8efc63e944db4ce54eeb257af22cf52 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 13 Dec 2023 14:42:41 -0800 Subject: [PATCH 009/201] fix issue with path in one test --- tests/unit/common/test_sample_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index 296783273..cdb5b2f4f 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -64,7 +64,7 @@ def test_new_dir(temp_output_dir: str): temporary output path for our tests """ # Test basic functionality - test_path = f"{os.getcwd()}/test_new_dir" + test_path = f"{os.getcwd()}/test_sample_index/test_new_dir" new_dir(test_path) assert os.path.exists(test_path) From 638a27e47ef1d7ace2dc2045d3564465d450a6d1 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 13 Dec 2023 14:44:12 -0800 Subject: [PATCH 010/201] rework CONFIG functionality for testing --- merlin/config/__init__.py | 30 +++++++ tests/conftest.py | 91 +++++++++++++------- tests/context_managers/encryption_manager.py | 49 ----------- tests/context_managers/server_manager.py | 29 ++++++- tests/unit/common/test_encryption.py | 31 ++++--- 5 files changed, 133 insertions(+), 97 deletions(-) delete mode 100644 tests/context_managers/encryption_manager.py diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index b58e3b2a9..c2dd4d12b 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -31,6 +31,7 @@ """ Used to store the application configuration. """ +from copy import copy from types import SimpleNamespace from typing import Dict, List, Optional @@ -56,6 +57,35 @@ def __init__(self, app_dict): self.results_backend: Optional[SimpleNamespace] self.load_app_into_namespaces(app_dict) + def __copy__(self): + """ + A magic method to allow this class to be copied with copy(instance_of_Config). + """ + cls = self.__class__ + result = cls.__new__(cls) + copied_attrs = { + "celery": copy(self.__dict__["celery"]), + "broker": copy(self.__dict__["broker"]), + "results_backend": copy(self.__dict__["results_backend"]), + } + result.__dict__.update(copied_attrs) + return result + + def __str__(self): + """ + A magic method so we can print the CONFIG class. + """ + formatted_str = "config:" + attrs = {"celery": self.celery, "broker": self.broker, "results_backend": self.results_backend} + for name, attr in attrs.items(): + if attr is not None: + items = (f" {k}: {v!r}" for k, v in attr.__dict__.items()) + joined_items = "\n".join(items) + formatted_str += f"\n {name}: \n{joined_items}" + else: + formatted_str += f"\n {name}: \n None" + return formatted_str + def load_app_into_namespaces(self, app_dict: Dict) -> None: """ Makes the application dictionary into a namespace, sets the attributes of the Config from the namespace values. diff --git a/tests/conftest.py b/tests/conftest.py index 037d5868f..992b5203b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,22 +28,25 @@ # SOFTWARE. ############################################################################### """ -This module contains pytest fixtures to be used throughout the entire -integration test suite. +This module contains pytest fixtures to be used throughout the entire test suite. """ import os +import yaml +from copy import copy from time import sleep -from typing import Dict +from typing import Any, Dict import pytest from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature +from merlin.config.configfile import CONFIG from tests.context_managers.celery_workers_manager import CeleryWorkersManager -from tests.context_managers.encryption_manager import EncryptionManager from tests.context_managers.server_manager import RedisServerManager +REDIS_PASS = "merlin-test-server" + @pytest.fixture(scope="session") def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @@ -67,15 +70,27 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def redis_server(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name +def merlin_server_dir(temp_output_dir: str) -> str: + """ + The path to the merlin_server directory that will be created by the `redis_server` fixture. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture + """ + return f"{temp_output_dir}/merlin_server" + + +@pytest.fixture(scope="session") +def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # pylint: disable=redefined-outer-name """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ - with RedisServerManager(temp_output_dir) as redis_server_manager: + with RedisServerManager(merlin_server_dir, REDIS_PASS, test_encryption_key) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() # Yield the redis_server uri to any fixtures/tests that may need it @@ -145,35 +160,53 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl @pytest.fixture(scope="session") -def encryption_output_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name +def test_encryption_key() -> bytes: """ - Get a temporary output directory for our encryption tests. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + An encryption key to be used for tests that need it. + + :returns: The test encryption key """ - encryption_dir = f"{temp_output_dir}/encryption_tests" - os.mkdir(encryption_dir) - return encryption_dir + return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" @pytest.fixture(scope="session") -def test_encryption_key() -> bytes: - """An encryption key to be used for tests that need it""" - return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" +def app_yaml(merlin_server_dir: str, redis_server: str) -> Dict[str, Any]: # pylint: disable=redefined-outer-name + """ + Load in the app.yaml file generated by starting the redis server. + :param merlin_server_dir: The directory to the merlin test server configuration + :param redis_server: The fixture that starts up the redis server + :returns: The contents of the app.yaml file created by starting the redis server + """ + with open(f"{merlin_server_dir}/app.yaml", "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + return app_yaml -@pytest.fixture(scope="class") -def use_fake_encrypt_data_key(encryption_output_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name + +@pytest.fixture(scope="function") +def config(app_yaml: str): # pylint: disable=redefined-outer-name """ - Create a fake encrypt data key to use for these tests. This will save the - current data key so we can set it back to what it was prior to running - the tests. + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object. This will modify the CONFIG object to use static test values + that shouldn't change. - :param encryption_output_dir: The path to the temporary output directory we'll be using for this test run + :param app_yaml: The contents of the app.yaml created by starting the containerized redis server """ - # Use a context manager to ensure cleanup runs even if an error occurs - with EncryptionManager(encryption_output_dir, test_encryption_key) as encrypt_manager: - # Set the fake encryption key - encrypt_manager.set_fake_key() - # Yield control to the tests - yield + global CONFIG + orig_config = copy(CONFIG) + + CONFIG.broker.password = app_yaml["broker"]["password"] + CONFIG.broker.port = app_yaml["broker"]["port"] + CONFIG.broker.server = app_yaml["broker"]["server"] + CONFIG.broker.username = app_yaml["broker"]["username"] + CONFIG.broker.vhost = app_yaml["broker"]["vhost"] + + CONFIG.results_backend.password = app_yaml["results_backend"]["password"] + CONFIG.results_backend.port = app_yaml["results_backend"]["port"] + CONFIG.results_backend.server = app_yaml["results_backend"]["server"] + CONFIG.results_backend.username = app_yaml["results_backend"]["username"] + CONFIG.results_backend.encryption_key = app_yaml["results_backend"]["encryption_key"] + + yield + + CONFIG = orig_config diff --git a/tests/context_managers/encryption_manager.py b/tests/context_managers/encryption_manager.py deleted file mode 100644 index 84b2e4a1e..000000000 --- a/tests/context_managers/encryption_manager.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Module to define functionality for managing encryption settings -while running our test suite. -""" -import os -from types import TracebackType -from typing import Type - -from merlin.config.configfile import CONFIG - - -class EncryptionManager: - """ - A class to handle safe setup and teardown of encryption tests. - """ - - def __init__(self, temp_output_dir: str, test_encryption_key: bytes): - self.temp_output_dir = temp_output_dir - self.key_path = os.path.abspath(os.path.expanduser(f"{self.temp_output_dir}/encrypt_data_key")) - self.test_encryption_key = test_encryption_key - self.orig_results_backend = CONFIG.results_backend - - def __enter__(self): - """This magic method is necessary for allowing this class to be used as a context manager""" - return self - - def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): - """ - This will always run at the end of a context with statement, even if an error is raised. - It's a safe way to ensure all of our encryption settings at the start of the tests are reset. - """ - self.reset_encryption_settings() - - def set_fake_key(self): - """ - Create a fake encrypt data key to use for tests. This will save the fake encryption key to - our temporary output directory located at: - /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/encryption_tests/ - """ - with open(self.key_path, "w") as key_file: - key_file.write(self.test_encryption_key.decode("utf-8")) - - CONFIG.results_backend.encryption_key = self.key_path - - def reset_encryption_settings(self): - """ - Reset the encryption settings to what they were prior to running our encryption tests. - """ - CONFIG.results_backend = self.orig_results_backend diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index d373c1f1c..9a10e0cbf 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -32,9 +32,10 @@ class RedisServerManager: spun up here may never be stopped. """ - def __init__(self, temp_output_dir: str): - self._redis_pass = "merlin-test-server" - self.server_dir = f"{temp_output_dir}/merlin_server" + def __init__(self, server_dir: str, redis_pass: str, test_encryption_key: bytes): + self._redis_pass = redis_pass + self._test_encryption_key = test_encryption_key + self.server_dir = server_dir self.host = "localhost" self.port = 6379 self.database = 0 @@ -66,6 +67,26 @@ def initialize_server(self): if not os.path.exists(self.server_dir): raise ServerInitError("The merlin server was not initialized properly.") + def _create_fake_encryption_key(self): + """ + For testing we'll use a specific encryption key. We'll create a file for that and + save it to the app.yaml created for testing. + """ + # Create a fake encryption key file for testing purposes + encryption_file = f"{self.server_dir}/encrypt_data_key" + with open(encryption_file, "w") as key_file: + key_file.write(self._test_encryption_key.decode("utf-8")) + + # Load up the app.yaml that was created by starting the server + server_app_yaml = f"{self.server_dir}/app.yaml" + with open(server_app_yaml, "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + + # Modify the path to the encryption key and then save it + app_yaml["results_backend"]["encryption_key"] = encryption_file + with open(server_app_yaml, "w") as app_yaml_file: + yaml.dump(app_yaml, app_yaml_file) + def start_server(self): """Attempt to start the local redis server.""" try: @@ -81,6 +102,8 @@ def start_server(self): if not redis_client.ping(): raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + self._create_fake_encryption_key() + def stop_server(self): """Stop the server.""" # Attempt to stop the server gracefully with `merlin server` diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 6f978ddfe..012c5c540 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -16,41 +16,40 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + def test_encrypt(self, config: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + def test_decrypt(self, config: "fixture"): # noqa: F821 """ - Test that our decryption function is decrypting the bytes that we're - passing to it. + Test that our decryption function is decrypting the bytes that we're passing to it. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, use_fake_encrypt_data_key: "fixture"): # noqa F821 + def test_get_key_path(self, config: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) user = os.getlogin() actual_default = _get_key_path() - assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encryption_tests/encrypt_data_key") + assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encrypt_data_key") # Test with having the encryption key set to None temp = CONFIG.results_backend.encryption_key @@ -67,14 +66,14 @@ def test_get_key_path(self, use_fake_encrypt_data_key: "fixture"): # noqa F821 assert actual_no_results_backend == os.path.abspath(os.path.expanduser("~/.merlin/encrypt_data_key")) CONFIG.results_backend = orig_results_backend - def test_gen_key(self, encryption_output_dir: str): + def test_gen_key(self, temp_output_dir: str): """ Test the `_gen_key` function. - :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + :param temp_output_dir: The path to the temporary output directory for this test run """ # Create the file but don't put anything in it - key_gen_test_file = f"{encryption_output_dir}/key_gen_test" + key_gen_test_file = f"{temp_output_dir}/key_gen_test" with open(key_gen_test_file, "w"): pass @@ -89,13 +88,13 @@ def test_gen_key(self, encryption_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, use_fake_encrypt_data_key: str, encryption_output_dir: str, test_encryption_key: bytes): + def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, config: "fixture"): # noqa: F821 """ Test the `_get_key` function. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing - :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() @@ -103,7 +102,7 @@ def test_get_key(self, use_fake_encrypt_data_key: str, encryption_output_dir: st # Modify the permission of the key file so that it can't be read by anyone # (we're purposefully trying to raise an IOError) - key_path = f"{encryption_output_dir}/encrypt_data_key" + key_path = f"{merlin_server_dir}/encrypt_data_key" orig_file_permissions = os.stat(key_path).st_mode os.chmod(key_path, 0o222) with pytest.raises(IOError): From 9b342ab0325a724d39a101e066740b84dbdcb407 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 11:52:08 -0800 Subject: [PATCH 011/201] refactor config fixture so it doesn't depend on redis server to be started --- tests/conftest.py | 159 ++++++++++++++++++----- tests/context_managers/server_manager.py | 25 +--- tests/unit/common/test_encryption.py | 19 +-- 3 files changed, 135 insertions(+), 68 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 992b5203b..4c970992c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,81 @@ from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager -REDIS_PASS = "merlin-test-server" +SERVER_PASS = "merlin-test-server" + + +####################################### +#### Helper Functions for Fixtures #### +####################################### + + +def create_pass_file(pass_filepath: str): + """ + Check if a password file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the password to the file. + + :param pass_filepath: The path to the password file that we need to check for/create + """ + if not os.path.exists(pass_filepath): + with open(pass_filepath, "w") as pass_file: + pass_file.write(SERVER_PASS) + + +def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_filepath: str = None): + """ + Check if an encryption file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the encryption key to the file. If an app.yaml + filepath has been passed to this function then we'll need to update it so that the encryption + key points to the `key_filepath`. + + :param key_filepath: The path to the file that will store our encryption key + :param encryption_key: An encryption key to be used for testing + :param app_yaml_filepath: A path to the app.yaml file that needs to be updated + """ + if not os.path.exists(key_filepath): + with open(key_filepath, "w") as key_file: + key_file.write(encryption_key.decode("utf-8")) + + if app_yaml_filepath is not None: + # Load up the app.yaml that was created by starting the server + with open(app_yaml_filepath, "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + + # Modify the path to the encryption key and then save it + app_yaml["results_backend"]["encryption_key"] = key_filepath + with open(app_yaml_filepath, "w") as app_yaml_file: + yaml.dump(app_yaml, app_yaml_file) + + +def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): + """ + Given configuration options for the broker and results_backend, update + the CONFIG object. + + :param broker: A dict of the configuration settings for the broker + :param results_backend: A dict of configuration settings for the results_backend + """ + global CONFIG + + # Set the broker configuration for testing + CONFIG.broker.password = broker["password"] + CONFIG.broker.port = broker["port"] + CONFIG.broker.server = broker["server"] + CONFIG.broker.username = broker["username"] + CONFIG.broker.vhost = broker["vhost"] + CONFIG.broker.name = broker["name"] + + # Set the results_backend configuration for testing + CONFIG.results_backend.password = results_backend["password"] + CONFIG.results_backend.port = results_backend["port"] + CONFIG.results_backend.server = results_backend["server"] + CONFIG.results_backend.username = results_backend["username"] + CONFIG.results_backend.encryption_key = results_backend["encryption_key"] + + +####################################### +######### Fixture Definitions ######### +####################################### @pytest.fixture(scope="session") @@ -77,7 +151,10 @@ def merlin_server_dir(temp_output_dir: str) -> str: :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture """ - return f"{temp_output_dir}/merlin_server" + server_dir = f"{temp_output_dir}/merlin_server" + if not os.path.exists(server_dir): + os.mkdir(server_dir) + return server_dir @pytest.fixture(scope="session") @@ -90,9 +167,10 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ - with RedisServerManager(merlin_server_dir, REDIS_PASS, test_encryption_key) as redis_server_manager: + with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() + create_encryption_file(f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml") # Yield the redis_server uri to any fixtures/tests that may need it yield redis_server_manager.redis_server_uri # The server will be stopped once this context reaches the end of it's execution here @@ -163,50 +241,61 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl def test_encryption_key() -> bytes: """ An encryption key to be used for tests that need it. - + :returns: The test encryption key """ return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" -@pytest.fixture(scope="session") -def app_yaml(merlin_server_dir: str, redis_server: str) -> Dict[str, Any]: # pylint: disable=redefined-outer-name - """ - Load in the app.yaml file generated by starting the redis server. - - :param merlin_server_dir: The directory to the merlin test server configuration - :param redis_server: The fixture that starts up the redis server - :returns: The contents of the app.yaml file created by starting the redis server - """ - with open(f"{merlin_server_dir}/app.yaml", "r") as app_yaml_file: - app_yaml = yaml.load(app_yaml_file, yaml.Loader) - return app_yaml - - @pytest.fixture(scope="function") -def config(app_yaml: str): # pylint: disable=redefined-outer-name +def redis_config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name """ This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object. This will modify the CONFIG object to use static test values - that shouldn't change. + that uses the CONFIG object with a Redis broker and results_backend. - :param app_yaml: The contents of the app.yaml created by starting the containerized redis server + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: An encryption key to be used for testing """ global CONFIG - orig_config = copy(CONFIG) - CONFIG.broker.password = app_yaml["broker"]["password"] - CONFIG.broker.port = app_yaml["broker"]["port"] - CONFIG.broker.server = app_yaml["broker"]["server"] - CONFIG.broker.username = app_yaml["broker"]["username"] - CONFIG.broker.vhost = app_yaml["broker"]["vhost"] - - CONFIG.results_backend.password = app_yaml["results_backend"]["password"] - CONFIG.results_backend.port = app_yaml["results_backend"]["port"] - CONFIG.results_backend.server = app_yaml["results_backend"]["server"] - CONFIG.results_backend.username = app_yaml["results_backend"]["username"] - CONFIG.results_backend.encryption_key = app_yaml["results_backend"]["encryption_key"] + # Create a copy of the CONFIG option so we can reset it after the test + orig_config = copy(CONFIG) + # Create a password file and encryption key file (if they don't already exist) + pass_file = f"{merlin_server_dir}/redis.pass" + key_file = f"{merlin_server_dir}/encrypt_data_key" + create_pass_file(pass_file) + create_encryption_file(key_file, test_encryption_key) + + # Create the broker and results_backend configuration to use + broker = { + "cert_reqs": "none", + "password": pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + "name": "redis", + } + + results_backend = { + "cert_reqs": "none", + "db_num": 0, + "encryption_key": key_file, + "password": pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "name": "redis", + } + + # Set the configuration + set_config(broker, results_backend) + + # Go run the tests yield - CONFIG = orig_config + # Reset the configuration + CONFIG.celery = orig_config.celery + CONFIG.broker = orig_config.broker + CONFIG.results_backend = orig_config.results_backend diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index 9a10e0cbf..ea6a731ff 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -32,9 +32,8 @@ class RedisServerManager: spun up here may never be stopped. """ - def __init__(self, server_dir: str, redis_pass: str, test_encryption_key: bytes): + def __init__(self, server_dir: str, redis_pass: str): self._redis_pass = redis_pass - self._test_encryption_key = test_encryption_key self.server_dir = server_dir self.host = "localhost" self.port = 6379 @@ -67,26 +66,6 @@ def initialize_server(self): if not os.path.exists(self.server_dir): raise ServerInitError("The merlin server was not initialized properly.") - def _create_fake_encryption_key(self): - """ - For testing we'll use a specific encryption key. We'll create a file for that and - save it to the app.yaml created for testing. - """ - # Create a fake encryption key file for testing purposes - encryption_file = f"{self.server_dir}/encrypt_data_key" - with open(encryption_file, "w") as key_file: - key_file.write(self._test_encryption_key.decode("utf-8")) - - # Load up the app.yaml that was created by starting the server - server_app_yaml = f"{self.server_dir}/app.yaml" - with open(server_app_yaml, "r") as app_yaml_file: - app_yaml = yaml.load(app_yaml_file, yaml.Loader) - - # Modify the path to the encryption key and then save it - app_yaml["results_backend"]["encryption_key"] = encryption_file - with open(server_app_yaml, "w") as app_yaml_file: - yaml.dump(app_yaml, app_yaml_file) - def start_server(self): """Attempt to start the local redis server.""" try: @@ -102,8 +81,6 @@ def start_server(self): if not redis_client.ping(): raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") - self._create_fake_encryption_key() - def stop_server(self): """Stop the server.""" # Attempt to stop the server gracefully with `merlin server` diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 012c5c540..d65d201f2 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -1,6 +1,7 @@ """ Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. """ +import getpass import os import celery @@ -16,38 +17,38 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, config: "fixture"): # noqa: F821 + def test_encrypt(self, redis_config: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, config: "fixture"): # noqa: F821 + def test_decrypt(self, redis_config: "fixture"): # noqa: F821 """ Test that our decryption function is decrypting the bytes that we're passing to it. - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, config: "fixture"): # noqa: F821 + def test_get_key_path(self, redis_config: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) - user = os.getlogin() + user = getpass.getuser() actual_default = _get_key_path() assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encrypt_data_key") @@ -88,13 +89,13 @@ def test_gen_key(self, temp_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, config: "fixture"): # noqa: F821 + def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_config: "fixture"): # noqa: F821 """ Test the `_get_key` function. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() From 661ab71daa9ec365eb1f11b35389a86b0457395b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 14:24:52 -0800 Subject: [PATCH 012/201] split CONFIG fixtures into rabbit and redis configs, run fix-style --- merlin/config/__init__.py | 1 - tests/conftest.py | 114 ++++++++++++++++++--------- tests/unit/common/test_encryption.py | 4 +- 3 files changed, 77 insertions(+), 42 deletions(-) diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index c2dd4d12b..d0a0bf9c5 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -32,7 +32,6 @@ Used to store the application configuration. """ from copy import copy - from types import SimpleNamespace from typing import Dict, List, Optional diff --git a/tests/conftest.py b/tests/conftest.py index 4c970992c..3415385f9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,12 +31,12 @@ This module contains pytest fixtures to be used throughout the entire test suite. """ import os -import yaml from copy import copy from time import sleep -from typing import Any, Dict +from typing import Dict import pytest +import yaml from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature @@ -45,6 +45,7 @@ from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager + SERVER_PASS = "merlin-test-server" @@ -84,7 +85,7 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi # Load up the app.yaml that was created by starting the server with open(app_yaml_filepath, "r") as app_yaml_file: app_yaml = yaml.load(app_yaml_file, yaml.Loader) - + # Modify the path to the encryption key and then save it app_yaml["results_backend"]["encryption_key"] = key_filepath with open(app_yaml_filepath, "w") as app_yaml_file: @@ -99,8 +100,6 @@ def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): :param broker: A dict of the configuration settings for the broker :param results_backend: A dict of configuration settings for the results_backend """ - global CONFIG - # Set the broker configuration for testing CONFIG.broker.password = broker["password"] CONFIG.broker.port = broker["port"] @@ -144,7 +143,7 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str) -> str: +def merlin_server_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name """ The path to the merlin_server directory that will be created by the `redis_server` fixture. @@ -170,7 +169,9 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() - create_encryption_file(f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml") + create_encryption_file( + f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml" + ) # Yield the redis_server uri to any fixtures/tests that may need it yield redis_server_manager.redis_server_uri # The server will be stopped once this context reaches the end of it's execution here @@ -248,49 +249,42 @@ def test_encryption_key() -> bytes: @pytest.fixture(scope="function") -def redis_config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name +def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name """ - This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a Redis broker and results_backend. + DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. + This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` + fixtures. It sets up the CONFIG object but leaves certain broker settings unset. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: An encryption key to be used for testing """ - global CONFIG + # global CONFIG # Create a copy of the CONFIG option so we can reset it after the test orig_config = copy(CONFIG) - # Create a password file and encryption key file (if they don't already exist) - pass_file = f"{merlin_server_dir}/redis.pass" + # Create an encryption key file (if it doesn't already exist) key_file = f"{merlin_server_dir}/encrypt_data_key" - create_pass_file(pass_file) create_encryption_file(key_file, test_encryption_key) - # Create the broker and results_backend configuration to use - broker = { - "cert_reqs": "none", - "password": pass_file, - "port": 6379, - "server": "127.0.0.1", - "username": "default", - "vhost": "host4testing", - "name": "redis", - } - - results_backend = { - "cert_reqs": "none", - "db_num": 0, - "encryption_key": key_file, - "password": pass_file, - "port": 6379, - "server": "127.0.0.1", - "username": "default", - "name": "redis", - } - - # Set the configuration - set_config(broker, results_backend) + # Set the broker configuration for testing + CONFIG.broker.password = "password path not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.port = "port not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.name = "name not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.server = "127.0.0.1" + CONFIG.broker.username = "default" + CONFIG.broker.vhost = "host4testing" + CONFIG.broker.cert_reqs = "none" + + # Set the results_backend configuration for testing + CONFIG.results_backend.password = f"{merlin_server_dir}/redis.pass" + CONFIG.results_backend.port = 6379 + CONFIG.results_backend.server = "127.0.0.1" + CONFIG.results_backend.username = "default" + CONFIG.results_backend.cert_reqs = "none" + CONFIG.results_backend.encryption_key = key_file + CONFIG.results_backend.db_num = 0 + CONFIG.results_backend.name = "redis" # Go run the tests yield @@ -299,3 +293,47 @@ def redis_config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: CONFIG.celery = orig_config.celery CONFIG.broker = orig_config.broker CONFIG.results_backend = orig_config.results_backend + + +@pytest.fixture(scope="function") +def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a Redis broker and results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + # global CONFIG + + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + + CONFIG.broker.password = pass_file + CONFIG.broker.port = 6379 + CONFIG.broker.name = "redis" + + yield + + +@pytest.fixture(scope="function") +def rabbit_config( + merlin_server_dir: str, config: "fixture" +): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a RabbitMQ broker and Redis results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + # global CONFIG + + pass_file = f"{merlin_server_dir}/rabbit.pass" + create_pass_file(pass_file) + + CONFIG.broker.password = pass_file + CONFIG.broker.port = 5671 + CONFIG.broker.name = "rabbitmq" + + yield diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index d65d201f2..6392cf8da 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -1,7 +1,6 @@ """ Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. """ -import getpass import os import celery @@ -48,9 +47,8 @@ def test_get_key_path(self, redis_config: "fixture"): # noqa: F821 """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) - user = getpass.getuser() actual_default = _get_key_path() - assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encrypt_data_key") + assert actual_default.startswith("/tmp/") and actual_default.endswith("/encrypt_data_key") # Test with having the encryption key set to None temp = CONFIG.results_backend.encryption_key From db1f20a073ce0cd580948f96974f37ef7db9d80f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 14:25:15 -0800 Subject: [PATCH 013/201] add unit tests for broker.py --- merlin/config/broker.py | 14 +- tests/unit/config/test_broker.py | 549 +++++++++++++++++++++++++++++++ 2 files changed, 553 insertions(+), 10 deletions(-) create mode 100644 tests/unit/config/test_broker.py diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 385b8c1df..152c6a9b8 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -85,13 +85,13 @@ def get_rabbit_connection(include_password, conn="amqps"): password_filepath = CONFIG.broker.password LOG.debug(f"Broker: password filepath = {password_filepath}") password_filepath = os.path.abspath(expanduser(password_filepath)) - except KeyError as e: # pylint: disable=C0103 - raise ValueError("Broker: No password provided for RabbitMQ") from e + except (AttributeError, KeyError) as exc: + raise ValueError("Broker: No password provided for RabbitMQ") from exc try: password = read_file(password_filepath) - except IOError as e: # pylint: disable=C0103 - raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from e + except IOError as exc: + raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from exc try: port = CONFIG.broker.port @@ -205,12 +205,6 @@ def get_connection_string(include_password=True): except AttributeError: broker = "" - try: - config_path = CONFIG.celery.certs - config_path = os.path.abspath(os.path.expanduser(config_path)) - except AttributeError: - config_path = None - if broker not in BROKERS: raise ValueError(f"Error: {broker} is not a supported broker.") return _sort_valid_broker(broker, include_password) diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py new file mode 100644 index 000000000..9d4760f3e --- /dev/null +++ b/tests/unit/config/test_broker.py @@ -0,0 +1,549 @@ +""" +Tests for the `broker.py` file. +""" +import os +from ssl import CERT_NONE +from typing import Any, Dict + +import pytest + +from merlin.config.broker import ( + RABBITMQ_CONNECTION, + REDISSOCK_CONNECTION, + get_connection_string, + get_rabbit_connection, + get_redis_connection, + get_redissock_connection, + get_ssl_config, + read_file, +) +from merlin.config.configfile import CONFIG +from tests.conftest import SERVER_PASS, create_pass_file + + +def test_read_file(merlin_server_dir: str): + """ + Test the `read_file` function. We'll start up our containerized redis server + so that we have a password file to read here. + + :param merlin_server_dir: The directory to the merlin test server configuration + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + actual = read_file(pass_file) + assert actual == SERVER_PASS + + +def test_get_connection_string_invalid_broker(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: + ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "invalid_broker" + with pytest.raises(ValueError): + get_connection_string() + + +def test_get_connection_string_no_broker(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function without a broker name value in the CONFIG object. This + should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.name + with pytest.raises(ValueError): + get_connection_string() + + +def test_get_connection_string_simple(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function in the simplest way that we can. This function + will automatically check for a broker url and if it finds one in the CONFIG object it will just + return the value it finds. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + test_url = "test_url" + CONFIG.broker.url = test_url + actual = get_connection_string() + assert actual == test_url + + +def test_get_ssl_config_no_broker(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function without a broker. This should return False. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.name + assert not get_ssl_config() + + +class TestRabbitBroker: + """ + This class will house all tests necessary for our broker module when using a + rabbit broker. + """ + + def run_get_rabbit_connection(self, expected_vals: Dict[str, Any], include_password: bool, conn: str): + """ + Helper method to run the tests for the `get_rabbit_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"conn": "", + "vhost": "host4testing", + "username": "default", + "password": "", + "server": "127.0.0.1", + "port": } + :param include_password: If True, include the password in the output. Otherwise don't. + :param conn: The connection type to pass in (either amqp or amqps) + """ + expected = RABBITMQ_CONNECTION.format(**expected_vals) + actual = get_rabbit_connection(include_password=include_password, conn=conn) + assert actual == expected + + def test_get_rabbit_connection(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_dont_include_password(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function but set include_password to False. This should * out the + password + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": "******", + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=False, conn=conn) + + def test_get_rabbit_connection_no_port_amqp(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use + 5672 as the port since we're using amqp as the connection. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + CONFIG.broker.name = "amqp" + conn = "amqp" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5672, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_no_port_amqps(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use + 5671 as the port since we're using amqps as the connection. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_no_password(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no password file set. This should raise a ValueError. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.password + with pytest.raises(ValueError) as excinfo: + get_rabbit_connection(True) + assert "Broker: No password provided for RabbitMQ" in str(excinfo.value) + + def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with an invalid password filepath. + This should raise a ValueError. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.password = "invalid_filepath" + expanded_filepath = os.path.abspath(os.path.expanduser(CONFIG.broker.password)) + with pytest.raises(ValueError) as excinfo: + get_rabbit_connection(True) + assert f"Broker: RabbitMQ password file {expanded_filepath} does not exist" in str(excinfo.value) + + def run_get_connection_string(self, expected_vals: Dict[str, Any]): + """ + Helper method to run the tests for the `get_connection_string`. + + :param expected_vals: A dict of expected values for this test. Format: + {"conn": "", + "vhost": "host4testing", + "username": "default", + "password": "", + "server": "127.0.0.1", + "port": } + """ + expected = RABBITMQ_CONNECTION.format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_connection_string_rabbitmq(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rabbitmq as the broker. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "conn": "amqps", + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_connection_string(expected_vals) + + def test_get_connection_string_amqp(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with amqp as the broker. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + CONFIG.broker.name = "amqp" + expected_vals = { + "conn": "amqp", + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5672, + } + self.run_get_connection_string(expected_vals) + + +class TestRedisBroker: + """ + This class will house all tests necessary for our broker module when using a + redis broker. + """ + + def run_get_redissock_connection(self, expected_vals: Dict[str, str]): + """ + Helper method to run the tests for the `get_redissock_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"db_num": "", "path": ""} + """ + expected = REDISSOCK_CONNECTION.format(**expected_vals) + actual = get_redissock_connection() + assert actual == expected + + def test_get_redissock_connection(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with both a db_num and a broker path set. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Create and store a fake path and db_num for testing + test_path = "/fake/path/to/broker" + test_db_num = "45" + CONFIG.broker.path = test_path + CONFIG.broker.db_num = test_db_num + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": test_db_num, "path": test_path} + self.run_get_redissock_connection(expected_vals) + + def test_get_redissock_connection_no_db(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with a broker path set but no db num. + This should default the db_num to 0. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Create and store a fake path for testing + test_path = "/fake/path/to/broker" + CONFIG.broker.path = test_path + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": 0, "path": test_path} + self.run_get_redissock_connection(expected_vals) + + def test_get_redissock_connection_no_path(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with a db num set but no broker path. + This should raise an AttributeError since there will be no path value to read from + in `CONFIG.broker`. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.db_num = "45" + with pytest.raises(AttributeError): + get_redissock_connection() + + def test_get_redissock_connection_no_path_nor_db(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with neither a broker path nor a db num set. + This should raise an AttributeError since there will be no path value to read from + in `CONFIG.broker`. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + with pytest.raises(AttributeError): + get_redissock_connection() + + def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_password: bool, use_ssl: bool): + """ + Helper method to run the tests for the `get_redis_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"urlbase": "", "spass": "", "server": "127.0.0.1", "port": , "db_num": } + :param include_password: If True, include the password in the output. Otherwise don't. + :param use_ssl: If True, use ssl for the connection. Otherwise don't. + """ + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) + assert expected == actual + + def test_get_redis_connection(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_no_port(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the port setting from the CONFIG object. This should still run and give us + port = 6379. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_with_db(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after adding the db_num setting to the CONFIG object. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + test_db_num = "45" + CONFIG.broker.db_num = test_db_num + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": test_db_num, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_no_username(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the username setting from the CONFIG object. This should still run and give us + username = ''. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.username + expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_invalid_pass_file(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after changing the permissions of the password file so it can't be opened. This should still + run and give us password = CONFIG.broker.password. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Capture the initial permissions of the password file so we can reset them + orig_file_permissions = os.stat(CONFIG.broker.password).st_mode + + # Change the permissions of the password file so it can't be read + os.chmod(CONFIG.broker.password, 0o222) + + try: + # Run the test + expected_vals = { + "urlbase": "redis", + "spass": f"default:{CONFIG.broker.password}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + except AssertionError as exc: + # If this test failed, make sure to reset the permissions in case other tests need to read this file + os.chmod(CONFIG.broker.password, orig_file_permissions) + raise AssertionError from exc + + os.chmod(CONFIG.broker.password, orig_file_permissions) + + def test_get_redis_connection_dont_include_password(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function without including the password. This should place 6 *s + where the password would normally be placed in spass. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) + + def test_get_redis_connection_use_ssl(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "rediss", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) + + def test_get_redis_connection_no_password(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the password setting from the CONFIG object. This should still run and give us + spass = ''. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.password + expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_connection_string_redis(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis as the broker (this is what our CONFIG + is set to by default with the redis_config fixture). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert expected == actual + + def test_get_connection_string_rediss(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rediss (with two 's') as the broker. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "rediss" + expected_vals = { + "urlbase": "rediss", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert expected == actual + + def test_get_connection_string_redis_socket(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis+socket as the broker. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Change our broker + CONFIG.broker.name = "redis+socket" + + # Create and store a fake path and db_num for testing + test_path = "/fake/path/to/broker" + test_db_num = "45" + CONFIG.broker.path = test_path + CONFIG.broker.db_num = test_db_num + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": test_db_num, "path": test_path} + expected = REDISSOCK_CONNECTION.format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_ssl_config_redis(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). + This should return False. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert not get_ssl_config() + + def test_get_ssl_config_rediss(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss (with two 's') as the broker. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "rediss" + expected = {"ssl_cert_reqs": CERT_NONE} + actual = get_ssl_config() + assert actual == expected From 896898e89c432221d708f2d7ca83986ac59a9b04 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 15:31:57 -0800 Subject: [PATCH 014/201] add unit tests for the Config object --- merlin/config/__init__.py | 4 +- setup.cfg | 5 + tests/conftest.py | 4 +- tests/unit/config/test_config_object.py | 149 ++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/unit/config/test_config_object.py diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index d0a0bf9c5..af1562ae4 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -80,9 +80,9 @@ def __str__(self): if attr is not None: items = (f" {k}: {v!r}" for k, v in attr.__dict__.items()) joined_items = "\n".join(items) - formatted_str += f"\n {name}: \n{joined_items}" + formatted_str += f"\n {name}:\n{joined_items}" else: - formatted_str += f"\n {name}: \n None" + formatted_str += f"\n {name}:\n None" return formatted_str def load_app_into_namespaces(self, app_dict: Dict) -> None: diff --git a/setup.cfg b/setup.cfg index a000df59a..0eaa116ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,3 +26,8 @@ max-line-length = 127 files=best_practices,test ignore_missing_imports=true + +[coverage:run] +omit = + merlin/ascii.py + merlin/config/celeryconfig.py diff --git a/tests/conftest.py b/tests/conftest.py index 3415385f9..79ad427bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -318,8 +318,8 @@ def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylin @pytest.fixture(scope="function") def rabbit_config( - merlin_server_dir: str, config: "fixture" -): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a RabbitMQ broker and Redis results_backend. diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py new file mode 100644 index 000000000..bd658bc66 --- /dev/null +++ b/tests/unit/config/test_config_object.py @@ -0,0 +1,149 @@ +""" +Test the functionality of the Config object. +""" +from copy import copy, deepcopy +from types import SimpleNamespace + +from merlin.config import Config + + +class TestConfig: + """ + Class for testing the Config object. We'll store a valid `app_dict` + as an attribute here so that each test doesn't have to redefine it + each time. + """ + + app_dict = { + "celery": {"override": {"visibility_timeout": 86400}}, + "broker": { + "cert_reqs": "none", + "name": "rabbitmq", + "password": "/path/to/pass_file", + "port": 5671, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + }, + "results_backend": { + "cert_reqs": "none", + "db_num": 0, + "name": "rediss", + "password": "/path/to/pass_file", + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + "encryption_key": "/path/to/encryption_key", + }, + } + + def test_config_creation(self): + """ + Test the creation of the Config object. This should create nested namespaces + for each key in the `app_dict` variable and save them to their respective + attributes in the object. + """ + config = Config(self.app_dict) + + # Create the nested namespace for celery and compare result + override_namespace = SimpleNamespace(**self.app_dict["celery"]["override"]) + updated_celery_dict = deepcopy(self.app_dict) + updated_celery_dict["celery"]["override"] = override_namespace + celery_namespace = SimpleNamespace(**updated_celery_dict["celery"]) + assert config.celery == celery_namespace + + # Broker and Results Backend are easier since there's no nested namespace here + assert config.broker == SimpleNamespace(**self.app_dict["broker"]) + assert config.results_backend == SimpleNamespace(**self.app_dict["results_backend"]) + + def test_config_creation_no_celery(self): + """ + Test the creation of the Config object without the celery key. This should still + work and just not set anything for the celery attribute. + """ + + # Copy the celery section so we can restore it later and then delete it + celery_section = copy(self.app_dict["celery"]) + del self.app_dict["celery"] + config = Config(self.app_dict) + + # Broker and Results Backend are the only things loaded here + assert config.broker == SimpleNamespace(**self.app_dict["broker"]) + assert config.results_backend == SimpleNamespace(**self.app_dict["results_backend"]) + + # Ensure the celery attribute is not loaded + assert "celery" not in dir(config) + + # Reset celery section in case other tests use it after this + self.app_dict["celery"] = celery_section + + def test_config_copy(self): + """ + Test the `__copy__` magic method of the Config object. Here we'll make sure + each attribute was copied properly but the ids should be different. + """ + orig_config = Config(self.app_dict) + copied_config = copy(orig_config) + + assert orig_config.celery == copied_config.celery + assert orig_config.broker == copied_config.broker + assert orig_config.results_backend == copied_config.results_backend + + assert id(orig_config) != id(copied_config) + + def test_config_str(self): + """ + Test the `__str__` magic method of the Config object. This should just give us + a formatted string of the attributes in the object. + """ + config = Config(self.app_dict) + + # Test normal printing + actual = config.__str__() + expected = ( + "config:\n" + " celery:\n" + " override: namespace(visibility_timeout=86400)\n" + " broker:\n" + " cert_reqs: 'none'\n" + " name: 'rabbitmq'\n" + " password: '/path/to/pass_file'\n" + " port: 5671\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " results_backend:\n" + " cert_reqs: 'none'\n" + " db_num: 0\n" + " name: 'rediss'\n" + " password: '/path/to/pass_file'\n" + " port: 6379\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " encryption_key: '/path/to/encryption_key'" + ) + + assert actual == expected + + # Test printing with one section set to None + config.results_backend = None + actual_with_none = config.__str__() + expected_with_none = ( + "config:\n" + " celery:\n" + " override: namespace(visibility_timeout=86400)\n" + " broker:\n" + " cert_reqs: 'none'\n" + " name: 'rabbitmq'\n" + " password: '/path/to/pass_file'\n" + " port: 5671\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " results_backend:\n" + " None" + ) + + assert actual_with_none == expected_with_none From 2a5148c9b233a95bfe45762ce27dd0072be61e9a Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 15:32:08 -0800 Subject: [PATCH 015/201] update CHANGELOG --- CHANGELOG.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0074e9d..3d0bea05d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Pytest fixtures in the `conftest.py` file of the integration test suite +- Pytest fixtures in the `conftest.py` file of the test suite - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command -- Tests for the `celeryadapter.py` module -- New CeleryTestWorkersManager context to help with starting/stopping workers for tests +- Coverage to the test suite. This includes adding tests for: + - `merlin/common/` + - `merlin/config/` + - `celeryadapter.py` +- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures + - RedisServerManager: context to help with starting/stopping a redis server for tests + - CeleryWorkersManager: context to help with starting/stopping workers for tests +- Ability to copy and print the `Config` object from `merlin/config/__init__.py` ### Fixed - The `merlin status` command so that it's consistent in its output whether using redis or rabbitmq as the broker From 3d7228e1b981962d873520bbbd65ea78b9b76dd0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 08:54:15 -0800 Subject: [PATCH 016/201] make CONFIG fixtures more flexible for tests --- tests/conftest.py | 79 ++++++++++++++---- tests/unit/common/test_encryption.py | 16 ++-- tests/unit/config/test_broker.py | 118 +++++++++++++-------------- 3 files changed, 132 insertions(+), 81 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 79ad427bb..b681ade35 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -248,6 +248,21 @@ def test_encryption_key() -> bytes: return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" +####################################### +########### CONFIG Fixtures ########### +####################################### +# These are intended to be used # +# either by themselves or together # +# For example, you can use a rabbit # +# broker config and a redis results # +# backend config together # +####################################### +############ !!!WARNING!!! ############ +# DO NOT USE THE `config` FIXTURE # +# IN A TEST; IT HAS UNSET VALUES # +####################################### + + @pytest.fixture(scope="function") def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name """ @@ -258,7 +273,6 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: An encryption key to be used for testing """ - # global CONFIG # Create a copy of the CONFIG option so we can reset it after the test orig_config = copy(CONFIG) @@ -268,23 +282,24 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab create_encryption_file(key_file, test_encryption_key) # Set the broker configuration for testing - CONFIG.broker.password = "password path not yet set" # This will be updated in `redis_config` or `rabbit_config` - CONFIG.broker.port = "port not yet set" # This will be updated in `redis_config` or `rabbit_config` - CONFIG.broker.name = "name not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.password = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.port = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.name = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` CONFIG.broker.server = "127.0.0.1" CONFIG.broker.username = "default" CONFIG.broker.vhost = "host4testing" CONFIG.broker.cert_reqs = "none" # Set the results_backend configuration for testing - CONFIG.results_backend.password = f"{merlin_server_dir}/redis.pass" - CONFIG.results_backend.port = 6379 + CONFIG.results_backend.password = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` + CONFIG.results_backend.name = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` CONFIG.results_backend.server = "127.0.0.1" CONFIG.results_backend.username = "default" CONFIG.results_backend.cert_reqs = "none" CONFIG.results_backend.encryption_key = key_file CONFIG.results_backend.db_num = 0 - CONFIG.results_backend.name = "redis" # Go run the tests yield @@ -296,7 +311,7 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab @pytest.fixture(scope="function") -def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +def redis_broker_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis broker and results_backend. @@ -304,8 +319,6 @@ def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylin :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - # global CONFIG - pass_file = f"{merlin_server_dir}/redis.pass" create_pass_file(pass_file) @@ -317,18 +330,35 @@ def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylin @pytest.fixture(scope="function") -def rabbit_config( +def redis_results_backend_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a Redis results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.port = 6379 + CONFIG.results_backend.name = "redis" + + yield + + +@pytest.fixture(scope="function") +def rabbit_broker_config( merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument ): """ This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a RabbitMQ broker and Redis results_backend. + that uses the CONFIG object with a RabbitMQ broker. :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - # global CONFIG - pass_file = f"{merlin_server_dir}/rabbit.pass" create_pass_file(pass_file) @@ -337,3 +367,24 @@ def rabbit_config( CONFIG.broker.name = "rabbitmq" yield + + +@pytest.fixture(scope="function") +def mysql_results_backend_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a MySQL results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/mysql.pass" + create_pass_file(pass_file) + + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.name = "mysql" + CONFIG.results_backend.dbname = "test_mysql_db" + + yield diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 6392cf8da..d0069f09e 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -16,34 +16,34 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, redis_config: "fixture"): # noqa: F821 + def test_encrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, redis_config: "fixture"): # noqa: F821 + def test_decrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 """ Test that our decryption function is decrypting the bytes that we're passing to it. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, redis_config: "fixture"): # noqa: F821 + def test_get_key_path(self, redis_results_backend_config: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) @@ -87,13 +87,13 @@ def test_gen_key(self, temp_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_config: "fixture"): # noqa: F821 + def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture"): # noqa: F821 """ Test the `_get_key` function. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 9d4760f3e..490b47649 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -34,37 +34,37 @@ def test_read_file(merlin_server_dir: str): assert actual == SERVER_PASS -def test_get_connection_string_invalid_broker(redis_config: "fixture"): # noqa: F821 +def test_get_connection_string_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "invalid_broker" with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_no_broker(redis_config: "fixture"): # noqa: F821 +def test_get_connection_string_no_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function without a broker name value in the CONFIG object. This should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_simple(redis_config: "fixture"): # noqa: F821 +def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function in the simplest way that we can. This function will automatically check for a broker url and if it finds one in the CONFIG object it will just return the value it finds. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_url = "test_url" CONFIG.broker.url = test_url @@ -72,11 +72,11 @@ def test_get_connection_string_simple(redis_config: "fixture"): # noqa: F821 assert actual == test_url -def test_get_ssl_config_no_broker(redis_config: "fixture"): # noqa: F821 +def test_get_ssl_config_no_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function without a broker. This should return False. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name assert not get_ssl_config() @@ -106,11 +106,11 @@ def run_get_rabbit_connection(self, expected_vals: Dict[str, Any], include_passw actual = get_rabbit_connection(include_password=include_password, conn=conn) assert actual == expected - def test_get_rabbit_connection(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ conn = "amqps" expected_vals = { @@ -123,12 +123,12 @@ def test_get_rabbit_connection(self, rabbit_config: "fixture"): # noqa: F821 } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) - def test_get_rabbit_connection_dont_include_password(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_dont_include_password(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function but set include_password to False. This should * out the password - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ conn = "amqps" expected_vals = { @@ -141,12 +141,12 @@ def test_get_rabbit_connection_dont_include_password(self, rabbit_config: "fixtu } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=False, conn=conn) - def test_get_rabbit_connection_no_port_amqp(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_no_port_amqp(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use 5672 as the port since we're using amqp as the connection. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port CONFIG.broker.name = "amqp" @@ -161,12 +161,12 @@ def test_get_rabbit_connection_no_port_amqp(self, rabbit_config: "fixture"): # } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) - def test_get_rabbit_connection_no_port_amqps(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_no_port_amqps(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use 5671 as the port since we're using amqps as the connection. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port conn = "amqps" @@ -180,23 +180,23 @@ def test_get_rabbit_connection_no_port_amqps(self, rabbit_config: "fixture"): # } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) - def test_get_rabbit_connection_no_password(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_no_password(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with no password file set. This should raise a ValueError. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.password with pytest.raises(ValueError) as excinfo: get_rabbit_connection(True) assert "Broker: No password provided for RabbitMQ" in str(excinfo.value) - def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with an invalid password filepath. This should raise a ValueError. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.password = "invalid_filepath" expanded_filepath = os.path.abspath(os.path.expanduser(CONFIG.broker.password)) @@ -220,11 +220,11 @@ def run_get_connection_string(self, expected_vals: Dict[str, Any]): actual = get_connection_string() assert actual == expected - def test_get_connection_string_rabbitmq(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_connection_string_rabbitmq(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rabbitmq as the broker. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "conn": "amqps", @@ -236,11 +236,11 @@ def test_get_connection_string_rabbitmq(self, rabbit_config: "fixture"): # noqa } self.run_get_connection_string(expected_vals) - def test_get_connection_string_amqp(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_connection_string_amqp(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with amqp as the broker. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port CONFIG.broker.name = "amqp" @@ -272,11 +272,11 @@ def run_get_redissock_connection(self, expected_vals: Dict[str, str]): actual = get_redissock_connection() assert actual == expected - def test_get_redissock_connection(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with both a db_num and a broker path set. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path and db_num for testing test_path = "/fake/path/to/broker" @@ -288,12 +288,12 @@ def test_get_redissock_connection(self, redis_config: "fixture"): # noqa: F821 expected_vals = {"db_num": test_db_num, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_db(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a broker path set but no db num. This should default the db_num to 0. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path for testing test_path = "/fake/path/to/broker" @@ -303,25 +303,25 @@ def test_get_redissock_connection_no_db(self, redis_config: "fixture"): # noqa: expected_vals = {"db_num": 0, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_path(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a db num set but no broker path. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.db_num = "45" with pytest.raises(AttributeError): get_redissock_connection() - def test_get_redissock_connection_no_path_nor_db(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with neither a broker path nor a db num set. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ with pytest.raises(AttributeError): get_redissock_connection() @@ -339,11 +339,11 @@ def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_passwo actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) assert expected == actual - def test_get_redis_connection(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -354,13 +354,13 @@ def test_get_redis_connection(self, redis_config: "fixture"): # noqa: F821 } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_port(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the port setting from the CONFIG object. This should still run and give us port = 6379. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port expected_vals = { @@ -372,12 +372,12 @@ def test_get_redis_connection_no_port(self, redis_config: "fixture"): # noqa: F } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_with_db(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after adding the db_num setting to the CONFIG object. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_db_num = "45" CONFIG.broker.db_num = test_db_num @@ -390,25 +390,25 @@ def test_get_redis_connection_with_db(self, redis_config: "fixture"): # noqa: F } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_username(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_username(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the username setting from the CONFIG object. This should still run and give us username = ''. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.username expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_invalid_pass_file(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after changing the permissions of the password file so it can't be opened. This should still run and give us password = CONFIG.broker.password. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Capture the initial permissions of the password file so we can reset them orig_file_permissions = os.stat(CONFIG.broker.password).st_mode @@ -433,21 +433,21 @@ def test_get_redis_connection_invalid_pass_file(self, redis_config: "fixture"): os.chmod(CONFIG.broker.password, orig_file_permissions) - def test_get_redis_connection_dont_include_password(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_dont_include_password(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function without including the password. This should place 6 *s where the password would normally be placed in spass. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) - def test_get_redis_connection_use_ssl(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "rediss", @@ -458,24 +458,24 @@ def test_get_redis_connection_use_ssl(self, redis_config: "fixture"): # noqa: F } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) - def test_get_redis_connection_no_password(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_password(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the password setting from the CONFIG object. This should still run and give us spass = ''. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.password expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_connection_string_redis(self, redis_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis as the broker (this is what our CONFIG - is set to by default with the redis_config fixture). + is set to by default with the redis_broker_config fixture). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -488,11 +488,11 @@ def test_get_connection_string_redis(self, redis_config: "fixture"): # noqa: F8 actual = get_connection_string() assert expected == actual - def test_get_connection_string_rediss(self, redis_config: "fixture"): # noqa: F821 + def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rediss (with two 's') as the broker. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected_vals = { @@ -506,11 +506,11 @@ def test_get_connection_string_rediss(self, redis_config: "fixture"): # noqa: F actual = get_connection_string() assert expected == actual - def test_get_connection_string_redis_socket(self, redis_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis+socket as the broker. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Change our broker CONFIG.broker.name = "redis+socket" @@ -527,21 +527,21 @@ def test_get_connection_string_redis_socket(self, redis_config: "fixture"): # n actual = get_connection_string() assert actual == expected - def test_get_ssl_config_redis(self, redis_config: "fixture"): # noqa: F821 + def test_get_ssl_config_redis(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). This should return False. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert not get_ssl_config() - def test_get_ssl_config_rediss(self, redis_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss (with two 's') as the broker. This should return a dict of cert reqs with ssl.CERT_NONE as the value. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected = {"ssl_cert_reqs": CERT_NONE} From 525e4032baf54f1cca811d0ffb5c96af688371d3 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 12:55:05 -0800 Subject: [PATCH 017/201] add tests for results_backend.py --- merlin/config/results_backend.py | 6 + tests/conftest.py | 21 + tests/unit/config/test_results_backend.py | 570 ++++++++++++++++++++++ 3 files changed, 597 insertions(+) create mode 100644 tests/unit/config/test_results_backend.py diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index b88655399..6775cb854 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -236,6 +236,12 @@ def get_mysql(certs_path=None, mysql_certs=None, include_password=True): mysql_config["password"] = "******" mysql_config["server"] = server + # Ensure the ssl_key, ssl_ca, and ssl_cert keys are all set + if mysql_certs == MYSQL_CONFIG_FILENAMES: + for key, cert_file in mysql_certs.items(): + if key not in mysql_config: + mysql_config[key] = os.path.join(certs_path, cert_file) + return MYSQL_CONNECTION_STRING.format(**mysql_config) diff --git a/tests/conftest.py b/tests/conftest.py index b681ade35..88eeaddb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,11 @@ SERVER_PASS = "merlin-test-server" +CERT_FILES = { + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem", +} ####################################### @@ -92,6 +97,20 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi yaml.dump(app_yaml, app_yaml_file) +def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): + """ + Check if cert files already exist and if they don't then create them. + + :param cert_filepath: The path to the cert files + :param cert_files: A dict of certification files to create + """ + for cert_file in cert_files.values(): + full_cert_filepath = f"{cert_filepath}/{cert_file}" + if not os.path.exists(full_cert_filepath): + with open(full_cert_filepath, "w"): + pass + + def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): """ Given configuration options for the broker and results_backend, update @@ -383,6 +402,8 @@ def mysql_results_backend_config( pass_file = f"{merlin_server_dir}/mysql.pass" create_pass_file(pass_file) + create_cert_files(merlin_server_dir, CERT_FILES) + CONFIG.results_backend.password = pass_file CONFIG.results_backend.name = "mysql" CONFIG.results_backend.dbname = "test_mysql_db" diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py new file mode 100644 index 000000000..3531a83a2 --- /dev/null +++ b/tests/unit/config/test_results_backend.py @@ -0,0 +1,570 @@ +""" +Tests for the `results_backend.py` file. +""" +import os +import pytest +from ssl import CERT_NONE +from typing import Any, Dict + +from merlin.config.configfile import CONFIG +from merlin.config.results_backend import ( + MYSQL_CONFIG_FILENAMES, + MYSQL_CONNECTION_STRING, + SQLITE_CONNECTION_STRING, + get_backend_password, + get_connection_string, + get_mysql, + get_mysql_config, + get_redis, + get_ssl_config +) +from tests.conftest import CERT_FILES, SERVER_PASS, create_cert_files, create_pass_file + +RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" + + +def test_get_backend_password_pass_file_in_merlin(): + """ + Test the `get_backend_password` function with the password file in the ~/.merlin/ + directory. We'll create a dummy file in this directory and delete it once the test + is done. + """ + + # Check if the .merlin directory exists and create it if it doesn't + remove_merlin_dir_after_test = False + path_to_merlin_dir = os.path.expanduser("~/.merlin") + if not os.path.exists(path_to_merlin_dir): + remove_merlin_dir_after_test = True + os.mkdir(path_to_merlin_dir) + + # Create the test password file + pass_filename = "test.pass" + full_pass_filepath = f"{path_to_merlin_dir}/{pass_filename}" + create_pass_file(full_pass_filepath) + + try: + # Run the test + assert get_backend_password(pass_filename) == SERVER_PASS + # Cleanup + os.remove(full_pass_filepath) + if remove_merlin_dir_after_test: + os.rmdir(path_to_merlin_dir) + except AssertionError as exc: + # If the test fails, make sure we clean up the files/dirs created + os.remove(full_pass_filepath) + if remove_merlin_dir_after_test: + os.rmdir(path_to_merlin_dir) + raise AssertionError from exc + + +def test_get_backend_password_pass_file_not_in_merlin(temp_output_dir: str): + """ + Test the `get_backend_password` function with the password file not in the ~/.merlin/ + directory. By using the `temp_output_dir` fixture, our cwd will be the temporary directory. + We'll create a password file in the this directory for this test and have `get_backend_password` + read from that. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + pass_file = "test.pass" + create_pass_file(pass_file) + + assert get_backend_password(pass_file) == SERVER_PASS + + +def test_get_backend_password_directly_pass_password(): + """ + Test the `get_backend_password` function by passing the password directly to this + function instead of a password file. + """ + assert get_backend_password(SERVER_PASS) == SERVER_PASS + + +def test_get_backend_password_using_certs_path(temp_output_dir: str): + """ + Test the `get_backend_password` function with certs_path set to our temporary testing path. + We'll create a password file in the temporary directory for this test and have `get_backend_password` + read from that. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + pass_filename = "test_certs.pass" + test_dir = RESULTS_BACKEND_DIR.format(temp_output_dir=temp_output_dir) + if not os.path.exists(test_dir): + os.mkdir(test_dir) + full_pass_filepath = f"{test_dir}/{pass_filename}" + create_pass_file(full_pass_filepath) + + assert get_backend_password(pass_filename, certs_path=test_dir) == SERVER_PASS + + +def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with no results_backend set. This should return False. + NOTE: we're using the config fixture here to make sure values are reset after this test finishes. + We won't actually use anything from the config fixture. + + :param config: A fixture to set up the CONFIG object for us + """ + del CONFIG.results_backend.name + assert get_ssl_config() is False + + +def test_get_connection_string_no_results_backend(config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with no results_backend set. + This should raise a ValueError. + NOTE: we're using the config fixture here to make sure values are reset after this test finishes. + We won't actually use anything from the config fixture. + + :param config: A fixture to set up the CONFIG object for us + """ + del CONFIG.results_backend.name + with pytest.raises(ValueError) as excinfo: + get_connection_string() + + assert "'' is not a supported results backend" in str(excinfo.value) + + +class TestRedisResultsBackend: + """ + This class will house all tests necessary for our results_backend module when using a + redis results_backend. + """ + + def run_get_redis( + self, + expected_vals: Dict[str, Any], + certs_path: str = None, + include_password: bool = True, + ssl: bool = False, + ): + """ + Helper method for running tests for the `get_redis` function. + + :param expected_vals: A dict of expected values for this test. Format: + {"urlbase": "redis", + "spass": "", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0} + :param certs_path: A string denoting the path to the certification files + :param include_password: If True, include the password in the output. Otherwise don't. + :param ssl: If True, use ssl. Otherwise, don't. + """ + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_redis(certs_path=certs_path, include_password=include_password, ssl=ssl) + assert actual == expected + + def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with default functionality. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_dont_include_password(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with the password hidden. This should * out the password. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:******@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=False, ssl=False) + + def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with ssl enabled. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "rediss", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=True) + + def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no port in our CONFIG object. This should default to port=6379. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.port + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no db_num in our CONFIG object. This should default to db_num=0. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.db_num + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no username in our CONFIG object. This should default to username=''. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.username + expected_vals = { + "urlbase": "redis", + "spass": f":{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no password filepath in our CONFIG object. This should default to spass=''. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.password + expected_vals = { + "urlbase": "redis", + "spass": "", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function. We'll run this after changing the permissions of the password file so it + can't be opened. This should still run and give us password=CONFIG.results_backend.password. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + + # Capture the initial permissions of the password file so we can reset them + orig_file_permissions = os.stat(CONFIG.results_backend.password).st_mode + + # Change the permissions of the password file so it can't be read + os.chmod(CONFIG.results_backend.password, 0o222) + + try: + # Run the test + expected_vals = { + "urlbase": "redis", + "spass": f"default:{CONFIG.results_backend.password}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + os.chmod(CONFIG.results_backend.password, orig_file_permissions) + except AssertionError as exc: + # If this test failed, make sure to reset the permissions in case other tests need to read this file + os.chmod(CONFIG.results_backend.password, orig_file_permissions) + raise AssertionError from exc + + def test_get_ssl_config_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with redis as the results_backend. This should return False since + ssl requires using rediss (with two 's'). + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config() is False + + def test_get_ssl_config_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss as the results_backend. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "rediss" + assert get_ssl_config() == {"ssl_cert_reqs": CERT_NONE} + + def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss as the results_backend and no cert_reqs set. + This should return True. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.cert_reqs + CONFIG.results_backend.name = "rediss" + assert get_ssl_config() is True + + def test_get_connection_string_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis as the results_backend. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_connection_string_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rediss as the results_backend. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "rediss" + expected_vals = { + "urlbase": "rediss", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + +class TestMySQLResultsBackend: + """ + This class will house all tests necessary for our results_backend module when using a + MySQL results_backend. + NOTE: You'll notice a lot of these tests are setting CONFIG.results_backend.name to be + "invalid". This is so that we can get by the first if statement in the `get_mysql_config` + function. + """ + + def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql_config` function with the certs dict getting set and returned. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected = {} + for key, cert_file in CERT_FILES.items(): + expected[key] = f"{merlin_server_dir}/{cert_file}" + actual = get_mysql_config(merlin_server_dir, CERT_FILES) + assert actual == expected + + def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql_config` function with mysql_ssl being found. This should just return the ssl value that's found. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_mysql_config(None, None) == {"cert_reqs": CERT_NONE} + + def test_get_mysql_config_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql_config` function with no mysql certs dict. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + assert get_mysql_config(merlin_server_dir, {}) == {} + + def test_get_mysql_config_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql_config` function with an invalid certs path. This should return False. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "invalid" + assert get_mysql_config("invalid/path", CERT_FILES) is False + + def run_get_mysql(self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool): + """ + Helper method for running tests for the `get_mysql` function. + + :param expected_vals: A dict of expected values for this test. Format: + {"cert_reqs": cert reqs dict, + "user": "default", + "password": "", + "server": "127.0.0.1", + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem"} + :param certs_path: A string denoting the path to the certification files + :param mysql_certs: A dict of cert files + :param include_password: If True, include the password in the output. Otherwise don't. + """ + expected = MYSQL_CONNECTION_STRING.format(**expected_vals) + actual = get_mysql(certs_path=certs_path, mysql_certs=mysql_certs, include_password=include_password) + assert actual == expected + + def test_get_mysql(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function with default behavior. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + for key, cert_file in CERT_FILES.items(): + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True) + + def test_get_mysql_dont_include_password(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function but set include_password to False. This should * out the password. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": "******", + "server": "127.0.0.1", + } + for key, cert_file in CERT_FILES.items(): + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False) + + def test_get_mysql_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function with no mysql_certs passed in. This should use default config filenames so we'll + have to create these default files. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + + create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + + for key, cert_file in MYSQL_CONFIG_FILENAMES.items(): + # Password file is already is already set in expected_vals dict + if key == "password": + continue + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=None, include_password=True) + + def test_get_mysql_no_server(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql` function with no server set. This should raise a TypeError. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.server = False + with pytest.raises(TypeError) as excinfo: + get_mysql() + assert f"Results backend: server False does not have a configuration" in str(excinfo.value) + + def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql` function with an invalid certs_path. This should raise a TypeError. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "invalid" + with pytest.raises(TypeError) as excinfo: + get_mysql(certs_path="invalid_path", mysql_certs=CERT_FILES) + err_msg = f"""The connection information for MySQL could not be set, cannot find:\n + {CERT_FILES}\ncheck the celery/certs path or set the ssl information in the app.yaml file.""" + assert err_msg in str(excinfo.value) + + def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with mysql as the results_backend. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config() == {"cert_reqs": CERT_NONE} + + def test_get_ssl_config_mysql_celery_check(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with mysql as the results_backend and celery_check set. + This should return False. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config(celery_check=True) is False + + def test_get_connection_string_mysql(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_connection_string` function with MySQL as the results_backend. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.celery.certs = merlin_server_dir + + create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + for key, cert_file in MYSQL_CONFIG_FILENAMES.items(): + # Password file is already is already set in expected_vals dict + if key == "password": + continue + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + + assert MYSQL_CONNECTION_STRING.format(**expected_vals) == get_connection_string(include_password=True) + + def test_get_connection_string_sqlite(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with sqlite as the results_backend. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "sqlite" + assert get_connection_string() == SQLITE_CONNECTION_STRING From 47a0b4e5f834d120cfe54ca095264f2ac622240b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 13:09:17 -0800 Subject: [PATCH 018/201] fix lint issues for most recent changes --- tests/conftest.py | 18 ++++++++---- tests/unit/common/test_encryption.py | 4 ++- tests/unit/config/test_results_backend.py | 36 +++++++++++++++-------- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 88eeaddb0..56ba762ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -101,7 +101,7 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): """ Check if cert files already exist and if they don't then create them. - :param cert_filepath: The path to the cert files + :param cert_filepath: The path to the cert files :param cert_files: A dict of certification files to create """ for cert_file in cert_files.values(): @@ -310,9 +310,13 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab CONFIG.broker.cert_reqs = "none" # Set the results_backend configuration for testing - CONFIG.results_backend.password = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.password = ( + None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + ) CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` - CONFIG.results_backend.name = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.name = ( + None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + ) CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` CONFIG.results_backend.server = "127.0.0.1" CONFIG.results_backend.username = "default" @@ -330,7 +334,9 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab @pytest.fixture(scope="function") -def redis_broker_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +def redis_broker_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis broker and results_backend. @@ -349,7 +355,9 @@ def redis_broker_config(merlin_server_dir: str, config: "fixture"): # noqa: F82 @pytest.fixture(scope="function") -def redis_results_backend_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +def redis_results_backend_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis results_backend. diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index d0069f09e..d797f68c0 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -87,7 +87,9 @@ def test_gen_key(self, temp_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_key( + self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture" # noqa: F821 + ): """ Test the `_get_key` function. diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 3531a83a2..80ce05657 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -2,10 +2,11 @@ Tests for the `results_backend.py` file. """ import os -import pytest from ssl import CERT_NONE from typing import Any, Dict +import pytest + from merlin.config.configfile import CONFIG from merlin.config.results_backend import ( MYSQL_CONFIG_FILENAMES, @@ -16,10 +17,11 @@ get_mysql, get_mysql_config, get_redis, - get_ssl_config + get_ssl_config, ) from tests.conftest import CERT_FILES, SERVER_PASS, create_cert_files, create_pass_file + RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" @@ -36,7 +38,7 @@ def test_get_backend_password_pass_file_in_merlin(): if not os.path.exists(path_to_merlin_dir): remove_merlin_dir_after_test = True os.mkdir(path_to_merlin_dir) - + # Create the test password file pass_filename = "test.pass" full_pass_filepath = f"{path_to_merlin_dir}/{pass_filename}" @@ -179,7 +181,7 @@ def test_get_redis_dont_include_password(self, redis_results_backend_config: "fi """ expected_vals = { "urlbase": "redis", - "spass": f"default:******@", + "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0, @@ -372,7 +374,7 @@ class TestMySQLResultsBackend: def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 """ - Test the `get_mysql_config` function with the certs dict getting set and returned. + Test the `get_mysql_config` function with the certs dict getting set and returned. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here :param merlin_server_dir: The directory that has the test certification files @@ -392,9 +394,11 @@ def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixtur """ assert get_mysql_config(None, None) == {"cert_reqs": CERT_NONE} - def test_get_mysql_config_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + def test_get_mysql_config_no_mysql_certs( + self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 + ): """ - Test the `get_mysql_config` function with no mysql certs dict. + Test the `get_mysql_config` function with no mysql certs dict. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here :param merlin_server_dir: The directory that has the test certification files @@ -411,7 +415,9 @@ def test_get_mysql_config_invalid_certs_path(self, mysql_results_backend_config: CONFIG.results_backend.name = "invalid" assert get_mysql_config("invalid/path", CERT_FILES) is False - def run_get_mysql(self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool): + def run_get_mysql( + self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool + ): """ Helper method for running tests for the `get_mysql` function. @@ -447,9 +453,13 @@ def test_get_mysql(self, mysql_results_backend_config: "fixture", merlin_server_ } for key, cert_file in CERT_FILES.items(): expected_vals[key] = f"{merlin_server_dir}/{cert_file}" - self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True) + self.run_get_mysql( + expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True + ) - def test_get_mysql_dont_include_password(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + def test_get_mysql_dont_include_password( + self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 + ): """ Test the `get_mysql` function but set include_password to False. This should * out the password. @@ -465,7 +475,9 @@ def test_get_mysql_dont_include_password(self, mysql_results_backend_config: "fi } for key, cert_file in CERT_FILES.items(): expected_vals[key] = f"{merlin_server_dir}/{cert_file}" - self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False) + self.run_get_mysql( + expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False + ) def test_get_mysql_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 """ @@ -502,7 +514,7 @@ def test_get_mysql_no_server(self, mysql_results_backend_config: "fixture"): # CONFIG.results_backend.server = False with pytest.raises(TypeError) as excinfo: get_mysql() - assert f"Results backend: server False does not have a configuration" in str(excinfo.value) + assert "Results backend: server False does not have a configuration" in str(excinfo.value) def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 """ From 91a3f2f2cd3babaa42003c396293b03322a25d05 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 16:31:19 -0800 Subject: [PATCH 019/201] fix filename issue in setup.cfg and move celeryadapter tests to integration suite --- setup.cfg | 2 +- tests/{unit/study => integration}/test_celeryadapter.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{unit/study => integration}/test_celeryadapter.py (100%) diff --git a/setup.cfg b/setup.cfg index 0eaa116ea..6b4278799 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,5 +29,5 @@ ignore_missing_imports=true [coverage:run] omit = - merlin/ascii.py + merlin/ascii_art.py merlin/config/celeryconfig.py diff --git a/tests/unit/study/test_celeryadapter.py b/tests/integration/test_celeryadapter.py similarity index 100% rename from tests/unit/study/test_celeryadapter.py rename to tests/integration/test_celeryadapter.py From 78b019b67a8ad426176168dab1a7479e71bb4090 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 16:32:53 -0800 Subject: [PATCH 020/201] add ssl filepaths to mysql config object --- tests/conftest.py | 15 +++++++++++++++ tests/unit/config/test_results_backend.py | 17 +++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 56ba762ff..e9d6027fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,7 @@ This module contains pytest fixtures to be used throughout the entire test suite. """ import os +import shutil from copy import copy from time import sleep from typing import Dict @@ -111,6 +112,17 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): pass +def create_app_yaml(app_yaml_filepath: str): + """ + Create a dummy app.yaml file at `app_yaml_filepath`. + + :param app_yaml_filepath: The location to create an app.yaml file at + """ + full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" + if not os.path.exists(full_app_yaml_filepath): + shutil.copy(f"{os.path.dirname(__file__)}/dummy_app.yaml", full_app_yaml_filepath) + + def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): """ Given configuration options for the broker and results_backend, update @@ -415,5 +427,8 @@ def mysql_results_backend_config( CONFIG.results_backend.password = pass_file CONFIG.results_backend.name = "mysql" CONFIG.results_backend.dbname = "test_mysql_db" + CONFIG.results_backend.keyfile = CERT_FILES["ssl_key"] + CONFIG.results_backend.certfile = CERT_FILES["ssl_cert"] + CONFIG.results_backend.ca_certs = CERT_FILES["ssl_ca"] yield diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 80ce05657..59e53a5ae 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -386,13 +386,16 @@ def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture actual = get_mysql_config(merlin_server_dir, CERT_FILES) assert actual == expected - def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture"): # noqa: F821 + def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 """ Test the `get_mysql_config` function with mysql_ssl being found. This should just return the ssl value that's found. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - assert get_mysql_config(None, None) == {"cert_reqs": CERT_NONE} + expected = {key: f"{temp_output_dir}/{cert_file}" for key, cert_file in CERT_FILES.items()} + expected["cert_reqs"] = CERT_NONE + assert get_mysql_config(None, None) == expected def test_get_mysql_config_no_mysql_certs( self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 @@ -529,14 +532,17 @@ def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixtu {CERT_FILES}\ncheck the celery/certs path or set the ssl information in the app.yaml file.""" assert err_msg in str(excinfo.value) - def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 """ Test the `get_ssl_config` function with mysql as the results_backend. This should return a dict of cert reqs with ssl.CERT_NONE as the value. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - assert get_ssl_config() == {"cert_reqs": CERT_NONE} + expected = {key: f"{temp_output_dir}/{cert_file}" for key, cert_file in CERT_FILES.items()} + expected["cert_reqs"] = CERT_NONE + assert get_ssl_config() == expected def test_get_ssl_config_mysql_celery_check(self, mysql_results_backend_config: "fixture"): # noqa: F821 """ @@ -557,6 +563,9 @@ def test_get_connection_string_mysql(self, mysql_results_backend_config: "fixtur CONFIG.celery.certs = merlin_server_dir create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + CONFIG.results_backend.keyfile = MYSQL_CONFIG_FILENAMES["ssl_key"] + CONFIG.results_backend.certfile = MYSQL_CONFIG_FILENAMES["ssl_cert"] + CONFIG.results_backend.ca_certs = MYSQL_CONFIG_FILENAMES["ssl_ca"] expected_vals = { "cert_reqs": CERT_NONE, From 275fbd474cc260ca755785b102969a7b5615515e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 09:24:17 -0800 Subject: [PATCH 021/201] add unit tests for configfile.py --- tests/conftest.py | 12 - tests/unit/config/dummy_app.yaml | 33 + tests/unit/config/old_test_configfile.py | 96 --- tests/unit/config/old_test_results_backend.py | 66 -- tests/unit/config/test_configfile.py | 705 ++++++++++++++++++ 5 files changed, 738 insertions(+), 174 deletions(-) create mode 100644 tests/unit/config/dummy_app.yaml delete mode 100644 tests/unit/config/old_test_configfile.py delete mode 100644 tests/unit/config/old_test_results_backend.py create mode 100644 tests/unit/config/test_configfile.py diff --git a/tests/conftest.py b/tests/conftest.py index e9d6027fd..446e3118b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,7 +31,6 @@ This module contains pytest fixtures to be used throughout the entire test suite. """ import os -import shutil from copy import copy from time import sleep from typing import Dict @@ -112,17 +111,6 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): pass -def create_app_yaml(app_yaml_filepath: str): - """ - Create a dummy app.yaml file at `app_yaml_filepath`. - - :param app_yaml_filepath: The location to create an app.yaml file at - """ - full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" - if not os.path.exists(full_app_yaml_filepath): - shutil.copy(f"{os.path.dirname(__file__)}/dummy_app.yaml", full_app_yaml_filepath) - - def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): """ Given configuration options for the broker and results_backend, update diff --git a/tests/unit/config/dummy_app.yaml b/tests/unit/config/dummy_app.yaml new file mode 100644 index 000000000..966156566 --- /dev/null +++ b/tests/unit/config/dummy_app.yaml @@ -0,0 +1,33 @@ +broker: + cert_reqs: none + name: redis + password: redis.pass + port: '6379' + server: 127.0.0.1 + username: default + vhost: host4gunny +celery: + override: + visibility_timeout: 86400 +container: + config: redis.conf + config_dir: ./merlin_server/ + format: singularity + image: redis_latest.sif + image_type: redis + pass_file: redis.pass + pfile: merlin_server.pf + url: docker://redis + user_file: redis.users +process: + kill: kill {pid} + status: pgrep -P {pid} +results_backend: + cert_reqs: none + db_num: 0 + encryption_key: encrypt_data_key + name: redis + password: redis.pass + port: '6379' + server: 127.0.0.1 + username: default \ No newline at end of file diff --git a/tests/unit/config/old_test_configfile.py b/tests/unit/config/old_test_configfile.py deleted file mode 100644 index 1ee970531..000000000 --- a/tests/unit/config/old_test_configfile.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Tests for the configfile module.""" -import os -import shutil -import tempfile -import unittest -from getpass import getuser - -from merlin.config import configfile - -from .utils import mkfile - - -CONFIG_FILE_CONTENTS = """ -celery: - certs: path/to/celery/config/files - -broker: - name: rabbitmq - username: testuser - password: rabbit.password # The filename that contains the password. - server: jackalope.llnl.gov - -results_backend: - name: mysql - dbname: testuser - username: mlsi - password: mysql.password # The filename that contains the password. - server: rabbit.llnl.gov - -""" - - -class TestFindConfigFile(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.appfile = mkfile(self.tmpdir, "app.yaml") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_tempdir(self): - self.assertTrue(os.path.isdir(self.tmpdir)) - - def test_find_config_file(self): - """ - Given the path to a vaild config file, find and return the full - filepath. - """ - path = configfile.find_config_file(path=self.tmpdir) - expected = os.path.join(self.tmpdir, self.appfile) - self.assertEqual(path, expected) - - def test_find_config_file_error(self): - """Given an invalid path, return None.""" - invalid = "invalid/path" - expected = None - - path = configfile.find_config_file(path=invalid) - self.assertEqual(path, expected) - - -class TestConfigFile(unittest.TestCase): - """Unit tests for loading the config file.""" - - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.configfile = mkfile(self.tmpdir, "app.yaml", content=CONFIG_FILE_CONTENTS) - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_get_config(self): - """ - Given the directory path to a valid merlin config file, then - `get_config` should find the merlin config file and load the YAML - contents to a dictionary. - """ - expected = { - "broker": { - "name": "rabbitmq", - "password": "rabbit.password", - "server": "jackalope.llnl.gov", - "username": "testuser", - "vhost": getuser(), - }, - "celery": {"certs": "path/to/celery/config/files"}, - "results_backend": { - "dbname": "testuser", - "name": "mysql", - "password": "mysql.password", - "server": "rabbit.llnl.gov", - "username": "mlsi", - }, - } - - self.assertDictEqual(configfile.get_config(self.tmpdir), expected) diff --git a/tests/unit/config/old_test_results_backend.py b/tests/unit/config/old_test_results_backend.py deleted file mode 100644 index d6e0c2f22..000000000 --- a/tests/unit/config/old_test_results_backend.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Tests for the results_backend module.""" -import os -import shutil -import tempfile -import unittest - -from merlin.config import results_backend - -from .utils import mkfile - - -class TestResultsBackend(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - - # Create test files. - self.tmpfile1 = mkfile(self.tmpdir, "mysql_test1.txt") - self.tmpfile2 = mkfile(self.tmpdir, "mysql_test2.txt") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_mysql_config(self): - """ - Given the path to a directory containing the MySQL cert files and a - dictionary of files to look for, then find and return the full path to - all the certs. - """ - certs = {"test1": "mysql_test1.txt", "test2": "mysql_test2.txt"} - - # This will just be the above dictionary with the full file paths. - expected = { - "test1": os.path.join(self.tmpdir, certs["test1"]), - "test2": os.path.join(self.tmpdir, certs["test2"]), - } - results = results_backend.get_mysql_config(self.tmpdir, certs) - self.assertDictEqual(results, expected) - - def test_mysql_config_no_files(self): - """ - Given the path to a directory containing the MySQL cert files and - an empty dictionary, then `get_mysql_config` should return an empty - dictionary. - """ - files = {} - result = results_backend.get_mysql_config(self.tmpdir, files) - self.assertEqual(result, {}) - - -class TestConfingMysqlErrorPath(unittest.TestCase): - """ - Test `get_mysql_config` against cases were the given path does not exist. - """ - - def test_mysql_config_false(self): - """ - Given a path that does not exist, then `get_mysql_config` should return - False. - """ - path = "invalid/path" - - # We don't need the dictionary populated for this test. The function - # should return False before trying to process the dictionary. - certs = {} - result = results_backend.get_mysql_config(path, certs) - self.assertFalse(result) diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py new file mode 100644 index 000000000..5d635e79b --- /dev/null +++ b/tests/unit/config/test_configfile.py @@ -0,0 +1,705 @@ +""" +Tests for the configfile.py module. +""" +import getpass +import os +import shutil +import ssl +from copy import copy, deepcopy + +import pytest +import yaml + +from merlin.config.configfile import ( + CONFIG, + default_config_info, + find_config_file, + get_cert_file, + get_config, + get_ssl_entries, + is_debug, + load_config, + load_default_celery, + load_default_user_names, + load_defaults, + merge_sslmap, + process_ssl_map, +) +from tests.conftest import CERT_FILES + + +CONFIGFILE_DIR = "{temp_output_dir}/test_configfile" +COPIED_APP_FILENAME = "app_copy.yaml" +DUMMY_APP_FILEPATH = f"{os.path.dirname(__file__)}/dummy_app.yaml" + + +def create_configfile_dir(temp_output_dir: str): + """ + Create the configfile dir if it doesn't exist yet. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + full_configfile_dirpath = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + if not os.path.exists(full_configfile_dirpath): + os.mkdir(full_configfile_dirpath) + + +def create_app_yaml(app_yaml_filepath: str): + """ + Create a dummy app.yaml file at `app_yaml_filepath`. + + :param app_yaml_filepath: The location to create an app.yaml file at + """ + full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" + if not os.path.exists(full_app_yaml_filepath): + shutil.copy(DUMMY_APP_FILEPATH, full_app_yaml_filepath) + + +def test_load_config(temp_output_dir: str): + """ + Test the `load_config` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: + expected = yaml.load(dummy_app_file, yaml.Loader) + + actual = load_config(f"{configfile_dir}/app.yaml") + assert actual == expected + + +def test_load_config_invalid_file(): + """ + Test the `load_config` function with an invalid filepath. + """ + assert load_config("invalid/filepath") is None + + +def test_find_config_file_valid_path(temp_output_dir: str): + """ + Test the `find_config_file` function with passing a valid path in. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + assert find_config_file(configfile_dir) == f"{configfile_dir}/app.yaml" + + +def test_find_config_file_invalid_path(): + """ + Test the `find_config_file` function with passing an invalid path in. + """ + assert find_config_file("invalid/path") is None + + +def test_find_config_file_local_path(temp_output_dir: str): + """ + Test the `find_config_file` function by having it find a local (in our cwd) app.yaml file. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + # Move into the configfile directory and run the test + os.chdir(configfile_dir) + try: + assert find_config_file() == f"{os.getcwd()}/app.yaml" + except AssertionError as exc: + # Move back to the temp output directory even if the test fails + os.chdir(temp_output_dir) + raise AssertionError from exc + + # Move back to the temp output directory + os.chdir(temp_output_dir) + + +def test_find_config_file_merlin_home_path(temp_output_dir: str): + """ + Test the `find_config_file` function by having it find an app.yaml file in our merlin directory. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + merlin_home = os.path.expanduser("~/.merlin") + if not os.path.exists(merlin_home): + os.mkdir(merlin_home) + create_app_yaml(merlin_home) + assert find_config_file() == f"{merlin_home}/app.yaml" + + +def check_for_and_move_app_yaml(dir_to_check: str) -> bool: + """ + Check for any app.yaml files in `dir_to_check`. If one is found, rename it. + Return True if an app.yaml was found, false otherwise. + + :param dir_to_check: The directory to search for an app.yaml in + :returns: True if an app.yaml was found. False otherwise. + """ + for filename in os.listdir(dir_to_check): + full_path = os.path.join(dir_to_check, filename) + if os.path.isfile(full_path) and filename == "app.yaml": + os.rename(full_path, f"{dir_to_check}/{COPIED_APP_FILENAME}") + return True + return False + + +def test_find_config_file_no_path(temp_output_dir: str): + """ + Test the `find_config_file` function by making it unable to find any app.yaml path. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Rename any app.yaml in the cwd + cwd_path = os.getcwd() + cwd_had_app_yaml = check_for_and_move_app_yaml(cwd_path) + + # Rename any app.yaml in the merlin home directory + merlin_home_dir = os.path.expanduser("~/.merlin") + merlin_home_had_app_yaml = check_for_and_move_app_yaml(merlin_home_dir) + + try: + assert find_config_file() is None + except AssertionError as exc: + # Reset the cwd app.yaml even if the test fails + if cwd_had_app_yaml: + os.rename(f"{cwd_path}/{COPIED_APP_FILENAME}", f"{cwd_path}/app.yaml") + + # Reset the merlin home app.yaml even if the test fails + if merlin_home_had_app_yaml: + os.rename(f"{merlin_home_dir}/{COPIED_APP_FILENAME}", f"{merlin_home_dir}/app.yaml") + + raise AssertionError from exc + + # Reset the cwd app.yaml + if cwd_had_app_yaml: + os.rename(f"{cwd_path}/{COPIED_APP_FILENAME}", f"{cwd_path}/app.yaml") + + # Reset the merlin home app.yaml + if merlin_home_had_app_yaml: + os.rename(f"{merlin_home_dir}/{COPIED_APP_FILENAME}", f"{merlin_home_dir}/app.yaml") + + +def test_load_default_user_names_nothing_to_load(): + """ + Test the `load_default_user_names` function with nothing to load. In other words, in this + test the config dict will have a username and vhost already set for the broker. We'll + create the dict then make a copy of it to test against after calling the function. + """ + actual_config = {"broker": {"username": "default", "vhost": "host4testing"}} + expected_config = deepcopy(actual_config) + assert actual_config is not expected_config + + load_default_user_names(actual_config) + + # Ensure that nothing was modified after our call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_user_names_no_username(): + """ + Test the `load_default_user_names` function with no username. In other words, in this + test the config dict will have vhost already set for the broker but not a username. + """ + expected_config = {"broker": {"username": getpass.getuser(), "vhost": "host4testing"}} + actual_config = {"broker": {"vhost": "host4testing"}} + load_default_user_names(actual_config) + + # Ensure that the username was set in the call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_user_names_no_vhost(): + """ + Test the `load_default_user_names` function with no vhost. In other words, in this + test the config dict will have username already set for the broker but not a vhost. + """ + expected_config = {"broker": {"username": "default", "vhost": getpass.getuser()}} + actual_config = {"broker": {"username": "default"}} + load_default_user_names(actual_config) + + # Ensure that the vhost was set in the call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_celery_nothing_to_load(): + """ + Test the `load_default_celery` function with nothing to load. In other words, in this + test the config dict will have a celery entry containing omit_queue_tag, queue_tag, and + override. We'll create the dict then make a copy of it to test against after calling + the function. + """ + actual_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + expected_config = deepcopy(actual_config) + assert actual_config is not expected_config + + load_default_celery(actual_config) + + # Ensure that nothing was modified after our call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_omit_queue_tag(): + """ + Test the `load_default_celery` function with no omit_queue_tag. The function should + create a default entry of False for this. + """ + actual_config = {"celery": {"queue_tag": "[merlin]_", "override": None}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the omit_queue_tag was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_queue_tag(): + """ + Test the `load_default_celery` function with no queue_tag. The function should + create a default entry of '[merlin]_' for this. + """ + actual_config = {"celery": {"omit_queue_tag": False, "override": None}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the queue_tag was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_override(): + """ + Test the `load_default_celery` function with no override. The function should + create a default entry of None for this. + """ + actual_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_"}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the override was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_celery_block(): + """ + Test the `load_default_celery` function with no celery block. The function should + create a default entry of + {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} for this. + """ + actual_config = {} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the celery block was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_defaults(): + """ + Test that the `load_defaults` function loads the user names and the celery block properly. + """ + actual_config = {"broker": {}} + expected_config = { + "broker": {"username": getpass.getuser(), "vhost": getpass.getuser()}, + "celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}, + } + load_defaults(actual_config) + + assert actual_config == expected_config + + +def test_get_config(temp_output_dir: str): + """ + Test the `get_config` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + # Load up the contents of the dummy app.yaml file that we copied + with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: + expected = yaml.load(dummy_app_file, yaml.Loader) + + # Add in default settings that should be added + expected["celery"]["omit_queue_tag"] = False + expected["celery"]["queue_tag"] = "[merlin]_" + + actual = get_config(configfile_dir) + + assert actual == expected + + +def test_get_config_invalid_path(): + """ + Test the `get_config` function with an invalid path. This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + get_config("invalid/path") + + assert "Cannot find a merlin config file!" in str(excinfo.value) + + +def test_is_debug_no_merlin_debug(): + """ + Test the `is_debug` function without having MERLIN_DEBUG in the environment. + This should return False. + """ + + # Delete the current val of MERLIN_DEBUG and store it (if there is one) + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + del os.environ["MERLIN_DEBUG"] + reset_merlin_debug = True + + # Run the test + try: + assert is_debug() is False + except AssertionError as exc: + # Make sure to reset the value of MERLIN_DEBUG even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + raise AssertionError from exc + + # Reset the value of MERLIN_DEBUG + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + + +def test_is_debug_with_merlin_debug(): + """ + Test the `is_debug` function with having MERLIN_DEBUG in the environment. + This should return True. + """ + + # Grab the current value of MERLIN_DEBUG if there is one + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ and int(os.environ["MERLIN_DEBUG"]) != 1: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + reset_merlin_debug = True + + # Set the MERLIN_DEBUG value to be 1 + os.environ["MERLIN_DEBUG"] = "1" + + try: + assert is_debug() is True + except AssertionError as exc: + # Make sure to reset the value of MERLIN_DEBUG even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + raise AssertionError from exc + + # Reset the value of MERLIN_DEBUG + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + + +def test_default_config_info(temp_output_dir: str): + """ + Test the `default_config_info` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + cwd = os.getcwd() + os.chdir(configfile_dir) + + # Delete the current val of MERLIN_DEBUG and store it (if there is one) + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + del os.environ["MERLIN_DEBUG"] + reset_merlin_debug = True + + # Create the merlin home directory if it doesn't already exist + merlin_home = f"{os.path.expanduser('~')}/.merlin" + remove_merlin_home = False + if not os.path.exists(merlin_home): + os.mkdir(merlin_home) + remove_merlin_home = True + + # Run the test + try: + expected = { + "config_file": f"{configfile_dir}/app.yaml", + "is_debug": False, + "merlin_home": merlin_home, + "merlin_home_exists": True, + } + actual = default_config_info() + assert actual == expected + except AssertionError as exc: + # Make sure to reset values even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + if remove_merlin_home: + os.rmdir(merlin_home) + raise AssertionError from exc + + # Reset values if necessary + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + if remove_merlin_home: + os.rmdir(merlin_home) + + os.chdir(cwd) + + +def test_get_cert_file_all_valid_args(mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_cert_file` function with all valid arguments. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + expected = f"{merlin_server_dir}/{CERT_FILES['ssl_key']}" + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="keyfile", cert_path=merlin_server_dir + ) + assert actual == expected + + +def test_get_cert_file_invalid_cert_name(mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_cert_file` function with an invalid cert_name argument. This should just return None. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="invalid", cert_path=merlin_server_dir + ) + assert actual is None + + +def test_get_cert_file_nonexistent_cert_path( + mysql_results_backend_config: "fixture", temp_output_dir: str, merlin_server_dir: str # noqa: F821 +): + """ + Test the `get_cert_file` function with cert_path argument that doesn't exist. + This should still return the nonexistent path at the root of our temporary directory for testing. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + CONFIG.results_backend.certfile = "new_certfile.pem" + expected = f"{temp_output_dir}/new_certfile.pem" + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="certfile", cert_path=merlin_server_dir + ) + assert actual == expected + + +def test_get_ssl_entries_required_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll make + cert reqs be required. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "required" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_REQUIRED, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_optional_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll make + cert reqs be optional. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "optional" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_OPTIONAL, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_none_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we won't require + any cert reqs. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "none" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_NONE, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_omit_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll completely + omit the cert_reqs option + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + del CONFIG.results_backend.cert_reqs + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_REQUIRED, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_with_ssl_protocol(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll add in a + dummy ssl_protocol value that should get added to the dict that's output. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + protocol = "test_protocol" + CONFIG.results_backend.ssl_protocol = protocol + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_NONE, + "ssl_protocol": protocol, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_process_ssl_map_mysql(): + """Test the `process_ssl_map` function with mysql as the server name.""" + expected = {"keyfile": "ssl_key", "certfile": "ssl_cert", "ca_certs": "ssl_ca"} + actual = process_ssl_map("mysql") + assert actual == expected + + +def test_process_ssl_map_rediss(): + """Test the `process_ssl_map` function with rediss as the server name.""" + expected = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = process_ssl_map("rediss") + assert actual == expected + + +def test_merge_sslmap_all_keys_present(): + """ + Test the `merge_sslmap` function with all keys from server_ssl in ssl_map. + We'll assume we're using a rediss server for this. + """ + expected = { + "ssl_keyfile": "/path/to/keyfile", + "ssl_certfile": "/path/to/certfile", + "ssl_ca_certs": "/path/to/ca_file", + "ssl_cert_reqs": ssl.CERT_NONE, + } + test_server_ssl = { + "keyfile": "/path/to/keyfile", + "certfile": "/path/to/certfile", + "ca_certs": "/path/to/ca_file", + "cert_reqs": ssl.CERT_NONE, + } + test_ssl_map = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = merge_sslmap(test_server_ssl, test_ssl_map) + assert actual == expected + + +def test_merge_sslmap_some_keys_present(): + """ + Test the `merge_sslmap` function with some keys from server_ssl in ssl_map and others not. + We'll assume we're using a rediss server for this. + """ + expected = { + "ssl_keyfile": "/path/to/keyfile", + "ssl_certfile": "/path/to/certfile", + "ssl_ca_certs": "/path/to/ca_file", + "ssl_cert_reqs": ssl.CERT_NONE, + "new_key": "new_val", + "second_new_key": "second_new_val", + } + test_server_ssl = { + "keyfile": "/path/to/keyfile", + "certfile": "/path/to/certfile", + "ca_certs": "/path/to/ca_file", + "cert_reqs": ssl.CERT_NONE, + "new_key": "new_val", + "second_new_key": "second_new_val", + } + test_ssl_map = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = merge_sslmap(test_server_ssl, test_ssl_map) + assert actual == expected From 9669bd0cc559be15c689512016a14fe8cfd3c544 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 10:40:13 -0800 Subject: [PATCH 022/201] add tests for the utils.py file in config/ --- tests/unit/config/test_utils.py | 83 +++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/unit/config/test_utils.py diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py new file mode 100644 index 000000000..a02bc1ff1 --- /dev/null +++ b/tests/unit/config/test_utils.py @@ -0,0 +1,83 @@ +""" +Tests for the merlin/config/utils.py module. +""" + +import pytest + +from merlin.config.configfile import CONFIG +from merlin.config.utils import Priority, get_priority, is_rabbit_broker, is_redis_broker + + +def test_is_rabbit_broker(): + """Test the `is_rabbit_broker` by passing in rabbit as the broker""" + assert is_rabbit_broker("rabbitmq") is True + assert is_rabbit_broker("amqp") is True + assert is_rabbit_broker("amqps") is True + + +def test_is_rabbit_broker_invalid(): + """Test the `is_rabbit_broker` by passing in an invalid broker""" + assert is_rabbit_broker("redis") is False + assert is_rabbit_broker("") is False + + +def test_is_redis_broker(): + """Test the `is_redis_broker` by passing in redis as the broker""" + assert is_redis_broker("redis") is True + assert is_redis_broker("rediss") is True + assert is_redis_broker("redis+socket") is True + + +def test_is_redis_broker_invalid(): + """Test the `is_redis_broker` by passing in an invalid broker""" + assert is_redis_broker("rabbitmq") is False + assert is_redis_broker("") is False + + +def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with rabbit as the broker. + Low priority for rabbit is 1 and high is 10. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_priority(Priority.LOW) == 1 + assert get_priority(Priority.MID) == 5 + assert get_priority(Priority.HIGH) == 10 + + +def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with redis as the broker. + Low priority for redis is 10 and high is 1. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_priority(Priority.LOW) == 10 + assert get_priority(Priority.MID) == 5 + assert get_priority(Priority.HIGH) == 1 + + +def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with an invalid broker. + This should raise a ValueError. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "invalid" + with pytest.raises(ValueError) as excinfo: + get_priority(Priority.LOW) + assert "Function get_priority has reached unknown state! Maybe unsupported broker invalid?" in str(excinfo.value) + + +def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with an invalid priority. + This should raise a TypeError. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + with pytest.raises(TypeError) as excinfo: + get_priority("invalid_priority") + assert "Unrecognized priority 'invalid_priority'!" in str(excinfo.value) From 2a209e58056f5c2091cc0f066869ae1dff4dcb70 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 13:10:02 -0800 Subject: [PATCH 023/201] create utilities file and constants file --- tests/conftest.py | 60 +---------------------- tests/constants.py | 10 ++++ tests/unit/config/test_broker.py | 3 +- tests/unit/config/test_results_backend.py | 3 +- tests/unit/config/utils.py | 23 --------- tests/utils.py | 35 +++++++++++++ 6 files changed, 51 insertions(+), 83 deletions(-) create mode 100644 tests/constants.py delete mode 100644 tests/unit/config/utils.py create mode 100644 tests/utils.py diff --git a/tests/conftest.py b/tests/conftest.py index 446e3118b..20749d4cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,16 +42,10 @@ from celery.canvas import Signature from merlin.config.configfile import CONFIG +from tests.constants import SERVER_PASS, CERT_FILES from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager - - -SERVER_PASS = "merlin-test-server" -CERT_FILES = { - "ssl_cert": "test-rabbit-client-cert.pem", - "ssl_ca": "test-mysql-ca-cert.pem", - "ssl_key": "test-rabbit-client-key.pem", -} +from tests.utils import create_cert_files, create_pass_file ####################################### @@ -59,18 +53,6 @@ ####################################### -def create_pass_file(pass_filepath: str): - """ - Check if a password file already exists (it will if the redis server has been started) - and if it hasn't then create one and write the password to the file. - - :param pass_filepath: The path to the password file that we need to check for/create - """ - if not os.path.exists(pass_filepath): - with open(pass_filepath, "w") as pass_file: - pass_file.write(SERVER_PASS) - - def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_filepath: str = None): """ Check if an encryption file already exists (it will if the redis server has been started) @@ -97,44 +79,6 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi yaml.dump(app_yaml, app_yaml_file) -def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): - """ - Check if cert files already exist and if they don't then create them. - - :param cert_filepath: The path to the cert files - :param cert_files: A dict of certification files to create - """ - for cert_file in cert_files.values(): - full_cert_filepath = f"{cert_filepath}/{cert_file}" - if not os.path.exists(full_cert_filepath): - with open(full_cert_filepath, "w"): - pass - - -def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): - """ - Given configuration options for the broker and results_backend, update - the CONFIG object. - - :param broker: A dict of the configuration settings for the broker - :param results_backend: A dict of configuration settings for the results_backend - """ - # Set the broker configuration for testing - CONFIG.broker.password = broker["password"] - CONFIG.broker.port = broker["port"] - CONFIG.broker.server = broker["server"] - CONFIG.broker.username = broker["username"] - CONFIG.broker.vhost = broker["vhost"] - CONFIG.broker.name = broker["name"] - - # Set the results_backend configuration for testing - CONFIG.results_backend.password = results_backend["password"] - CONFIG.results_backend.port = results_backend["port"] - CONFIG.results_backend.server = results_backend["server"] - CONFIG.results_backend.username = results_backend["username"] - CONFIG.results_backend.encryption_key = results_backend["encryption_key"] - - ####################################### ######### Fixture Definitions ######### ####################################### diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 000000000..a2b354146 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,10 @@ +""" +This module will store constants that will be used throughout our test suite. +""" + +SERVER_PASS = "merlin-test-server" +CERT_FILES = { + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem", +} \ No newline at end of file diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 490b47649..8af1dda75 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -18,7 +18,8 @@ read_file, ) from merlin.config.configfile import CONFIG -from tests.conftest import SERVER_PASS, create_pass_file +from tests.constants import SERVER_PASS +from tests.utils import create_pass_file def test_read_file(merlin_server_dir: str): diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 59e53a5ae..314df6ce7 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -19,7 +19,8 @@ get_redis, get_ssl_config, ) -from tests.conftest import CERT_FILES, SERVER_PASS, create_cert_files, create_pass_file +from tests.constants import CERT_FILES, SERVER_PASS +from tests.utils import create_cert_files, create_pass_file RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" diff --git a/tests/unit/config/utils.py b/tests/unit/config/utils.py deleted file mode 100644 index 11510c5fd..000000000 --- a/tests/unit/config/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Utils module for common test functionality. -""" -import os - - -def mkfile(tmpdir, filename, content=""): - """ - A simple function for creating a file and returning the path. This is to - abstract out file creation logic in the tests. - - :param tmpdir: (str) The path to the temp directory. - :param filename: (str) The name of the file. - :param contents: (str) Optional contents to write to the file. Defaults to - an empty string. - :returns: (str) The appended path of the given tempdir and filename. - """ - filepath = os.path.join(tmpdir, filename) - - with open(filepath, "w") as f: - f.write(content) - - return filepath diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..51fbd56cc --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,35 @@ +""" +Utility functions for our test suite. +""" +import os +from typing import Dict + +from tests.constants import SERVER_PASS + + +def create_pass_file(pass_filepath: str): + """ + Check if a password file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the password to the file. + + :param pass_filepath: The path to the password file that we need to check for/create + """ + if not os.path.exists(pass_filepath): + with open(pass_filepath, "w") as pass_file: + pass_file.write(SERVER_PASS) + + +def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): + """ + Check if cert files already exist and if they don't then create them. + + :param cert_filepath: The path to the cert files + :param cert_files: A dict of certification files to create + """ + for cert_file in cert_files.values(): + full_cert_filepath = f"{cert_filepath}/{cert_file}" + if not os.path.exists(full_cert_filepath): + with open(full_cert_filepath, "w"): + pass + + From ddb0588cd9170a077ded446a6fb9072a4e563add Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 16:42:23 -0800 Subject: [PATCH 024/201] move create_dir function to utils.py --- tests/unit/config/test_configfile.py | 24 +++++++----------------- tests/utils.py | 9 +++++++++ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py index 5d635e79b..aeb1da941 100644 --- a/tests/unit/config/test_configfile.py +++ b/tests/unit/config/test_configfile.py @@ -25,7 +25,8 @@ merge_sslmap, process_ssl_map, ) -from tests.conftest import CERT_FILES +from tests.constants import CERT_FILES +from tests.utils import create_dir CONFIGFILE_DIR = "{temp_output_dir}/test_configfile" @@ -33,17 +34,6 @@ DUMMY_APP_FILEPATH = f"{os.path.dirname(__file__)}/dummy_app.yaml" -def create_configfile_dir(temp_output_dir: str): - """ - Create the configfile dir if it doesn't exist yet. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - full_configfile_dirpath = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) - if not os.path.exists(full_configfile_dirpath): - os.mkdir(full_configfile_dirpath) - - def create_app_yaml(app_yaml_filepath: str): """ Create a dummy app.yaml file at `app_yaml_filepath`. @@ -61,8 +51,8 @@ def test_load_config(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: @@ -85,8 +75,8 @@ def test_find_config_file_valid_path(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) assert find_config_file(configfile_dir) == f"{configfile_dir}/app.yaml" @@ -109,8 +99,8 @@ def test_find_config_file_local_path(temp_output_dir: str): """ # Create the configfile directory and put an app.yaml file there - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) # Move into the configfile directory and run the test @@ -330,8 +320,8 @@ def test_get_config(temp_output_dir: str): """ # Create the configfile directory and put an app.yaml file there - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) # Load up the contents of the dummy app.yaml file that we copied @@ -422,8 +412,8 @@ def test_default_config_info(temp_output_dir: str): """ # Create the configfile directory and put an app.yaml file there - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) cwd = os.getcwd() os.chdir(configfile_dir) diff --git a/tests/utils.py b/tests/utils.py index 51fbd56cc..3a75622b8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,3 +33,12 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): pass +def create_dir(dirpath: str): + """ + Check if `dirpath` exists and if it doesn't then create it. + + :param dirpath: The directory to create + """ + if not os.path.exists(dirpath): + os.mkdir(dirpath) + From e5bc0fe239df705ce536ae67e5264c737e3f4176 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 16:42:52 -0800 Subject: [PATCH 025/201] add tests for merlin/examples/generator.py --- merlin/examples/generator.py | 8 + setup.cfg | 1 + tests/unit/test_examples_generator.py | 575 ++++++++++++++++++++++++++ 3 files changed, 584 insertions(+) create mode 100644 tests/unit/test_examples_generator.py diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 2fa5e61ce..f1e58f430 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -48,6 +48,14 @@ EXAMPLES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "workflows") +# TODO modify the example command to eliminate redundancy +# - e.g. running `merlin example flux_local` will produce the same output +# as running `merlin example flux_par` or `merlin example flux_par_restart`. +# This should just be `merlin example flux`. +# - restart and restart delay should be one example +# - feature demo and remote feature demo should be one example +# - all openfoam examples should just be under one openfoam label + def gather_example_dirs(): """Get all the example directories""" diff --git a/setup.cfg b/setup.cfg index 6b4278799..77ac2d84f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,3 +31,4 @@ ignore_missing_imports=true omit = merlin/ascii_art.py merlin/config/celeryconfig.py + merlin/examples/examples.py diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py new file mode 100644 index 000000000..7d7ccc5bf --- /dev/null +++ b/tests/unit/test_examples_generator.py @@ -0,0 +1,575 @@ +""" +Tests for the `merlin/examples/generator.py` module. +""" +import os +import pathlib +from typing import List + +from tabulate import tabulate + +from merlin.examples.generator import ( + EXAMPLES_DIR, + gather_all_examples, + gather_example_dirs, + list_examples, + setup_example, + write_example +) +from tests.utils import create_dir + + +EXAMPLES_GENERATOR_DIR = "{temp_output_dir}/examples_generator" + + +def test_gather_example_dirs(): + """Test the `gather_example_dirs` function.""" + example_workflows = [ + "feature_demo", + "flux", + "hello", + "hpc_demo", + "iterative_demo", + "lsf", + "null_spec", + "openfoam_wf", + "openfoam_wf_no_docker", + "openfoam_wf_singularity", + "optimization", + "remote_feature_demo", + "restart", + "restart_delay", + "simple_chain", + "slurm" + ] + expected = {} + for wf_dir in example_workflows: + expected[wf_dir] = wf_dir + actual = gather_example_dirs() + assert actual == expected + + +def test_gather_all_examples(): + """Test the `gather_all_examples` function.""" + expected = [ + f"{EXAMPLES_DIR}/feature_demo/feature_demo.yaml", + f"{EXAMPLES_DIR}/flux/flux_local.yaml", + f"{EXAMPLES_DIR}/flux/flux_par_restart.yaml", + f"{EXAMPLES_DIR}/flux/flux_par.yaml", + f"{EXAMPLES_DIR}/flux/paper.yaml", + f"{EXAMPLES_DIR}/hello/hello_samples.yaml", + f"{EXAMPLES_DIR}/hello/hello.yaml", + f"{EXAMPLES_DIR}/hello/my_hello.yaml", + f"{EXAMPLES_DIR}/hpc_demo/hpc_demo.yaml", + f"{EXAMPLES_DIR}/iterative_demo/iterative_demo.yaml", + f"{EXAMPLES_DIR}/lsf/lsf_par_srun.yaml", + f"{EXAMPLES_DIR}/lsf/lsf_par.yaml", + f"{EXAMPLES_DIR}/null_spec/null_chain.yaml", + f"{EXAMPLES_DIR}/null_spec/null_spec.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity.yaml", + f"{EXAMPLES_DIR}/optimization/optimization_basic.yaml", + f"{EXAMPLES_DIR}/remote_feature_demo/remote_feature_demo.yaml", + f"{EXAMPLES_DIR}/restart/restart.yaml", + f"{EXAMPLES_DIR}/restart_delay/restart_delay.yaml", + f"{EXAMPLES_DIR}/simple_chain/simple_chain.yaml", + f"{EXAMPLES_DIR}/slurm/slurm_par_restart.yaml", + f"{EXAMPLES_DIR}/slurm/slurm_par.yaml" + ] + actual = gather_all_examples() + assert sorted(actual) == sorted(expected) + + +def test_write_example_dir(temp_output_dir: str): + """ + Test the `write_example` function with the src_path as a directory. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + dir_to_copy = f"{EXAMPLES_DIR}/feature_demo/" + + write_example(dir_to_copy, generator_dir) + assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(generator_dir)) + + +def test_write_example_file(temp_output_dir: str): + """ + Test the `write_example` function with the src_path as a file. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + + dst_path = f"{generator_dir}/flux_par.yaml" + file_to_copy = f"{EXAMPLES_DIR}/flux/flux_par.yaml" + + write_example(file_to_copy, generator_dir) + assert os.path.exists(dst_path) + + +def test_list_examples(): + """Test the `list_examples` function to see if it gives us all of the examples that we want.""" + expected_headers = ["name", "description"] + expected_rows = [ + ["openfoam_wf_no_docker", "A parameter study that includes initializing, running,\n" \ + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" \ + "without using docker."], + ["optimization_basic", "Design Optimization Template\n" \ + "To use,\n" \ + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" \ + "2. Run the template_config file in current directory using `python template_config.py`\n" \ + "3. Merlin run as usual (merlin run optimization.yaml)\n" \ + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" \ + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts"], + ["feature_demo", "Run 10 hello worlds."], + ["flux_local", "Run a scan through Merlin/Maestro"], + ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], + ["flux_par_restart", "A simple ensemble of parallel MPI jobs run by flux."], + ["paper_flux", "Use flux to run single core MPI jobs and record timings."], + ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], + ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], + ["restart", "A simple ensemble of with restarts."], + ["restart_delay", "A simple ensemble of with restart delay times."], + ["simple_chain", "test to see that chains are not run in parallel"], + ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["remote_feature_demo", "Run 10 hello worlds."], + ["hello", "a very simple merlin workflow"], + ["hello_samples", "a very simple merlin workflow, with samples"], + ["hpc_demo", "Demo running a workflow on HPC machines"], + ["openfoam_wf", "A parameter study that includes initializing, running,\n" \ + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ + "using docker."], + ["openfoam_wf_singularity", "A parameter study that includes initializing, running,\n" \ + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ + "using singularity."], + ["null_chain", "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" \ + "May be used to measure overhead in merlin.\n" \ + "Iterates thru a chain of workflows."], + ["null_spec", "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin."], + ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ] + expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" + actual = list_examples() + assert actual == expected + + +def test_setup_example_invalid_name(): + """ + Test the `setup_example` function with an invalid example name. + This should just return None. + """ + assert setup_example("invalid_example_name", None) is None + + +def test_setup_example_no_outdir(temp_output_dir: str): + """ + Test the `setup_example` function with an invalid example name. + This should create a directory with the example name (in this case hello) + and copy all of the example contents to this folder. + We'll create a directory specifically for this test and move into it so that + the `setup_example` function creates the hello/ subdirectory in a directory with + the name of this test (setup_no_outdir). + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + cwd = os.getcwd() + + # Create the temp path to store this setup and move into that directory + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + setup_example_dir = os.path.join(generator_dir, "setup_no_outdir") + create_dir(setup_example_dir) + os.chdir(setup_example_dir) + + # This should still work and return to us the name of the example + try: + assert setup_example("hello", None) == "hello" + except AssertionError as exc: + os.chdir(cwd) + raise AssertionError from exc + + # All files from this example should be written to a directory with the example name + full_output_path = os.path.join(setup_example_dir, "hello") + expected_files = [ + os.path.join(full_output_path, "hello_samples.yaml"), + os.path.join(full_output_path, "hello.yaml"), + os.path.join(full_output_path, "my_hello.yaml"), + os.path.join(full_output_path, "requirements.txt"), + os.path.join(full_output_path, "make_samples.py"), + ] + try: + for file in expected_files: + assert os.path.exists(file) + except AssertionError as exc: + os.chdir(cwd) + raise AssertionError from exc + + +def test_setup_example_outdir_exists(temp_output_dir: str): + """ + Test the `setup_example` function with an output directory that already exists. + This should just return None. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + + assert setup_example("hello", generator_dir) is None + + +##################################### +# Tests for setting up each example # +##################################### + + +def run_setup_example(temp_output_dir: str, example_name: str, example_files: List[str], expected_return: str): + """ + Helper function to run tests for the `setup_example` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param example_name: The name of the example to setup + :param example_files: A list of filenames that should be copied by setup_example + :param expected_return: The expected return value from `setup_example` + """ + # Create the temp path to store this setup + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + setup_example_dir = os.path.join(generator_dir, f"setup_{example_name}") + + # Ensure that the example name is returned + actual = setup_example(example_name, setup_example_dir) + assert actual == expected_return + + # Ensure all of the files that should've been copied were copied + expected_files = [os.path.join(setup_example_dir, expected_file) for expected_file in example_files] + for file in expected_files: + assert os.path.exists(file) + + +def test_setup_example_feature_demo(temp_output_dir: str): + """ + Test the `setup_example` function for the feature_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "feature_demo" + example_files = [ + ".gitignore", + "feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_flux(temp_output_dir: str): + """ + Test the `setup_example` function for the flux example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_files = [ + "flux_local.yaml", + "flux_par_restart.yaml", + "flux_par.yaml", + "paper.yaml", + "requirements.txt", + "scripts/flux_info.py", + "scripts/hello_sleep.c", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/paper_workers.sbatch", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + "scripts/workers.bsub", + ] + + run_setup_example(temp_output_dir, "flux_local", example_files, "flux") + + +def test_setup_example_lsf(temp_output_dir: str): + """ + Test the `setup_example` function for the lsf example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # TODO should there be a workers.bsub for this example? + example_files = [ + "lsf_par_srun.yaml", + "lsf_par.yaml", + "scripts/hello.c", + "scripts/make_samples.py", + ] + + run_setup_example(temp_output_dir, "lsf_par", example_files, "lsf") + + +def test_setup_example_slurm(temp_output_dir: str): + """ + Test the `setup_example` function for the slurm example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_files = [ + "slurm_par.yaml", + "slurm_par_restart.yaml", + "requirements.txt", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + ] + + run_setup_example(temp_output_dir, "slurm_par", example_files, "slurm") + + +def test_setup_example_hello(temp_output_dir: str): + """ + Test the `setup_example` function for the hello example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "hello" + example_files = [ + "hello_samples.yaml", + "hello.yaml", + "my_hello.yaml", + "requirements.txt", + "make_samples.py", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_hpc(temp_output_dir: str): + """ + Test the `setup_example` function for the hpc_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "hpc_demo" + example_files = [ + "hpc_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_iterative(temp_output_dir: str): + """ + Test the `setup_example` function for the iterative_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "iterative_demo" + example_files = [ + "iterative_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_null(temp_output_dir: str): + """ + Test the `setup_example` function for the null_spec example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "null_spec" + example_files = [ + "null_spec.yaml", + "null_chain.yaml", + ".gitignore", + "Makefile", + "requirements.txt", + "scripts/aggregate_chain_output.sh", + "scripts/aggregate_output.sh", + "scripts/check_completion.sh", + "scripts/kill_all.sh", + "scripts/launch_chain_job.py", + "scripts/launch_jobs.py", + "scripts/make_samples.py", + "scripts/read_output_chain.py", + "scripts/read_output.py", + "scripts/search.sh", + "scripts/submit_chain.sbatch", + "scripts/submit.sbatch", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_openfoam(temp_output_dir: str): + """ + Test the `setup_example` function for the openfoam_wf example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "openfoam_wf" + example_files = [ + "openfoam_wf.yaml", + "openfoam_wf_template.yaml", + "README.md", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_openfoam_no_docker(temp_output_dir: str): + """ + Test the `setup_example` function for the openfoam_wf_no_docker example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "openfoam_wf_no_docker" + example_files = [ + "openfoam_wf_no_docker.yaml", + "openfoam_wf_no_docker_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_openfoam_singularity(temp_output_dir: str): + """ + Test the `setup_example` function for the openfoam_wf_singularity example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "openfoam_wf_singularity" + example_files = [ + "openfoam_wf_singularity.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_optimization(temp_output_dir: str): + """ + Test the `setup_example` function for the optimization example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_files = [ + "optimization_basic.yaml", + "requirements.txt", + "template_config.py", + "template_optimization.temp", + "scripts/collector.py", + "scripts/optimizer.py", + "scripts/test_functions.py", + "scripts/visualizer.py", + ] + + run_setup_example(temp_output_dir, "optimization_basic", example_files, "optimization") + + +def test_setup_example_remote_feature_demo(temp_output_dir: str): + """ + Test the `setup_example` function for the remote_feature_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "remote_feature_demo" + example_files = [ + ".gitignore", + "remote_feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_restart(temp_output_dir: str): + """ + Test the `setup_example` function for the restart example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "restart" + example_files = [ + "restart.yaml", + "scripts/make_samples.py" + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_restart_delay(temp_output_dir: str): + """ + Test the `setup_example` function for the restart_delay example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "restart_delay" + example_files = [ + "restart_delay.yaml", + "scripts/make_samples.py" + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_simple_chain(temp_output_dir: str): + """ + Test the `setup_example` function for the simple_chain example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the temp path to store this setup + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + output_file = os.path.join(generator_dir, "simple_chain.yaml") + + # Ensure that the example name is returned + actual = setup_example("simple_chain", output_file) + assert actual == "simple_chain" + assert os.path.exists(output_file) From 681bd717a06abe33d658fd7093787e65c28935e2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 16:47:28 -0800 Subject: [PATCH 026/201] run fix-style and update changelog --- CHANGELOG.md | 5 +- tests/conftest.py | 2 +- tests/constants.py | 3 +- tests/unit/test_examples_generator.py | 79 +++++++++++++++------------ tests/utils.py | 1 - 5 files changed, 51 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d0bea05d..01cc3b35e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Coverage to the test suite. This includes adding tests for: - `merlin/common/` - `merlin/config/` + - `merlin/examples/` - `celeryadapter.py` - Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures - - RedisServerManager: context to help with starting/stopping a redis server for tests - - CeleryWorkersManager: context to help with starting/stopping workers for tests + - `RedisServerManager`: context to help with starting/stopping a redis server for tests + - `CeleryWorkersManager`: context to help with starting/stopping workers for tests - Ability to copy and print the `Config` object from `merlin/config/__init__.py` ### Fixed diff --git a/tests/conftest.py b/tests/conftest.py index 20749d4cd..c444a2168 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ from celery.canvas import Signature from merlin.config.configfile import CONFIG -from tests.constants import SERVER_PASS, CERT_FILES +from tests.constants import CERT_FILES, SERVER_PASS from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager from tests.utils import create_cert_files, create_pass_file diff --git a/tests/constants.py b/tests/constants.py index a2b354146..26cfe4c0a 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -3,8 +3,9 @@ """ SERVER_PASS = "merlin-test-server" + CERT_FILES = { "ssl_cert": "test-rabbit-client-cert.pem", "ssl_ca": "test-mysql-ca-cert.pem", "ssl_key": "test-rabbit-client-key.pem", -} \ No newline at end of file +} diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 7d7ccc5bf..97948feaf 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -2,7 +2,6 @@ Tests for the `merlin/examples/generator.py` module. """ import os -import pathlib from typing import List from tabulate import tabulate @@ -13,7 +12,7 @@ gather_example_dirs, list_examples, setup_example, - write_example + write_example, ) from tests.utils import create_dir @@ -39,7 +38,7 @@ def test_gather_example_dirs(): "restart", "restart_delay", "simple_chain", - "slurm" + "slurm", ] expected = {} for wf_dir in example_workflows: @@ -76,7 +75,7 @@ def test_gather_all_examples(): f"{EXAMPLES_DIR}/restart_delay/restart_delay.yaml", f"{EXAMPLES_DIR}/simple_chain/simple_chain.yaml", f"{EXAMPLES_DIR}/slurm/slurm_par_restart.yaml", - f"{EXAMPLES_DIR}/slurm/slurm_par.yaml" + f"{EXAMPLES_DIR}/slurm/slurm_par.yaml", ] actual = gather_all_examples() assert sorted(actual) == sorted(expected) @@ -85,7 +84,7 @@ def test_gather_all_examples(): def test_write_example_dir(temp_output_dir: str): """ Test the `write_example` function with the src_path as a directory. - + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) @@ -98,7 +97,7 @@ def test_write_example_dir(temp_output_dir: str): def test_write_example_file(temp_output_dir: str): """ Test the `write_example` function with the src_path as a file. - + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) @@ -115,16 +114,22 @@ def test_list_examples(): """Test the `list_examples` function to see if it gives us all of the examples that we want.""" expected_headers = ["name", "description"] expected_rows = [ - ["openfoam_wf_no_docker", "A parameter study that includes initializing, running,\n" \ - "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" \ - "without using docker."], - ["optimization_basic", "Design Optimization Template\n" \ - "To use,\n" \ - "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" \ - "2. Run the template_config file in current directory using `python template_config.py`\n" \ - "3. Merlin run as usual (merlin run optimization.yaml)\n" \ - "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" \ - "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts"], + [ + "openfoam_wf_no_docker", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" + "without using docker.", + ], + [ + "optimization_basic", + "Design Optimization Template\n" + "To use,\n" + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" + "2. Run the template_config file in current directory using `python template_config.py`\n" + "3. Merlin run as usual (merlin run optimization.yaml)\n" + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", + ], ["feature_demo", "Run 10 hello worlds."], ["flux_local", "Run a scan through Merlin/Maestro"], ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], @@ -141,16 +146,28 @@ def test_list_examples(): ["hello", "a very simple merlin workflow"], ["hello_samples", "a very simple merlin workflow, with samples"], ["hpc_demo", "Demo running a workflow on HPC machines"], - ["openfoam_wf", "A parameter study that includes initializing, running,\n" \ - "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ - "using docker."], - ["openfoam_wf_singularity", "A parameter study that includes initializing, running,\n" \ - "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ - "using singularity."], - ["null_chain", "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" \ - "May be used to measure overhead in merlin.\n" \ - "Iterates thru a chain of workflows."], - ["null_spec", "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin."], + [ + "openfoam_wf", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" + "using docker.", + ], + [ + "openfoam_wf_singularity", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" + "using singularity.", + ], + [ + "null_chain", + "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" + "May be used to measure overhead in merlin.\n" + "Iterates thru a chain of workflows.", + ], + [ + "null_spec", + "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + ], ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" @@ -534,10 +551,7 @@ def test_setup_example_restart(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ example_name = "restart" - example_files = [ - "restart.yaml", - "scripts/make_samples.py" - ] + example_files = ["restart.yaml", "scripts/make_samples.py"] run_setup_example(temp_output_dir, example_name, example_files, example_name) @@ -549,10 +563,7 @@ def test_setup_example_restart_delay(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ example_name = "restart_delay" - example_files = [ - "restart_delay.yaml", - "scripts/make_samples.py" - ] + example_files = ["restart_delay.yaml", "scripts/make_samples.py"] run_setup_example(temp_output_dir, example_name, example_files, example_name) diff --git a/tests/utils.py b/tests/utils.py index 3a75622b8..d883b83cd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -41,4 +41,3 @@ def create_dir(dirpath: str): """ if not os.path.exists(dirpath): os.mkdir(dirpath) - From 4b8fab51f16898a0340aea74749bf9c48536fdb7 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 14 Feb 2024 13:18:17 -0800 Subject: [PATCH 027/201] add a 'pip freeze' call in github workflow to view reqs versions --- .github/workflows/push-pr_workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index eecbf3eeb..4b5de2373 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -95,6 +95,7 @@ jobs: python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt + pip freeze - name: Install singularity run: | From 3099d4c0e877c0bdbd45e687ba4de2d27a49f38e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 25 Apr 2024 12:42:09 -0700 Subject: [PATCH 028/201] re-delete the old config test files --- tests/unit/config/old_test_configfile.py | 97 ------------------- tests/unit/config/old_test_results_backend.py | 67 ------------- tests/unit/config/utils.py | 24 ----- 3 files changed, 188 deletions(-) delete mode 100644 tests/unit/config/old_test_configfile.py delete mode 100644 tests/unit/config/old_test_results_backend.py delete mode 100644 tests/unit/config/utils.py diff --git a/tests/unit/config/old_test_configfile.py b/tests/unit/config/old_test_configfile.py deleted file mode 100644 index 39139ec11..000000000 --- a/tests/unit/config/old_test_configfile.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Tests for the configfile module.""" - -import os -import shutil -import tempfile -import unittest -from getpass import getuser - -from merlin.config import configfile - -from .utils import mkfile - - -CONFIG_FILE_CONTENTS = """ -celery: - certs: path/to/celery/config/files - -broker: - name: rabbitmq - username: testuser - password: rabbit.password # The filename that contains the password. - server: jackalope.llnl.gov - -results_backend: - name: mysql - dbname: testuser - username: mlsi - password: mysql.password # The filename that contains the password. - server: rabbit.llnl.gov - -""" - - -class TestFindConfigFile(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.appfile = mkfile(self.tmpdir, "app.yaml") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_tempdir(self): - self.assertTrue(os.path.isdir(self.tmpdir)) - - def test_find_config_file(self): - """ - Given the path to a vaild config file, find and return the full - filepath. - """ - path = configfile.find_config_file(path=self.tmpdir) - expected = os.path.join(self.tmpdir, self.appfile) - self.assertEqual(path, expected) - - def test_find_config_file_error(self): - """Given an invalid path, return None.""" - invalid = "invalid/path" - expected = None - - path = configfile.find_config_file(path=invalid) - self.assertEqual(path, expected) - - -class TestConfigFile(unittest.TestCase): - """Unit tests for loading the config file.""" - - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.configfile = mkfile(self.tmpdir, "app.yaml", content=CONFIG_FILE_CONTENTS) - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_get_config(self): - """ - Given the directory path to a valid merlin config file, then - `get_config` should find the merlin config file and load the YAML - contents to a dictionary. - """ - expected = { - "broker": { - "name": "rabbitmq", - "password": "rabbit.password", - "server": "jackalope.llnl.gov", - "username": "testuser", - "vhost": getuser(), - }, - "celery": {"certs": "path/to/celery/config/files"}, - "results_backend": { - "dbname": "testuser", - "name": "mysql", - "password": "mysql.password", - "server": "rabbit.llnl.gov", - "username": "mlsi", - }, - } - - self.assertDictEqual(configfile.get_config(self.tmpdir), expected) diff --git a/tests/unit/config/old_test_results_backend.py b/tests/unit/config/old_test_results_backend.py deleted file mode 100644 index 638f13eb8..000000000 --- a/tests/unit/config/old_test_results_backend.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Tests for the results_backend module.""" - -import os -import shutil -import tempfile -import unittest - -from merlin.config import results_backend - -from .utils import mkfile - - -class TestResultsBackend(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - - # Create test files. - self.tmpfile1 = mkfile(self.tmpdir, "mysql_test1.txt") - self.tmpfile2 = mkfile(self.tmpdir, "mysql_test2.txt") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_mysql_config(self): - """ - Given the path to a directory containing the MySQL cert files and a - dictionary of files to look for, then find and return the full path to - all the certs. - """ - certs = {"test1": "mysql_test1.txt", "test2": "mysql_test2.txt"} - - # This will just be the above dictionary with the full file paths. - expected = { - "test1": os.path.join(self.tmpdir, certs["test1"]), - "test2": os.path.join(self.tmpdir, certs["test2"]), - } - results = results_backend.get_mysql_config(self.tmpdir, certs) - self.assertDictEqual(results, expected) - - def test_mysql_config_no_files(self): - """ - Given the path to a directory containing the MySQL cert files and - an empty dictionary, then `get_mysql_config` should return an empty - dictionary. - """ - files = {} - result = results_backend.get_mysql_config(self.tmpdir, files) - self.assertEqual(result, {}) - - -class TestConfingMysqlErrorPath(unittest.TestCase): - """ - Test `get_mysql_config` against cases were the given path does not exist. - """ - - def test_mysql_config_false(self): - """ - Given a path that does not exist, then `get_mysql_config` should return - False. - """ - path = "invalid/path" - - # We don't need the dictionary populated for this test. The function - # should return False before trying to process the dictionary. - certs = {} - result = results_backend.get_mysql_config(path, certs) - self.assertFalse(result) diff --git a/tests/unit/config/utils.py b/tests/unit/config/utils.py deleted file mode 100644 index 1765e8478..000000000 --- a/tests/unit/config/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Utils module for common test functionality. -""" - -import os - - -def mkfile(tmpdir, filename, content=""): - """ - A simple function for creating a file and returning the path. This is to - abstract out file creation logic in the tests. - - :param tmpdir: (str) The path to the temp directory. - :param filename: (str) The name of the file. - :param contents: (str) Optional contents to write to the file. Defaults to - an empty string. - :returns: (str) The appended path of the given tempdir and filename. - """ - filepath = os.path.join(tmpdir, filename) - - with open(filepath, "w") as f: - f.write(content) - - return filepath From 37839f63db701001fb646ce2aa0e0928cfb93681 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 25 Apr 2024 14:04:43 -0700 Subject: [PATCH 029/201] fix tests/bugs introduced by merging in develop --- merlin/config/utils.py | 10 +++- merlin/examples/generator.py | 1 + ...m_wf.yaml => openfoam_wf_singularity.yaml} | 0 tests/unit/config/test_utils.py | 50 ++++++++++++++++--- tests/unit/test_examples_generator.py | 6 ++- 5 files changed, 55 insertions(+), 12 deletions(-) rename merlin/examples/workflows/openfoam_wf_singularity/{openfoam_wf.yaml => openfoam_wf_singularity.yaml} (100%) diff --git a/merlin/config/utils.py b/merlin/config/utils.py index bb0dcd58b..6bb3186df 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -77,8 +77,14 @@ def get_priority(priority: Priority) -> int: :param priority: The priority value that we want :returns: The priority value as an integer """ - if priority not in Priority: - raise ValueError(f"Invalid priority: {priority}") + priority_err_msg = f"Invalid priority: {priority}" + try: + # In python 3.12+ if something is not in the enum it will just return False + if priority not in Priority: + raise ValueError(priority_err_msg) + # In python 3.11 and below, a TypeError is raised when looking for something in an enum that is not there + except TypeError: + raise ValueError(priority_err_msg) priority_map = determine_priority_map(CONFIG.broker.name.lower()) return priority_map.get(priority, priority_map[Priority.MID]) # Default to MID priority for unknown priorities diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index cb214fed4..120d2defd 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -146,4 +146,5 @@ def setup_example(name, outdir): LOG.info(f"Copying example '{name}' to {outdir}") write_example(src_path, outdir) + print(f'example: {example}') return example diff --git a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml similarity index 100% rename from merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml rename to merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py index a02bc1ff1..9d64c10c7 100644 --- a/tests/unit/config/test_utils.py +++ b/tests/unit/config/test_utils.py @@ -5,7 +5,7 @@ import pytest from merlin.config.configfile import CONFIG -from merlin.config.utils import Priority, get_priority, is_rabbit_broker, is_redis_broker +from merlin.config.utils import Priority, determine_priority_map, get_priority, is_rabbit_broker, is_redis_broker def test_is_rabbit_broker(): @@ -37,25 +37,27 @@ def test_is_redis_broker_invalid(): def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_priority` function with rabbit as the broker. - Low priority for rabbit is 1 and high is 10. + Low priority for rabbit is 1 and high is 9. :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_priority(Priority.LOW) == 1 assert get_priority(Priority.MID) == 5 - assert get_priority(Priority.HIGH) == 10 + assert get_priority(Priority.HIGH) == 9 + assert get_priority(Priority.RETRY) == 10 def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_priority` function with redis as the broker. - Low priority for redis is 10 and high is 1. + Low priority for redis is 10 and high is 2. :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_priority(Priority.LOW) == 10 assert get_priority(Priority.MID) == 5 - assert get_priority(Priority.HIGH) == 1 + assert get_priority(Priority.HIGH) == 2 + assert get_priority(Priority.RETRY) == 1 def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 @@ -68,7 +70,7 @@ def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F CONFIG.broker.name = "invalid" with pytest.raises(ValueError) as excinfo: get_priority(Priority.LOW) - assert "Function get_priority has reached unknown state! Maybe unsupported broker invalid?" in str(excinfo.value) + assert "Unsupported broker name: invalid" in str(excinfo.value) def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 @@ -78,6 +80,38 @@ def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ - with pytest.raises(TypeError) as excinfo: + with pytest.raises(ValueError) as excinfo: get_priority("invalid_priority") - assert "Unrecognized priority 'invalid_priority'!" in str(excinfo.value) + assert "Invalid priority: invalid_priority" in str(excinfo.value) + + +def test_determine_priority_map_rabbit(): + """ + Test the `determine_priority_map` function with rabbit as the broker. + This should return the following map: + {Priority.LOW: 1, Priority.MID: 5, Priority.HIGH: 9, Priority.RETRY: 10} + """ + expected = {Priority.LOW: 1, Priority.MID: 5, Priority.HIGH: 9, Priority.RETRY: 10} + actual = determine_priority_map("rabbitmq") + assert actual == expected + + +def test_determine_priority_map_redis(): + """ + Test the `determine_priority_map` function with redis as the broker. + This should return the following map: + {Priority.LOW: 10, Priority.MID: 5, Priority.HIGH: 2, Priority.RETRY: 1} + """ + expected = {Priority.LOW: 10, Priority.MID: 5, Priority.HIGH: 2, Priority.RETRY: 1} + actual = determine_priority_map("redis") + assert actual == expected + + +def test_determine_priority_map_invalid(): + """ + Test the `determine_priority_map` function with an invalid broker. + This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + determine_priority_map("invalid_broker") + assert "Unsupported broker name: invalid_broker" in str(excinfo.value) diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 97948feaf..5a05e3599 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -64,11 +64,12 @@ def test_gather_all_examples(): f"{EXAMPLES_DIR}/lsf/lsf_par.yaml", f"{EXAMPLES_DIR}/null_spec/null_chain.yaml", f"{EXAMPLES_DIR}/null_spec/null_spec.yaml", - f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_docker_template.yaml", f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf.yaml", f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml", f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml", f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity_template.yaml", f"{EXAMPLES_DIR}/optimization/optimization_basic.yaml", f"{EXAMPLES_DIR}/remote_feature_demo/remote_feature_demo.yaml", f"{EXAMPLES_DIR}/restart/restart.yaml", @@ -445,7 +446,7 @@ def test_setup_example_openfoam(temp_output_dir: str): example_name = "openfoam_wf" example_files = [ "openfoam_wf.yaml", - "openfoam_wf_template.yaml", + "openfoam_wf_docker_template.yaml", "README.md", "requirements.txt", "scripts/make_samples.py", @@ -492,6 +493,7 @@ def test_setup_example_openfoam_singularity(temp_output_dir: str): example_name = "openfoam_wf_singularity" example_files = [ "openfoam_wf_singularity.yaml", + "openfoam_wf_singularity_template.yaml", "requirements.txt", "scripts/make_samples.py", "scripts/blockMesh_template.txt", From b8185cc0fbabf730dc77adf44e1a78701fb52953 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 25 Apr 2024 16:36:46 -0700 Subject: [PATCH 030/201] add a unit test file for the dumper module --- tests/unit/common/test_dumper.py | 156 +++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 tests/unit/common/test_dumper.py diff --git a/tests/unit/common/test_dumper.py b/tests/unit/common/test_dumper.py new file mode 100644 index 000000000..7c437fde9 --- /dev/null +++ b/tests/unit/common/test_dumper.py @@ -0,0 +1,156 @@ +""" +Tests for the `dumper.py` file. +""" +import csv +import json +import os +import pytest + +from datetime import datetime +from time import sleep + +from merlin.common.dumper import dump_handler + +NUM_ROWS = 5 +CSV_INFO_TO_DUMP = {"row_num": [i for i in range(1, NUM_ROWS+1)], "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS+1)]} +JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS+1)} +DUMP_HANDLER_DIR = "{temp_output_dir}/dump_handler" + +def test_dump_handler_invalid_dump_file(): + """ + This is really testing the initialization of the Dumper class with an invalid file type. + This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + dump_handler("bad_file.txt", CSV_INFO_TO_DUMP) + assert "Invalid file type for bad_file.txt. Supported file types are: ['csv', 'json']" in str(excinfo.value) + +def get_output_file(temp_dir: str, file_name: str): + """ + Helper function to get a full path to the temporary output file. + + :param temp_dir: The path to the temporary output directory that pytest gives us + :param file_name: The name of the file + """ + dump_dir = DUMP_HANDLER_DIR.format(temp_output_dir=temp_dir) + if not os.path.exists(dump_dir): + os.mkdir(dump_dir) + dump_file = f"{dump_dir}/{file_name}" + return dump_file + +def run_csv_dump_test(dump_file: str, fmode: str): + """ + Run the test for csv dump. + + :param dump_file: The file that the dump was written to + :param fmode: The type of write that we're testing ("w" for write, "a" for append) + """ + + # Check that the file exists and that read in the contents of the file + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + reader = csv.reader(df) + written_data = list(reader) + + expected_rows = NUM_ROWS*2 if fmode == "a" else NUM_ROWS + assert len(written_data) == expected_rows+1 # Adding one because of the header row + for i, row in enumerate(written_data): + assert len(row) == 2 # Check number of columns + if i == 0: # Checking the header row + assert row[0] == "row_num" + assert row[1] == "other_info" + else: # Checking the data rows + assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i%NUM_ROWS)-1]) + assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i%NUM_ROWS)-1]) + +def test_dump_handler_csv_write(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class. + This should create a csv file and write to it. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "csv_write.csv") + + # Run the actual call to dump to the file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Assert that everything ran properly + run_csv_dump_test(dump_file, "w") + +def test_dump_handler_csv_append(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class with the file write mode set to append. + We'll write to a csv file first and then run again to make sure we can append to it properly. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "csv_append.csv") + + # Run the first call to create the csv file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Run the second call to append to the csv file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Assert that everything ran properly + run_csv_dump_test(dump_file, "a") + +def test_dump_handler_json_write(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class. + This should create a json file and write to it. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "json_write.json") + + # Run the actual call to dump to the file + dump_handler(dump_file, JSON_INFO_TO_DUMP) + + # Check that the file exists and that the contents are correct + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + contents = json.load(df) + assert contents == JSON_INFO_TO_DUMP + +def test_dump_handler_json_append(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class with the file write mode set to append. + We'll write to a json file first and then run again to make sure we can append to it properly. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "json_append.json") + + # Run the first call to create the file + timestamp_1 = str(datetime.now()) + first_dump = {timestamp_1: JSON_INFO_TO_DUMP} + dump_handler(dump_file, first_dump) + + # Sleep so we don't accidentally get the same timestamp + sleep(.5) + + # Run the second call to append to the file + timestamp_2 = str(datetime.now()) + second_dump = {timestamp_2: JSON_INFO_TO_DUMP} + dump_handler(dump_file, second_dump) + + # Check that the file exists and that the contents are correct + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + contents = json.load(df) + keys = contents.keys() + assert len(keys) == 2 + assert timestamp_1 in keys + assert timestamp_2 in keys + assert contents[timestamp_1] == JSON_INFO_TO_DUMP + assert contents[timestamp_2] == JSON_INFO_TO_DUMP \ No newline at end of file From e48fe32514ab772466ad5a79ec3c288463eadead Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 7 May 2024 14:58:41 -0700 Subject: [PATCH 031/201] begin work on server tests and modular fixtures --- merlin/server/server_util.py | 39 +++- tests/fixtures/server.py | 84 ++++++++ tests/unit/server/__init__.py | 0 tests/unit/server/test_server_util.py | 295 ++++++++++++++++++++++++++ 4 files changed, 411 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/server.py create mode 100644 tests/unit/server/__init__.py create mode 100644 tests/unit/server/test_server_util.py diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index c10e0e1d9..bdfd3652e 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -60,7 +60,7 @@ def valid_ipv4(ip: str) -> bool: # pylint: disable=C0103 return False for i in arr: - if int(i) < 0 and int(i) > 255: + if int(i) < 0 or int(i) > 255: return False return True @@ -121,6 +121,15 @@ def __init__(self, data: dict) -> None: self.pass_file = data["pass_file"] if "pass_file" in data else self.PASSWORD_FILE self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE + def __eq__(self, other: "ContainerFormatConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ContainerFormatConfig object to check if they're the same + """ + variables = ("format", "image_type", "image", "url", "config", "config_dir", "pfile", "pass_file", "user_file") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_format(self) -> str: """Getter method to get the container format""" return self.format @@ -208,6 +217,15 @@ def __init__(self, data: dict) -> None: self.stop_command = data["stop_command"] if "stop_command" in data else self.STOP_COMMAND self.pull_command = data["pull_command"] if "pull_command" in data else self.PULL_COMMAND + def __eq__(self, other: "ContainerFormatConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ContainerFormatConfig object to check if they're the same + """ + variables = ("command", "run_command", "stop_command", "pull_command") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_command(self) -> str: """Getter method to get the container command""" return self.command @@ -242,6 +260,15 @@ def __init__(self, data: dict) -> None: self.status = data["status"] if "status" in data else self.STATUS_COMMAND self.kill = data["kill"] if "kill" in data else self.KILL_COMMAND + def __eq__(self, other: "ProcessConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ProcessConfig object to check if they're the same + """ + variables = ("status", "kill") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_status_command(self) -> str: """Getter method to get the status command""" return self.status @@ -264,12 +291,10 @@ class ServerConfig: # pylint: disable=R0903 container_format: ContainerFormatConfig = None def __init__(self, data: dict) -> None: - if "container" in data: - self.container = ContainerConfig(data["container"]) - if "process" in data: - self.process = ProcessConfig(data["process"]) - if self.container.get_format() in data: - self.container_format = ContainerFormatConfig(data[self.container.get_format()]) + self.container = ContainerConfig(data["container"]) if "container" in data else None + self.process = ProcessConfig(data["process"]) if "process" in data else None + container_format_data = data.get(self.container.get_format() if self.container else None) + self.container_format = ContainerFormatConfig(container_format_data) if container_format_data else None class RedisConfig: diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py new file mode 100644 index 000000000..35efdcd65 --- /dev/null +++ b/tests/fixtures/server.py @@ -0,0 +1,84 @@ +""" +Fixtures specifically for help testing the modules in the server/ directory. +""" +import pytest +import shutil +from typing import Dict + +@pytest.fixture(scope="class") +def server_container_config_data(temp_output_dir: str): + """ + Fixture to provide sample data for ContainerConfig tests + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + return { + "format": "docker", + "image_type": "postgres", + "image": "postgres:latest", + "url": "postgres://localhost", + "config": "postgres.conf", + "config_dir": "/path/to/config", + "pfile": "merlin_server_postgres.pf", + "pass_file": f"{temp_output_dir}/postgres.pass", + "user_file": "postgres.users", + } + +@pytest.fixture(scope="class") +def server_container_format_config_data(): + """ + Fixture to provide sample data for ContainerFormatConfig tests + """ + return { + "command": "docker", + "run_command": "{command} run --name {name} -d {image}", + "stop_command": "{command} stop {name}", + "pull_command": "{command} pull {url}", + } + +@pytest.fixture(scope="class") +def server_process_config_data(): + """ + Fixture to provide sample data for ProcessConfig tests + """ + return { + "status": "status {pid}", + "kill": "terminate {pid}", + } + +@pytest.fixture(scope="class") +def server_server_config( + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], + server_container_format_config_data: Dict[str, str], +): + """ + Fixture to provide sample data for ServerConfig tests + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + """ + return { + "container": server_container_config_data, + "process": server_process_config_data, + "docker": server_container_format_config_data, + } + + +@pytest.fixture(scope="class") +def server_redis_conf_file(temp_output_dir: str): + """ + Fixture to copy the redis.conf file from the merlin/server/ directory to the + temporary output directory and provide the path to the copied file + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + # TODO + # - will probably have to do more than just copying over the conf file + # - likely want to create our own test conf file with the settings that + # can be modified by RedisConf instead + path_to_redis_conf = f"{os.path.dirname(os.path.abspath(__file__))}/../../merlin/server/redis.conf" + path_to_copied_redis = f"{temp_output_dir}/redis.conf" + shutil.copy(path_to_redis_conf, path_to_copied_redis) + return path_to_copied_redis \ No newline at end of file diff --git a/tests/unit/server/__init__.py b/tests/unit/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py new file mode 100644 index 000000000..384e1ea37 --- /dev/null +++ b/tests/unit/server/test_server_util.py @@ -0,0 +1,295 @@ +""" +Tests for the `server_util.py` module. +""" +import os +import pytest +from typing import Callable, Dict, Union + +from merlin.server.server_util import ( + AppYaml, + ContainerConfig, + ContainerFormatConfig, + ProcessConfig, + RedisConfig, + RedisUsers, + ServerConfig, + valid_ipv4, + valid_port +) + +@pytest.mark.parametrize("valid_ip", [ + "0.0.0.0", + "127.0.0.1", + "14.105.200.58", + "255.255.255.255", +]) +def test_valid_ipv4_valid_ip(valid_ip: str): + """ + Test the `valid_ipv4` function with valid IPs. + This should return True. + + :param valid_ip: A valid port to test. + These are pulled from the parametrized list defined above this test. + """ + assert valid_ipv4(valid_ip) + +@pytest.mark.parametrize("invalid_ip", [ + "256.0.0.1", + "-1.0.0.1", + None, + "127.0.01", +]) +def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): + """ + Test the `valid_ipv4` function with invalid IPs. + An IP is valid if every integer separated by the '.' delimiter are between 0 and 255. + This should return False for both IPs tested here. + + :param invalid_ip: An invalid port to test. + These are pulled from the parametrized list defined above this test. + """ + assert not valid_ipv4(invalid_ip) + +@pytest.mark.parametrize("valid_input", [ + 1, + 433, + 65535, +]) +def test_valid_port_valid_input(valid_input: int): + """ + Test the `valid_port` function with valid port numbers. + Valid ports are ports between 1 and 65535. + This should return True. + + :param valid_input: A valid input value to test. + These are pulled from the parametrized list defined above this test. + """ + assert valid_port(valid_input) + +@pytest.mark.parametrize("invalid_input", [ + -1, + 0, + 65536, +]) +def test_valid_port_invalid_input(invalid_input: int): + """ + Test the `valid_port` function with invalid inputs. + Valid ports are ports between 1 and 65535. + This should return False for each invalid input tested. + + :param invalid_input: An invalid input value to test. + These are pulled from the parametrized list defined above this test. + """ + assert not valid_port(invalid_input) + + +class TestContainerConfig: + """Tests for the ContainerConfig class.""" + + def test_init_with_complete_data(self, server_container_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + config = ContainerConfig(server_container_config_data) + assert config.format == server_container_config_data["format"] + assert config.image_type == server_container_config_data["image_type"] + assert config.image == server_container_config_data["image"] + assert config.url == server_container_config_data["url"] + assert config.config == server_container_config_data["config"] + assert config.config_dir == server_container_config_data["config_dir"] + assert config.pfile == server_container_config_data["pfile"] + assert config.pass_file == server_container_config_data["pass_file"] + assert config.user_file == server_container_config_data["user_file"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data + """ + incomplete_data = {"format": "docker"} + config = ContainerConfig(incomplete_data) + assert config.format == incomplete_data["format"] + assert config.image_type == ContainerConfig.IMAGE_TYPE + assert config.image == ContainerConfig.IMAGE_NAME + assert config.url == ContainerConfig.REDIS_URL + assert config.config == ContainerConfig.CONFIG_FILE + assert config.config_dir == ContainerConfig.CONFIG_DIR + assert config.pfile == ContainerConfig.PROCESS_FILE + assert config.pass_file == ContainerConfig.PASSWORD_FILE + assert config.user_file == ContainerConfig.USERS_FILE + + @pytest.mark.parametrize("attr_name", [ + "image", + "config", + "pfile", + "pass_file", + "user_file", + ]) + def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): + """ + Tests that get_*_path methods construct the correct path + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param attr_name: Name of the attribute to be tested. These are pulled from the parametrized list defined above this test. + """ + config = ContainerConfig(server_container_config_data) + get_path_method = getattr(config, f"get_{attr_name}_path") # Dynamically get the method based on attr_name + expected_path = os.path.join(server_container_config_data["config_dir"], server_container_config_data[attr_name]) + assert get_path_method() == expected_path + + @pytest.mark.parametrize("getter_name, expected_attr", [ + ("get_format", "format"), + ("get_image_type", "image_type"), + ("get_image_name", "image"), + ("get_image_url", "url"), + ("get_config_name", "config"), + ("get_config_dir", "config_dir"), + ("get_pfile_name", "pfile"), + ("get_pass_file_name", "pass_file"), + ("get_user_file_name", "user_file"), + ]) + def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ContainerConfig(server_container_config_data) + getter = getattr(config, getter_name) + assert getter() == server_container_config_data[expected_attr] + + def test_get_container_password(self, server_container_config_data: Dict[str, str]): + """ + Test that the get_container_password is reading the password file properly + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + # Write a fake password to the password file + test_password = "super-secret-password" + with open(server_container_config_data["pass_file"], "w") as pass_file: + pass_file.write(test_password) + + # Run the test + config = ContainerConfig(server_container_config_data) + assert config.get_container_password() == test_password + + +class TestContainerFormatConfig: + """Tests for the ContainerFormatConfig class.""" + + def test_init_with_complete_data(self, server_container_format_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + """ + config = ContainerFormatConfig(server_container_format_config_data) + assert config.command == server_container_format_config_data["command"] + assert config.run_command == server_container_format_config_data["run_command"] + assert config.stop_command == server_container_format_config_data["stop_command"] + assert config.pull_command == server_container_format_config_data["pull_command"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data + """ + incomplete_data = {"command": "docker"} + config = ContainerFormatConfig(incomplete_data) + assert config.command == incomplete_data["command"] + assert config.run_command == config.RUN_COMMAND + assert config.stop_command == config.STOP_COMMAND + assert config.pull_command == config.PULL_COMMAND + + @pytest.mark.parametrize("getter_name, expected_attr", [ + ("get_command", "command"), + ("get_run_command", "run_command"), + ("get_stop_command", "stop_command"), + ("get_pull_command", "pull_command"), + ]) + def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values + + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ContainerFormatConfig(server_container_format_config_data) + getter = getattr(config, getter_name) + assert getter() == server_container_format_config_data[expected_attr] + + +class TestProcessConfig: + """Tests for the ProcessConfig class.""" + + def test_init_with_complete_data(self, server_process_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + """ + config = ProcessConfig(server_process_config_data) + assert config.status == server_process_config_data["status"] + assert config.kill == server_process_config_data["kill"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data + """ + incomplete_data = {"status": "status {pid}"} + config = ProcessConfig(incomplete_data) + assert config.status == incomplete_data["status"] + assert config.kill == config.KILL_COMMAND + + @pytest.mark.parametrize("getter_name, expected_attr", [ + ("get_status_command", "status"), + ("get_kill_command", "kill"), + ]) + def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values + + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ProcessConfig(server_process_config_data) + getter = getattr(config, getter_name) + assert getter() == server_process_config_data[expected_attr] + + +class TestServerConfig: + """Tests for the ServerConfig class.""" + + def test_init_with_complete_data(self, server_server_config: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + config = ServerConfig(server_server_config) + assert config.container == ContainerConfig(server_server_config["container"]) + assert config.process == ProcessConfig(server_server_config["process"]) + assert config.container_format == ContainerFormatConfig(server_server_config["docker"]) + + def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): + """ + Tests that __init__ uses None for missing data + + :param server_process_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + incomplete_data = {"process": server_process_config_data} + config = ServerConfig(incomplete_data) + assert config.process == ProcessConfig(server_process_config_data) + assert config.container is None + assert config.container_format is None + + +# class TestRedisConfig: +# """Tests for the RedisConfig class.""" + +# def test_parse(self, server_redis_conf_file): +# raise ValueError From e1f667ddea170cbcc251182c4ad041aa699730ea Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 23 May 2024 08:00:48 -0700 Subject: [PATCH 032/201] start work on tests for RedisConfig --- tests/fixtures/server.py | 84 +++++++++++++++++++++------ tests/unit/server/test_server_util.py | 62 ++++++++++++++++++-- 2 files changed, 125 insertions(+), 21 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 35efdcd65..04c858f46 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -1,16 +1,17 @@ """ Fixtures specifically for help testing the modules in the server/ directory. """ +import os import pytest -import shutil from typing import Dict @pytest.fixture(scope="class") -def server_container_config_data(temp_output_dir: str): +def server_container_config_data(temp_output_dir: str) -> Dict[str, str]: """ Fixture to provide sample data for ContainerConfig tests :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: A dict containing the necessary key/values for the ContainerConfig object """ return { "format": "docker", @@ -25,9 +26,11 @@ def server_container_config_data(temp_output_dir: str): } @pytest.fixture(scope="class") -def server_container_format_config_data(): +def server_container_format_config_data() -> Dict[str, str]: """ Fixture to provide sample data for ContainerFormatConfig tests + + :returns: A dict containing the necessary key/values for the ContainerFormatConfig object """ return { "command": "docker", @@ -37,9 +40,11 @@ def server_container_format_config_data(): } @pytest.fixture(scope="class") -def server_process_config_data(): +def server_process_config_data() -> Dict[str, str]: """ Fixture to provide sample data for ProcessConfig tests + + :returns: A dict containing the necessary key/values for the ProcessConfig object """ return { "status": "status {pid}", @@ -51,13 +56,14 @@ def server_server_config( server_container_config_data: Dict[str, str], server_process_config_data: Dict[str, str], server_container_format_config_data: Dict[str, str], -): +) -> Dict[str, Dict[str, str]]: """ Fixture to provide sample data for ServerConfig tests :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :returns: A dictionary containing each of the configuration dicts we'll need """ return { "container": server_container_config_data, @@ -66,19 +72,63 @@ def server_server_config( } -@pytest.fixture(scope="class") -def server_redis_conf_file(temp_output_dir: str): +@pytest.fixture(scope="session") +def server_testing_dir(temp_output_dir: str) -> str: """ - Fixture to copy the redis.conf file from the merlin/server/ directory to the - temporary output directory and provide the path to the copied file + Fixture to create a temporary output directory for tests related to the server functionality. :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for server tests + """ + testing_dir = f"{temp_output_dir}/server_testing/" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir + + +@pytest.fixture(scope="session") +def server_redis_conf_file(server_testing_dir: str) -> str: + """ + Fixture to copy the redis.conf file from the merlin/server/ directory to the + temporary output directory and provide the path to the copied file. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :returns: The path to the redis configuration file we'll use for testing """ - # TODO - # - will probably have to do more than just copying over the conf file - # - likely want to create our own test conf file with the settings that - # can be modified by RedisConf instead - path_to_redis_conf = f"{os.path.dirname(os.path.abspath(__file__))}/../../merlin/server/redis.conf" - path_to_copied_redis = f"{temp_output_dir}/redis.conf" - shutil.copy(path_to_redis_conf, path_to_copied_redis) - return path_to_copied_redis \ No newline at end of file + redis_conf_file = f"{server_testing_dir}/redis.conf" + file_contents = """ + # ip address + bind 127.0.0.1 + + # port + port 6379 + + # password + requirepass merlin_password + + # directory + dir ./ + + # snapshot + save 300 100 + + # db file + dbfilename dump.rdb + + # append mode + appendfsync everysec + + # append file + appendfilename appendonly.aof + + # dummy trailing comment + """.strip().replace(" ", "") + + with open(redis_conf_file, "w") as rcf: + rcf.write(file_contents) + + return redis_conf_file diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 384e1ea37..c71e854eb 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,8 +1,10 @@ """ Tests for the `server_util.py` module. """ +import filecmp import os import pytest +import shutil from typing import Callable, Dict, Union from merlin.server.server_util import ( @@ -288,8 +290,60 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.container_format is None -# class TestRedisConfig: -# """Tests for the RedisConfig class.""" +class TestRedisConfig: + """Tests for the RedisConfig class.""" + + def test_initialization(self, server_redis_conf_file: str): + """ + Using a dummy redis configuration file, test that the initialization + of the RedisConfig class behaves as expected. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + expected_entries = { + "bind": "127.0.0.1", + "port": "6379", + "requirepass": "merlin_password", + "dir": "./", + "save": "300 100", + "dbfilename": "dump.rdb", + "appendfsync": "everysec", + "appendfilename": "appendonly.aof", + } + expected_comments = { + "bind": "# ip address\n", + "port": "\n# port\n", + "requirepass": "\n# password\n", + "dir": "\n# directory\n", + "save": "\n# snapshot\n", + "dbfilename": "\n# db file\n", + "appendfsync": "\n# append mode\n", + "appendfilename": "\n# append file\n", + } + expected_trailing_comment = "\n# dummy trailing comment" + expected_entry_order = list(expected_entries.keys()) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.filename == server_redis_conf_file + assert not redis_config.changed + assert redis_config.entries == expected_entries + assert redis_config.entry_order == expected_entry_order + assert redis_config.comments == expected_comments + assert redis_config.trailing_comments == expected_trailing_comment + + def test_write(self, server_redis_conf_file: str, server_testing_dir: str): + """ + """ + copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" + + # Create a RedisConf object with the basic redis conf file + redis_config = RedisConfig(server_redis_conf_file) + + # Change the filepath of the redis config file to be the copy that we'll write to + redis_config.filename = copy_redis_conf_file + + # Run the test + redis_config.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) -# def test_parse(self, server_redis_conf_file): -# raise ValueError From 9997d8e8f442349b4cc94c29a99cec3f4710b3f8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 4 Jun 2024 15:07:05 -0700 Subject: [PATCH 033/201] add tests for RedisConfig object --- merlin/server/server_commands.py | 4 +- merlin/server/server_util.py | 79 ++-- tests/unit/server/test_RedisConfig.py | 538 ++++++++++++++++++++++++++ tests/unit/server/test_server_util.py | 64 +-- 4 files changed, 577 insertions(+), 108 deletions(-) create mode 100644 tests/unit/server/test_RedisConfig.py diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 65d17c42b..40f2689d0 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -98,9 +98,7 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 redis_config.set_directory(args.directory) - redis_config.set_snapshot_seconds(args.snapshot_seconds) - - redis_config.set_snapshot_changes(args.snapshot_changes) + redis_config.set_snapshot(seconds=args.snapshot_seconds, changes=args.snapshot_changes) redis_config.set_snapshot_file(args.snapshot_file) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index aff641d4d..27a83376d 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -304,16 +304,14 @@ class RedisConfig: to write those changes into a redis readable config file. """ - filename = "" - entry_order = [] - entries = {} - comments = {} - trailing_comments = "" - changed = False - def __init__(self, filename) -> None: self.filename = filename self.changed = False + self.entry_order = [] + self.entries = {} + self.comments = {} + self.trailing_comments = "" + self.changed = False self.parse() def parse(self) -> None: @@ -393,7 +391,7 @@ def get_port(self) -> str: """Getter method to get the port from the redis config""" return self.get_config_value("port") - def set_port(self, port: str) -> bool: + def set_port(self, port: int) -> bool: """Validates and sets a given port""" if port is None: return False @@ -428,59 +426,56 @@ def set_directory(self, directory: str) -> bool: """ if directory is None: return False + # Create the directory if it doesn't exist if not os.path.exists(directory): os.mkdir(directory) LOG.info(f"Created directory {directory}") - # Validate the directory input - if os.path.exists(directory): - # Set the save directory to the redis config - if not self.set_config_value("dir", directory): - LOG.error("Unable to set directory for redis config") - return False - else: - LOG.error(f"Directory {directory} given does not exist and could not be created.") + # Set the save directory to the redis config + if not self.set_config_value("dir", directory): + LOG.error("Unable to set directory for redis config") return False LOG.info(f"Directory is set to {directory}") return True - def set_snapshot_seconds(self, seconds: int) -> bool: - """Sets the snapshot wait time""" - if seconds is None: - return False - # Set the snapshot second in the redis config - value = self.get_config_value("save") - if value is None: - LOG.error("Unable to get exisiting parameter values for snapshot") - return False + def set_snapshot(self, seconds: int = None, changes: int = None) -> bool: + """ + Sets the 'seconds' and/or 'changes' values of the snapshot setting, + depending on what the user requests. + + :param seconds: The first value of snapshot to change. If we're leaving it the + same this will be None. + :param changes: The second value of snapshot to change. If we're leaving it the + same this will be None. + :returns: True if successful, False otherwise. + """ - value = value.split() - value[0] = str(seconds) - value = " ".join(value) - if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") + # If both values are None, this method is doing nothing + if seconds is None and changes is None: return False - LOG.info(f"Snapshot wait time is set to {seconds} seconds") - return True - - def set_snapshot_changes(self, changes: int) -> bool: - """Sets the snapshot threshold""" - if changes is None: - return False - # Set the snapshot changes into the redis config + # Grab the snapshot value from the redis config value = self.get_config_value("save") if value is None: LOG.error("Unable to get exisiting parameter values for snapshot") return False + # Update the snapshot value value = value.split() - value[1] = str(changes) + log_msg = "" + if seconds is not None: + value[0] = str(seconds) + log_msg += f"Snapshot wait time is set to {seconds} seconds. " + if changes is not None: + value[1] = str(changes) + log_msg += f"Snapshot threshold is set to {changes} changes." value = " ".join(value) + + # Set the new snapshot value if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") + LOG.error("Unable to set snapshot value") return False - LOG.info(f"Snapshot threshold is set to {changes} changes") + LOG.info(log_msg) return True def set_snapshot_file(self, file: str) -> bool: @@ -508,7 +503,7 @@ def set_append_mode(self, mode: str) -> bool: LOG.error("Unable to set append_mode in redis config") return False else: - LOG.error("Not a valid append_mode(Only valid modes are always, everysec, no)") + LOG.error("Not a valid append_mode (Only valid modes are always, everysec, no)") return False LOG.info(f"Append mode is set to {mode}") diff --git a/tests/unit/server/test_RedisConfig.py b/tests/unit/server/test_RedisConfig.py new file mode 100644 index 000000000..12880d4d6 --- /dev/null +++ b/tests/unit/server/test_RedisConfig.py @@ -0,0 +1,538 @@ +""" +Tests for the RedisConfig class of the `server_util.py` module. + +This class is especially large so that's why these tests have been +moved to their own file. +""" +import filecmp +import logging +import pytest +from typing import Any + +from merlin.server.server_util import RedisConfig + +class TestRedisConfig: + """Tests for the RedisConfig class.""" + + def test_initialization(self, server_redis_conf_file: str): + """ + Using a dummy redis configuration file, test that the initialization + of the RedisConfig class behaves as expected. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + expected_entries = { + "bind": "127.0.0.1", + "port": "6379", + "requirepass": "merlin_password", + "dir": "./", + "save": "300 100", + "dbfilename": "dump.rdb", + "appendfsync": "everysec", + "appendfilename": "appendonly.aof", + } + expected_comments = { + "bind": "# ip address\n", + "port": "\n# port\n", + "requirepass": "\n# password\n", + "dir": "\n# directory\n", + "save": "\n# snapshot\n", + "dbfilename": "\n# db file\n", + "appendfsync": "\n# append mode\n", + "appendfilename": "\n# append file\n", + } + expected_trailing_comment = "\n# dummy trailing comment" + expected_entry_order = list(expected_entries.keys()) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.filename == server_redis_conf_file + assert not redis_config.changed + assert redis_config.entries == expected_entries + assert redis_config.entry_order == expected_entry_order + assert redis_config.comments == expected_comments + assert redis_config.trailing_comments == expected_trailing_comment + + def test_write(self, server_redis_conf_file: str, server_testing_dir: str): + """ + Test that the write functionality works by writing the contents of a dummy + configuration file to a blank configuration file. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" + + # Create a RedisConf object with the basic redis conf file + redis_config = RedisConfig(server_redis_conf_file) + + # Change the filepath of the redis config file to be the copy that we'll write to + redis_config.set_filename(copy_redis_conf_file) + + # Run the test + redis_config.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) + + @pytest.mark.parametrize("key, val, expected_return", [ + ("port", 1234, True), + ("invalid_key", "dummy_val", False) + ]) + def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, expected_return: bool): + """ + Test the `set_config_value` method with valid and invalid keys. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param key: The key value to modify with `set_config_value` + :param val: The value to set `key` to + :param expected_return: The expected return from `set_config_value` + """ + redis_config = RedisConfig(server_redis_conf_file) + actual_return = redis_config.set_config_value(key, val) + assert actual_return == expected_return + if expected_return: + assert redis_config.entries[key] == val + assert redis_config.changes_made() + else: + assert not redis_config.changes_made() + + @pytest.mark.parametrize("key, expected_val", [ + ("bind", "127.0.0.1"), + ("port", "6379"), + ("requirepass", "merlin_password"), + ("dir", "./"), + ("save", "300 100"), + ("dbfilename", "dump.rdb"), + ("appendfsync", "everysec"), + ("appendfilename", "appendonly.aof"), + ("invalid_key", None) + ]) + def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_val: str): + """ + Test the `get_config_value` method with valid and invalid keys. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param key: The key value to modify with `set_config_value` + :param expected_val: The value we're expecting to get by querying `key` + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.get_config_value(key) == expected_val + + @pytest.mark.parametrize("ip_to_set", [ + "127.0.0.1", # Most common IP + "0.0.0.0", # Edge case (low) + "255.255.255.255", # Edge case (high) + "123.222.199.20", # Random valid IP + ]) + def test_set_ip_address_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + ip_to_set: str + ): + """ + Test the `set_ip_address` method with valid ips. These should all return True + and set the 'bind' value to whatever `ip_to_set` is. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param ip_to_set: The ip address to set + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.set_ip_address(ip_to_set) + assert f"Ipaddress is set to {ip_to_set}" in caplog.text, "Missing expected log message" + assert redis_config.get_ip_address() == ip_to_set + + @pytest.mark.parametrize("ip_to_set, expected_log", [ + (None, None), # No IP + ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 + ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist + ]) + def test_set_ip_address_invalid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + ip_to_set: str, + expected_log: str, + ): + """ + Test the `set_ip_address` method with invalid ips. These should all return False. + and not modify the 'bind' setting. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param ip_to_set: The ip address to set + :param expected_log: The string we're expecting the logger to log + """ + redis_config = RedisConfig(server_redis_conf_file) + # For the test where bind is unset, delete bind from dict and set new ip val to a valid value + if ip_to_set == "bind-unset": + del redis_config.entries["bind"] + ip_to_set = "127.0.0.1" + assert not redis_config.set_ip_address(ip_to_set) + assert redis_config.get_ip_address() != ip_to_set + if expected_log is not None: + assert expected_log in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("port_to_set", [ + 6379, # Most common port + 1, # Edge case (low) + 65535, # Edge case (high) + 12345, # Random valid port + ]) + def test_set_port_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + port_to_set: str, + ): + """ + Test the `set_port` method with valid ports. These should all return True + and set the 'port' value to whatever `port_to_set` is. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param port_to_set: The port to set + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.set_port(port_to_set) + assert redis_config.get_port() == port_to_set + assert f"Port is set to {port_to_set}" in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("port_to_set, expected_log", [ + (None, None), # No port + (0, "Invalid port given."), # Edge case (low) + (65536, "Invalid port given."), # Edge case (high) + ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist + ]) + def test_set_port_invalid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + port_to_set: str, + expected_log: str, + ): + """ + Test the `set_port` method with invalid inputs. These should all return False + and not modify the 'port' setting. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param port_to_set: The port to set + :param expected_log: The string we're expecting the logger to log + """ + redis_config = RedisConfig(server_redis_conf_file) + # For the test where port is unset, delete port from dict and set port val to a valid value + if port_to_set == "port-unset": + del redis_config.entries["port"] + port_to_set = 5 + assert not redis_config.set_port(port_to_set) + assert redis_config.get_port() != port_to_set + if expected_log is not None: + assert expected_log in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("pass_to_set, expected_return", [ + ("valid_password", True), # Valid password + (None, False), # Invalid password + ]) + def test_set_password( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + pass_to_set: str, + expected_return: bool, + ): + """ + Test the `set_password` method with both valid and invalid input. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param pass_to_set: The password to set + :param expected_return: The expected return value + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.set_password(pass_to_set) == expected_return + if expected_return: + assert redis_conf.get_password() == pass_to_set + assert "New password set" in caplog.text, "Missing expected log message" + + def test_set_directory_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + server_testing_dir: str, + ): + """ + Test the `set_directory` method with valid input. This should return True, modify the + 'dir' value, and log some messages about creating/setting the directory. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + dir_to_set = f"{server_testing_dir}/dummy_dir" + assert redis_config.set_directory(dir_to_set) + assert redis_config.get_config_value("dir") == dir_to_set + assert f"Created directory {dir_to_set}" in caplog.text, "Missing created log message" + assert f"Directory is set to {dir_to_set}" in caplog.text, "Missing set log message" + + def test_set_directory_none(self, server_redis_conf_file: str): + """ + Test the `set_directory` method with None as the input. This should return False + and not modify the 'dir' setting. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_config = RedisConfig(server_redis_conf_file) + assert not redis_config.set_directory(None) + assert redis_config.get_config_value("dir") != None + + def test_set_directory_dir_unset( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + server_testing_dir: str, + ): + """ + Test the `set_directory` method with the 'dir' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + redis_config = RedisConfig(server_redis_conf_file) + del redis_config.entries["dir"] + dir_to_set = f"{server_testing_dir}/dummy_dir" + assert not redis_config.set_directory(dir_to_set) + assert "Unable to set directory for redis config" in caplog.text, "Missing expected log message" + + def test_set_snapshot_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'seconds' and 'changes'. + This should return True and modify both values of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + snap_sec_to_set = 20 + snap_changes_to_set = 30 + assert redis_conf.set_snapshot(seconds=snap_sec_to_set, changes=snap_changes_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == str(snap_sec_to_set) + assert save_val[1] == str(snap_changes_to_set) + expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " \ + f"Snapshot threshold is set to {snap_changes_to_set} changes" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_just_seconds(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'seconds'. This should + return True and modify the first value of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + orig_save = redis_conf.get_config_value("save").split() + snap_sec_to_set = 20 + assert redis_conf.set_snapshot(seconds=snap_sec_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == str(snap_sec_to_set) + assert save_val[1] == orig_save[1] + expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_just_changes(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'changes'. This should + return True and modify the second value of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + orig_save = redis_conf.get_config_value("save").split() + snap_changes_to_set = 30 + assert redis_conf.set_snapshot(changes=snap_changes_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == orig_save[0] + assert save_val[1] == str(snap_changes_to_set) + expected_log = f"Snapshot threshold is set to {snap_changes_to_set} changes" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_none(self, server_redis_conf_file: str): + """ + Test the `set_snapshot` method with None as the input for both seconds + and changes. This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_snapshot(seconds=None, changes=None) + + def test_set_snapshot_save_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with the 'save' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["save"] + assert not redis_conf.set_snapshot(seconds=20) + assert "Unable to get exisiting parameter values for snapshot" in caplog.text, "Missing expected log message" + + def test_set_snapshot_file_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot_file` method with a valid input. This should + return True and modify the value of 'dbfilename'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + filename = "dummy_file.rdb" + assert redis_conf.set_snapshot_file(filename) + assert redis_conf.get_config_value("dbfilename") == filename + assert f"Snapshot file is set to {filename}" in caplog.text, "Missing expected log message" + + def test_set_snapshot_file_none(self, server_redis_conf_file: str): + """ + Test the `set_snapshot_file` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_snapshot_file(None) + + def test_set_snapshot_file_dbfilename_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with the 'dbfilename' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["dbfilename"] + filename = "dummy_file.rdb" + assert not redis_conf.set_snapshot_file(filename) + assert redis_conf.get_config_value("dbfilename") != filename + assert "Unable to set snapshot_file name" in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("mode_to_set", [ + "always", + "everysec", + "no", + ]) + def test_set_append_mode_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + mode_to_set: str, + ): + """ + Test the `set_append_mode` method with valid modes. These should all return True + and modify the value of 'appendfsync'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param mode_to_set: The mode to set + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.set_append_mode(mode_to_set) + assert redis_conf.get_config_value("appendfsync") == mode_to_set + assert f"Append mode is set to {mode_to_set}" in caplog.text, "Missing expected log message" + + def test_set_append_mode_invalid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_mode` method with an invalid mode. This should return False + and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + invalid_mode = "invalid" + assert not redis_conf.set_append_mode(invalid_mode) + assert redis_conf.get_config_value("appendfsync") != invalid_mode + expected_log = "Not a valid append_mode (Only valid modes are always, everysec, no)" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_append_mode_none(self, server_redis_conf_file: str): + """ + Test the `set_append_mode` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_append_mode(None) + + def test_set_append_mode_appendfsync_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_mode` method with the 'appendfsync' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["appendfsync"] + mode = "no" + assert not redis_conf.set_append_mode(mode) + assert redis_conf.get_config_value("appendfsync") != mode + assert "Unable to set append_mode in redis config" in caplog.text, "Missing expected log message" + + def test_set_append_file_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_file` method with a valid file. This should return True + and modify the value of 'appendfilename'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + valid_file = "valid" + assert redis_conf.set_append_file(valid_file) + assert redis_conf.get_config_value("appendfilename") == f'"{valid_file}"' + assert f"Append file is set to {valid_file}" in caplog.text, "Missing expected log message" + + def test_set_append_file_none(self, server_redis_conf_file: str): + """ + Test the `set_append_file` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_append_file(None) + + def test_set_append_file_appendfilename_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_file` method with the 'appendfilename' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["appendfilename"] + filename = "valid_filename" + assert not redis_conf.set_append_file(filename) + assert redis_conf.get_config_value("appendfilename") != filename + assert "Unable to set append filename." in caplog.text, "Missing expected log message" diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index c71e854eb..0332be944 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,18 +1,15 @@ """ Tests for the `server_util.py` module. """ -import filecmp import os import pytest -import shutil -from typing import Callable, Dict, Union +from typing import Dict, Union from merlin.server.server_util import ( AppYaml, ContainerConfig, ContainerFormatConfig, ProcessConfig, - RedisConfig, RedisUsers, ServerConfig, valid_ipv4, @@ -288,62 +285,3 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.process == ProcessConfig(server_process_config_data) assert config.container is None assert config.container_format is None - - -class TestRedisConfig: - """Tests for the RedisConfig class.""" - - def test_initialization(self, server_redis_conf_file: str): - """ - Using a dummy redis configuration file, test that the initialization - of the RedisConfig class behaves as expected. - - :param server_redis_conf_file: The path to a dummy redis configuration file - """ - expected_entries = { - "bind": "127.0.0.1", - "port": "6379", - "requirepass": "merlin_password", - "dir": "./", - "save": "300 100", - "dbfilename": "dump.rdb", - "appendfsync": "everysec", - "appendfilename": "appendonly.aof", - } - expected_comments = { - "bind": "# ip address\n", - "port": "\n# port\n", - "requirepass": "\n# password\n", - "dir": "\n# directory\n", - "save": "\n# snapshot\n", - "dbfilename": "\n# db file\n", - "appendfsync": "\n# append mode\n", - "appendfilename": "\n# append file\n", - } - expected_trailing_comment = "\n# dummy trailing comment" - expected_entry_order = list(expected_entries.keys()) - redis_config = RedisConfig(server_redis_conf_file) - assert redis_config.filename == server_redis_conf_file - assert not redis_config.changed - assert redis_config.entries == expected_entries - assert redis_config.entry_order == expected_entry_order - assert redis_config.comments == expected_comments - assert redis_config.trailing_comments == expected_trailing_comment - - def test_write(self, server_redis_conf_file: str, server_testing_dir: str): - """ - """ - copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" - - # Create a RedisConf object with the basic redis conf file - redis_config = RedisConfig(server_redis_conf_file) - - # Change the filepath of the redis config file to be the copy that we'll write to - redis_config.filename = copy_redis_conf_file - - # Run the test - redis_config.write() - - # Check that the contents of the copied file match the contents of the basic file - assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) - From 52213f245a2d5e11d94f2fc58bb4ccfa8f5cbf56 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 4 Jun 2024 16:41:12 -0700 Subject: [PATCH 034/201] add tests for RedisUsers class --- merlin/server/server_util.py | 2 +- tests/fixtures/server.py | 48 +++++++- tests/unit/server/test_server_util.py | 167 ++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 3 deletions(-) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 27a83376d..9b7233097 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -623,7 +623,7 @@ def set_password(self, user: str, password: str): self.users[user].set_password(password) return True - def remove_user(self, user) -> bool: + def remove_user(self, user: str) -> bool: """Remove a user from the dict of users""" if user in self.users: del self.users[user] diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 04c858f46..5c5dea102 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -3,6 +3,7 @@ """ import os import pytest +import yaml from typing import Dict @pytest.fixture(scope="class") @@ -90,8 +91,7 @@ def server_testing_dir(temp_output_dir: str) -> str: @pytest.fixture(scope="session") def server_redis_conf_file(server_testing_dir: str) -> str: """ - Fixture to copy the redis.conf file from the merlin/server/ directory to the - temporary output directory and provide the path to the copied file. + Fixture to write a redis.conf file to the temporary output directory. If a test will modify this file with a file write, you should make a copy of this file to modify instead. @@ -132,3 +132,47 @@ def server_redis_conf_file(server_testing_dir: str) -> str: rcf.write(file_contents) return redis_conf_file + +@pytest.fixture(scope="session") +def server_users() -> dict: + """ + Create a dictionary of two test users with identical configuration settings. + + :returns: A dict containing the two test users and their settings + """ + users = { + "default": { + "channels": '*', + "commands": '@all', + "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', + "keys": '*', + "status": 'on', + }, + "test_user": { + "channels": '*', + "commands": '@all', + "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', + "keys": '*', + "status": 'on', + } + } + return users + +@pytest.fixture(scope="session") +def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: + """ + Fixture to write a redis.users file to the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :param server_users: A dict of test user configurations + :returns: The path to the redis user configuration file we'll use for testing + """ + redis_users_file = f"{server_testing_dir}/redis.users" + + with open(redis_users_file, "w") as ruf: + yaml.dump(server_users, ruf) + + return redis_users_file \ No newline at end of file diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 0332be944..61f29293c 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,6 +1,8 @@ """ Tests for the `server_util.py` module. """ +import filecmp +import hashlib import os import pytest from typing import Dict, Union @@ -285,3 +287,168 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.process == ProcessConfig(server_process_config_data) assert config.container is None assert config.container_format is None + +class TestRedisUsers: + """ + Tests for the RedisUsers class. + + TODO add integration test(s) for `apply_to_redis` method of this class. + """ + + class TestUser: + """Tests for the RedisUsers.User class""" + + def test_initializaiton(self): + """Test the initialization process of the User class.""" + user = RedisUsers.User() + assert user.status == "on" + assert user.hash_password == hashlib.sha256(b"password").hexdigest() + assert user.keys == "*" + assert user.channels == "*" + assert user.commands == "@all" + + def test_parse_dict(self): + """Test the `parse_dict` method of the User class.""" + test_dict = { + "status": "test_status", + "hash_password": "test_password", + "keys": "test_keys", + "channels": "test_channels", + "commands": "test_commands", + } + user = RedisUsers.User() + user.parse_dict(test_dict) + assert user.status == test_dict["status"] + assert user.hash_password == test_dict["hash_password"] + assert user.keys == test_dict["keys"] + assert user.channels == test_dict["channels"] + assert user.commands == test_dict["commands"] + + def test_get_user_dict(self): + """Test the `get_user_dict` method of the User class.""" + test_dict = { + "status": "test_status", + "hash_password": "test_password", + "keys": "test_keys", + "channels": "test_channels", + "commands": "test_commands", + "invalid_key": "invalid_val", + } + user = RedisUsers.User() + user.parse_dict(test_dict) # Set the test values + actual_dict = user.get_user_dict() + assert "invalid_key" not in actual_dict # Check that the invalid key isn't parsed + + # Check that the values are as expected + for key, val in actual_dict.items(): + if key == "status": + assert val == "on" + else: + assert val == test_dict[key] + + def test_set_password(self): + """Test the `set_password` method of the User class.""" + user = RedisUsers.User() + pass_to_set = "dummy_password" + user.set_password(pass_to_set) + assert user.hash_password == hashlib.sha256(bytes(pass_to_set, "utf-8")).hexdigest() + + def test_initialization(self, server_redis_users_file: str, server_users: dict): + """ + Test the initialization process of the RedisUsers class. + + :param server_redis_users_file: The path to a dummy redis users file + :param server_users: A dict of test user configurations + """ + redis_users = RedisUsers(server_redis_users_file) + assert redis_users.filename == server_redis_users_file + assert len(redis_users.users) == len(server_users) + + def test_write(self, server_redis_users_file: str, server_testing_dir: str): + """ + Test that the write functionality works by writing the contents of a dummy + users file to a blank users file. + + :param server_redis_users_file: The path to a dummy redis users file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_redis_users_file = f"{server_testing_dir}/redis_copy.users" + + # Create a RedisUsers object with the basic redis users file + redis_users = RedisUsers(server_redis_users_file) + + # Change the filepath of the redis users file to be the copy that we'll write to + redis_users.filename = copy_redis_users_file + + # Run the test + redis_users.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_users_file, copy_redis_users_file) + + def test_add_user_nonexistent(self, server_redis_users_file: str): + """ + Test the `add_user` method with a user that doesn't exists. + This should return True and add the user to the list of users. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + num_users_before = len(redis_users.users) + assert redis_users.add_user("new_user") + assert len(redis_users.users) == num_users_before + 1 + + def test_add_user_exists(self, server_redis_users_file: str): + """ + Test the `add_user` method with a user that already exists. + This should return False. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.add_user("test_user") + + def test_set_password_valid(self, server_redis_users_file: str): + """ + Test the `set_password` method with a user that exists. + This should return True and change the password for the user. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + pass_to_set = "new_password" + assert redis_users.set_password("test_user", pass_to_set) + expected_hash_pass = hashlib.sha256(bytes(pass_to_set, "utf-8")).hexdigest() + assert redis_users.users["test_user"].hash_password == expected_hash_pass + + def test_set_password_invalid(self, server_redis_users_file: str): + """ + Test the `set_password` method with a user that doesn't exist. + This should return False. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.set_password("nonexistent_user", "new_password") + + def test_remove_user_valid(self, server_redis_users_file: str): + """ + Test the `remove_user` method with a user that exists. + This should return True and remove the user from the list of users. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + num_users_before = len(redis_users.users) + assert redis_users.remove_user("test_user") + assert len(redis_users.users) == num_users_before - 1 + + def test_remove_user_invalid(self, server_redis_users_file: str): + """ + Test the `remove_user` method with a user that doesn't exist. + This should return False and not modify the user list. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.remove_user("nonexistent_user") \ No newline at end of file From a59243ffc65a77c9ce3dc2b67efd62a8dce06fc2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 10:01:03 -0700 Subject: [PATCH 035/201] change server fixtures to use redis config files --- tests/fixtures/server.py | 174 +++++++++++++++----------- tests/unit/server/test_server_util.py | 6 +- 2 files changed, 107 insertions(+), 73 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 5c5dea102..ae3a966c8 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -6,72 +6,6 @@ import yaml from typing import Dict -@pytest.fixture(scope="class") -def server_container_config_data(temp_output_dir: str) -> Dict[str, str]: - """ - Fixture to provide sample data for ContainerConfig tests - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: A dict containing the necessary key/values for the ContainerConfig object - """ - return { - "format": "docker", - "image_type": "postgres", - "image": "postgres:latest", - "url": "postgres://localhost", - "config": "postgres.conf", - "config_dir": "/path/to/config", - "pfile": "merlin_server_postgres.pf", - "pass_file": f"{temp_output_dir}/postgres.pass", - "user_file": "postgres.users", - } - -@pytest.fixture(scope="class") -def server_container_format_config_data() -> Dict[str, str]: - """ - Fixture to provide sample data for ContainerFormatConfig tests - - :returns: A dict containing the necessary key/values for the ContainerFormatConfig object - """ - return { - "command": "docker", - "run_command": "{command} run --name {name} -d {image}", - "stop_command": "{command} stop {name}", - "pull_command": "{command} pull {url}", - } - -@pytest.fixture(scope="class") -def server_process_config_data() -> Dict[str, str]: - """ - Fixture to provide sample data for ProcessConfig tests - - :returns: A dict containing the necessary key/values for the ProcessConfig object - """ - return { - "status": "status {pid}", - "kill": "terminate {pid}", - } - -@pytest.fixture(scope="class") -def server_server_config( - server_container_config_data: Dict[str, str], - server_process_config_data: Dict[str, str], - server_container_format_config_data: Dict[str, str], -) -> Dict[str, Dict[str, str]]: - """ - Fixture to provide sample data for ServerConfig tests - - :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class - :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class - :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class - :returns: A dictionary containing each of the configuration dicts we'll need - """ - return { - "container": server_container_config_data, - "process": server_process_config_data, - "docker": server_container_format_config_data, - } - @pytest.fixture(scope="session") def server_testing_dir(temp_output_dir: str) -> str: @@ -81,7 +15,7 @@ def server_testing_dir(temp_output_dir: str) -> str: :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :returns: The path to the temporary testing directory for server tests """ - testing_dir = f"{temp_output_dir}/server_testing/" + testing_dir = f"{temp_output_dir}/server_testing" if not os.path.exists(testing_dir): os.mkdir(testing_dir) @@ -96,7 +30,7 @@ def server_redis_conf_file(server_testing_dir: str) -> str: If a test will modify this file with a file write, you should make a copy of this file to modify instead. - :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to :returns: The path to the redis configuration file we'll use for testing """ redis_conf_file = f"{server_testing_dir}/redis.conf" @@ -133,6 +67,26 @@ def server_redis_conf_file(server_testing_dir: str) -> str: return redis_conf_file + +@pytest.fixture(scope="session") +def server_redis_pass_file(server_testing_dir: str) -> str: + """ + Fixture to create a redis password file in the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :returns: The path to the redis password file + """ + redis_pass_file = f"{server_testing_dir}/redis.pass" + + with open(redis_pass_file, "w") as rpf: + rpf.write("server-tests-password") + + return redis_pass_file + + @pytest.fixture(scope="session") def server_users() -> dict: """ @@ -158,6 +112,7 @@ def server_users() -> dict: } return users + @pytest.fixture(scope="session") def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: """ @@ -166,7 +121,7 @@ def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: If a test will modify this file with a file write, you should make a copy of this file to modify instead. - :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to :param server_users: A dict of test user configurations :returns: The path to the redis user configuration file we'll use for testing """ @@ -175,4 +130,83 @@ def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: with open(redis_users_file, "w") as ruf: yaml.dump(server_users, ruf) - return redis_users_file \ No newline at end of file + return redis_users_file + + +@pytest.fixture(scope="class") +def server_container_config_data( + server_testing_dir: str, + server_redis_conf_file: str, + server_redis_pass_file: str, + server_redis_users_file: str, +) -> Dict[str, str]: + """ + Fixture to provide sample data for ContainerConfig tests. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_redis_conf_file: A pytest fixture that defines a path to a redis configuration file + :param server_redis_pass_file: A pytest fixture that defines a path to a redis password file + :param server_redis_users_file: A pytest fixture that defines a path to a redis users file + :returns: A dict containing the necessary key/values for the ContainerConfig object + """ + + return { + "format": "singularity", + "image_type": "redis", + "image": "redis_latest.sif", + "url": "docker://redis", + "config": server_redis_conf_file.split("/")[-1], + "config_dir": server_testing_dir, + "pfile": "merlin_server.pf", + "pass_file": server_redis_pass_file.split("/")[-1], + "user_file": server_redis_users_file.split("/")[-1], + } + + +@pytest.fixture(scope="class") +def server_container_format_config_data() -> Dict[str, str]: + """ + Fixture to provide sample data for ContainerFormatConfig tests + + :returns: A dict containing the necessary key/values for the ContainerFormatConfig object + """ + return { + "command": "singularity", + "run_command": "{command} run -H {home_dir} {image} {config}", + "stop_command": "kill", + "pull_command": "{command} pull {image} {url}", + } + + +@pytest.fixture(scope="class") +def server_process_config_data() -> Dict[str, str]: + """ + Fixture to provide sample data for ProcessConfig tests + + :returns: A dict containing the necessary key/values for the ProcessConfig object + """ + return { + "status": "pgrep -P {pid}", + "kill": "kill {pid}", + } + + +@pytest.fixture(scope="class") +def server_server_config( + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], + server_container_format_config_data: Dict[str, str], +) -> Dict[str, Dict[str, str]]: + """ + Fixture to provide sample data for ServerConfig tests + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :returns: A dictionary containing each of the configuration dicts we'll need + """ + return { + "container": server_container_config_data, + "process": server_process_config_data, + "singularity": server_container_format_config_data, + } diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 61f29293c..2986e22de 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -169,7 +169,7 @@ def test_get_container_password(self, server_container_config_data: Dict[str, st :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ # Write a fake password to the password file - test_password = "super-secret-password" + test_password = "server-tests-password" with open(server_container_config_data["pass_file"], "w") as pass_file: pass_file.write(test_password) @@ -274,7 +274,7 @@ def test_init_with_complete_data(self, server_server_config: Dict[str, str]): config = ServerConfig(server_server_config) assert config.container == ContainerConfig(server_server_config["container"]) assert config.process == ProcessConfig(server_server_config["process"]) - assert config.container_format == ContainerFormatConfig(server_server_config["docker"]) + assert config.container_format == ContainerFormatConfig(server_server_config["singularity"]) def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): """ @@ -451,4 +451,4 @@ def test_remove_user_invalid(self, server_redis_users_file: str): :param server_redis_users_file: The path to a dummy redis users file """ redis_users = RedisUsers(server_redis_users_file) - assert not redis_users.remove_user("nonexistent_user") \ No newline at end of file + assert not redis_users.remove_user("nonexistent_user") From 0ef586e4d60ab7d0ca228ba9c311ef4d3924211b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 10:55:35 -0700 Subject: [PATCH 036/201] add tests for AppYaml class --- tests/fixtures/server.py | 62 ++++++++++++++++++++++++ tests/unit/server/test_server_util.py | 70 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index ae3a966c8..284084e2c 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -210,3 +210,65 @@ def server_server_config( "process": server_process_config_data, "singularity": server_container_format_config_data, } + + +@pytest.fixture(scope="function") +def server_app_yaml_contents( + server_redis_pass_file: str, + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], +) -> Dict[str, str]: + """ + Fixture to create the contents of an app.yaml file. + + :param server_redis_pass_file: A pytest fixture that defines a path to a redis password file + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :returns: A dict with typical app.yaml contents + """ + contents = { + "broker": { + "cert_reqs": "none", + "name": "redis", + "password": server_redis_pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "testhost", + }, + "container": server_container_config_data, + "process": server_process_config_data, + "results_backend": { + "cert_reqs": "none", + "db_num": 0, + "name": "redis", + "password": server_redis_pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + } + } + return contents + + +@pytest.fixture(scope="function") +def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> str: + """ + Fixture to create an app.yaml file in the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + NOTE this must be function scoped since server_app_yaml_contents is function scoped. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_app_yaml_contents: A pytest fixture that creates a dict of contents for an app.yaml file + :returns: The path to the app.yaml file + """ + app_yaml_file = f"{server_testing_dir}/app.yaml" + + if not os.path.exists(app_yaml_file): + with open(app_yaml_file, "w") as ayf: + yaml.dump(server_app_yaml_contents, ayf) + + return app_yaml_file \ No newline at end of file diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 2986e22de..20ff922bb 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -12,6 +12,7 @@ ContainerConfig, ContainerFormatConfig, ProcessConfig, + RedisConfig, RedisUsers, ServerConfig, valid_ipv4, @@ -288,6 +289,7 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.container is None assert config.container_format is None + class TestRedisUsers: """ Tests for the RedisUsers class. @@ -452,3 +454,71 @@ def test_remove_user_invalid(self, server_redis_users_file: str): """ redis_users = RedisUsers(server_redis_users_file) assert not redis_users.remove_user("nonexistent_user") + + +class TestAppYaml: + """Tests for the AppYaml class.""" + + def test_initialization(self, server_app_yaml: str, server_app_yaml_contents: dict): + """ + Test the initialization process of the AppYaml class. + + :param server_app_yaml: The path to an app.yaml file + :param server_app_yaml_contents: A dict of app.yaml configurations + """ + app_yaml = AppYaml(server_app_yaml) + assert app_yaml.get_data() == server_app_yaml_contents + + def test_apply_server_config(self, server_app_yaml: str, server_server_config: Dict[str, str]): + """ + Test the `apply_server_config` method. This should update the data attribute. + + :param server_app_yaml: The path to an app.yaml file + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + app_yaml = AppYaml(server_app_yaml) + server_config = ServerConfig(server_server_config) + redis_config = RedisConfig(server_config.container.get_config_path()) + app_yaml.apply_server_config(server_config) + + assert app_yaml.data[app_yaml.broker_name]["name"] == server_config.container.get_image_type() + assert app_yaml.data[app_yaml.broker_name]["username"] == "default" + assert app_yaml.data[app_yaml.broker_name]["password"] == server_config.container.get_pass_file_path() + assert app_yaml.data[app_yaml.broker_name]["server"] == redis_config.get_ip_address() + assert app_yaml.data[app_yaml.broker_name]["port"] == redis_config.get_port() + + assert app_yaml.data[app_yaml.results_name]["name"] == server_config.container.get_image_type() + assert app_yaml.data[app_yaml.results_name]["username"] == "default" + assert app_yaml.data[app_yaml.results_name]["password"] == server_config.container.get_pass_file_path() + assert app_yaml.data[app_yaml.results_name]["server"] == redis_config.get_ip_address() + assert app_yaml.data[app_yaml.results_name]["port"] == redis_config.get_port() + + def test_update_data(self, server_app_yaml: str): + """ + Test the `update_data` method. This should update the data attribute. + + :param server_app_yaml: The path to an app.yaml file + """ + app_yaml = AppYaml(server_app_yaml) + new_data = {app_yaml.broker_name: {"username": "new_user"}} + app_yaml.update_data(new_data) + + assert app_yaml.data[app_yaml.broker_name]["username"] == "new_user" + + def test_write(self, server_app_yaml: str, server_testing_dir: str): + """ + Test the `write` method. This should write data to a file. + + :param server_app_yaml: The path to an app.yaml file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_app_yaml = f"{server_testing_dir}/app_copy.yaml" + + # Create a AppYaml object with the basic app.yaml file + app_yaml = AppYaml(server_app_yaml) + + # Run the test + app_yaml.write(copy_app_yaml) + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_app_yaml, copy_app_yaml) From bde90797c0938ea6844fd74ed4e23a28eab341dd Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 12:06:09 -0700 Subject: [PATCH 037/201] final cleanup of server_utils --- tests/fixtures/server.py | 9 ++-- tests/unit/server/test_server_util.py | 67 ++++++++++++++++----------- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 284084e2c..01db7bd56 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -4,7 +4,7 @@ import os import pytest import yaml -from typing import Dict +from typing import Dict, Union @pytest.fixture(scope="session") @@ -88,7 +88,7 @@ def server_redis_pass_file(server_testing_dir: str) -> str: @pytest.fixture(scope="session") -def server_users() -> dict: +def server_users() -> Dict[str, Dict[str, str]]: """ Create a dictionary of two test users with identical configuration settings. @@ -217,7 +217,7 @@ def server_app_yaml_contents( server_redis_pass_file: str, server_container_config_data: Dict[str, str], server_process_config_data: Dict[str, str], -) -> Dict[str, str]: +) -> Dict[str, Union[str, int]]: """ Fixture to create the contents of an app.yaml file. @@ -256,9 +256,6 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> """ Fixture to create an app.yaml file in the temporary output directory. - If a test will modify this file with a file write, you should make a copy of - this file to modify instead. - NOTE this must be function scoped since server_app_yaml_contents is function scoped. :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 20ff922bb..c9b59e83e 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -30,8 +30,8 @@ def test_valid_ipv4_valid_ip(valid_ip: str): Test the `valid_ipv4` function with valid IPs. This should return True. - :param valid_ip: A valid port to test. - These are pulled from the parametrized list defined above this test. + :param valid_ip: A valid port to test. These are pulled from the parametrized + list defined above this test. """ assert valid_ipv4(valid_ip) @@ -47,8 +47,8 @@ def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): An IP is valid if every integer separated by the '.' delimiter are between 0 and 255. This should return False for both IPs tested here. - :param invalid_ip: An invalid port to test. - These are pulled from the parametrized list defined above this test. + :param invalid_ip: An invalid port to test. These are pulled from the parametrized + list defined above this test. """ assert not valid_ipv4(invalid_ip) @@ -63,8 +63,8 @@ def test_valid_port_valid_input(valid_input: int): Valid ports are ports between 1 and 65535. This should return True. - :param valid_input: A valid input value to test. - These are pulled from the parametrized list defined above this test. + :param valid_input: A valid input value to test. These are pulled from the parametrized + list defined above this test. """ assert valid_port(valid_input) @@ -79,8 +79,8 @@ def test_valid_port_invalid_input(invalid_input: int): Valid ports are ports between 1 and 65535. This should return False for each invalid input tested. - :param invalid_input: An invalid input value to test. - These are pulled from the parametrized list defined above this test. + :param invalid_input: An invalid input value to test. These are pulled from the parametrized + list defined above this test. """ assert not valid_port(invalid_input) @@ -90,7 +90,7 @@ class TestContainerConfig: def test_init_with_complete_data(self, server_container_config_data: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ @@ -107,7 +107,7 @@ def test_init_with_complete_data(self, server_container_config_data: Dict[str, s def test_init_with_missing_data(self): """ - Tests that __init__ uses defaults for missing data + Tests that __init__ uses defaults for missing data. """ incomplete_data = {"format": "docker"} config = ContainerConfig(incomplete_data) @@ -130,7 +130,7 @@ def test_init_with_missing_data(self): ]) def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): """ - Tests that get_*_path methods construct the correct path + Tests that get_*_path methods construct the correct path. :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class :param attr_name: Name of the attribute to be tested. These are pulled from the parametrized list defined above this test. @@ -153,7 +153,7 @@ def test_get_path_methods(self, server_container_config_data: Dict[str, str], at ]) def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ - Tests that all getter methods return the correct attribute values + Tests that all getter methods return the correct attribute values. :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. @@ -163,20 +163,31 @@ def test_getter_methods(self, server_container_config_data: Dict[str, str], gett getter = getattr(config, getter_name) assert getter() == server_container_config_data[expected_attr] - def test_get_container_password(self, server_container_config_data: Dict[str, str]): + def test_get_container_password(self, server_testing_dir: str, server_container_config_data: Dict[str, str]): """ - Test that the get_container_password is reading the password file properly + Test that the `get_container_password` method is reading the password file properly. + :param server_testing_dir: The path to the the temp output directory for server tests :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ # Write a fake password to the password file - test_password = "server-tests-password" - with open(server_container_config_data["pass_file"], "w") as pass_file: + test_password = "super-secret-password" + temp_pass_file = f"{server_testing_dir}/temp.pass" + with open(temp_pass_file, "w") as pass_file: pass_file.write(test_password) - # Run the test - config = ContainerConfig(server_container_config_data) - assert config.get_container_password() == test_password + # Use temp pass file + orig_pass_file = server_container_config_data["pass_file"] + server_container_config_data["pass_file"] = temp_pass_file + + try: + # Run the test + config = ContainerConfig(server_container_config_data) + assert config.get_container_password() == test_password + except Exception as exc: + # If there was a problem, reset to the original password file + server_container_config_data["pass_file"] = orig_pass_file + raise exc class TestContainerFormatConfig: @@ -184,7 +195,7 @@ class TestContainerFormatConfig: def test_init_with_complete_data(self, server_container_format_config_data: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class """ @@ -196,7 +207,7 @@ def test_init_with_complete_data(self, server_container_format_config_data: Dict def test_init_with_missing_data(self): """ - Tests that __init__ uses defaults for missing data + Tests that __init__ uses defaults for missing data. """ incomplete_data = {"command": "docker"} config = ContainerFormatConfig(incomplete_data) @@ -213,7 +224,7 @@ def test_init_with_missing_data(self): ]) def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ - Tests that all getter methods return the correct attribute values + Tests that all getter methods return the correct attribute values. :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. @@ -229,7 +240,7 @@ class TestProcessConfig: def test_init_with_complete_data(self, server_process_config_data: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class """ @@ -239,7 +250,7 @@ def test_init_with_complete_data(self, server_process_config_data: Dict[str, str def test_init_with_missing_data(self): """ - Tests that __init__ uses defaults for missing data + Tests that __init__ uses defaults for missing data. """ incomplete_data = {"status": "status {pid}"} config = ProcessConfig(incomplete_data) @@ -252,7 +263,7 @@ def test_init_with_missing_data(self): ]) def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ - Tests that all getter methods return the correct attribute values + Tests that all getter methods return the correct attribute values. :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. @@ -268,7 +279,7 @@ class TestServerConfig: def test_init_with_complete_data(self, server_server_config: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ @@ -279,7 +290,7 @@ def test_init_with_complete_data(self, server_server_config: Dict[str, str]): def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): """ - Tests that __init__ uses None for missing data + Tests that __init__ uses None for missing data. :param server_process_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ @@ -298,7 +309,7 @@ class TestRedisUsers: """ class TestUser: - """Tests for the RedisUsers.User class""" + """Tests for the RedisUsers.User class.""" def test_initializaiton(self): """Test the initialization process of the User class.""" From da94020e72e5aa66544a30efe18969dbd04ee704 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 12:31:00 -0700 Subject: [PATCH 038/201] fix lint issues --- merlin/examples/generator.py | 2 +- merlin/server/server_util.py | 8 +- tests/conftest.py | 15 ++- tests/context_managers/server_manager.py | 1 + tests/fixtures/server.py | 37 +++--- tests/fixtures/status.py | 5 +- tests/unit/common/test_dumper.py | 34 ++++-- tests/unit/common/test_encryption.py | 1 + tests/unit/common/test_sample_index.py | 1 + tests/unit/common/test_util_sampling.py | 1 + tests/unit/config/test_broker.py | 1 + tests/unit/config/test_config_object.py | 1 + tests/unit/config/test_configfile.py | 1 + tests/unit/config/test_results_backend.py | 1 + tests/unit/server/test_RedisConfig.py | 132 ++++++++++++--------- tests/unit/server/test_server_util.py | 138 +++++++++++++--------- tests/unit/test_examples_generator.py | 1 + tests/utils.py | 1 + 18 files changed, 232 insertions(+), 149 deletions(-) diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d05f5c234..285b946d8 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -146,5 +146,5 @@ def setup_example(name, outdir): LOG.info(f"Copying example '{name}' to {outdir}") write_example(src_path, outdir) - print(f'example: {example}') + print(f"example: {example}") return example diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 9b7233097..741cdb832 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -124,7 +124,7 @@ def __init__(self, data: dict) -> None: def __eq__(self, other: "ContainerFormatConfig"): """ Equality magic method used for testing this class - + :param other: Another ContainerFormatConfig object to check if they're the same """ variables = ("format", "image_type", "image", "url", "config", "config_dir", "pfile", "pass_file", "user_file") @@ -220,7 +220,7 @@ def __init__(self, data: dict) -> None: def __eq__(self, other: "ContainerFormatConfig"): """ Equality magic method used for testing this class - + :param other: Another ContainerFormatConfig object to check if they're the same """ variables = ("command", "run_command", "stop_command", "pull_command") @@ -263,7 +263,7 @@ def __init__(self, data: dict) -> None: def __eq__(self, other: "ProcessConfig"): """ Equality magic method used for testing this class - + :param other: Another ProcessConfig object to check if they're the same """ variables = ("status", "kill") @@ -441,7 +441,7 @@ def set_snapshot(self, seconds: int = None, changes: int = None) -> bool: """ Sets the 'seconds' and/or 'changes' values of the snapshot setting, depending on what the user requests. - + :param seconds: The first value of snapshot to change. If we're leaving it the same this will be None. :param changes: The second value of snapshot to change. If we're leaving it the diff --git a/tests/conftest.py b/tests/conftest.py index ce5cf7571..6ddfb8474 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,9 @@ from tests.utils import create_cert_files, create_pass_file +# pylint: disable=redefined-outer-name + + ####################################### # Loading in Module Specific Fixtures # ####################################### @@ -117,7 +120,7 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name +def merlin_server_dir(temp_output_dir: str) -> str: """ The path to the merlin_server directory that will be created by the `redis_server` fixture. @@ -131,7 +134,7 @@ def merlin_server_dir(temp_output_dir: str) -> str: # pylint: disable=redefined @pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # pylint: disable=redefined-outer-name +def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. @@ -152,7 +155,7 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # @pytest.fixture(scope="session") -def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer-name +def celery_app(redis_server: str) -> Celery: """ Create the celery app to be used throughout our integration tests. @@ -163,7 +166,7 @@ def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer- @pytest.fixture(scope="session") -def sleep_sig(celery_app: Celery) -> Signature: # pylint: disable=redefined-outer-name +def sleep_sig(celery_app: Celery) -> Signature: """ Create a task registered to our celery app and return a signature for it. Once requested by a test, you can set the queue you'd like to send this to @@ -195,7 +198,7 @@ def worker_queue_map() -> Dict[str, str]: @pytest.fixture(scope="class") -def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pylint: disable=redefined-outer-name +def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): """ Launch the workers on the celery app fixture using the worker and queue names defined in the worker_queue_map fixture. @@ -238,7 +241,7 @@ def test_encryption_key() -> bytes: @pytest.fixture(scope="function") -def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name +def config(merlin_server_dir: str, test_encryption_key: bytes): """ DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index ea6a731ff..c88948772 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -2,6 +2,7 @@ Module to define functionality for managing the containerized server used for testing. """ + import os import signal import subprocess diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 01db7bd56..156a374d7 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -1,10 +1,15 @@ """ Fixtures specifically for help testing the modules in the server/ directory. """ + import os +from typing import Dict, Union + import pytest import yaml -from typing import Dict, Union + + +# pylint: disable=redefined-outer-name @pytest.fixture(scope="session") @@ -60,7 +65,9 @@ def server_redis_conf_file(server_testing_dir: str) -> str: appendfilename appendonly.aof # dummy trailing comment - """.strip().replace(" ", "") + """.strip().replace( + " ", "" + ) with open(redis_conf_file, "w") as rcf: rcf.write(file_contents) @@ -96,19 +103,19 @@ def server_users() -> Dict[str, Dict[str, str]]: """ users = { "default": { - "channels": '*', - "commands": '@all', - "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', - "keys": '*', - "status": 'on', + "channels": "*", + "commands": "@all", + "hash_password": "1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a", + "keys": "*", + "status": "on", }, "test_user": { - "channels": '*', - "commands": '@all', - "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', - "keys": '*', - "status": 'on', - } + "channels": "*", + "commands": "@all", + "hash_password": "1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a", + "keys": "*", + "status": "on", + }, } return users @@ -246,7 +253,7 @@ def server_app_yaml_contents( "port": 6379, "server": "127.0.0.1", "username": "default", - } + }, } return contents @@ -268,4 +275,4 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> with open(app_yaml_file, "w") as ayf: yaml.dump(server_app_yaml_contents, ayf) - return app_yaml_file \ No newline at end of file + return app_yaml_file diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index ab0de5d1e..3ae8dcaa7 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -9,6 +9,9 @@ import pytest +# pylint: disable=redefined-outer-name + + @pytest.fixture(scope="class") def status_testing_dir(temp_output_dir: str) -> str: """ @@ -24,7 +27,7 @@ def status_testing_dir(temp_output_dir: str) -> str: @pytest.fixture(scope="class") -def status_empty_file(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_empty_file(status_testing_dir: str) -> str: """ A pytest fixture to create an empty status file. diff --git a/tests/unit/common/test_dumper.py b/tests/unit/common/test_dumper.py index 7c437fde9..c52e9fe90 100644 --- a/tests/unit/common/test_dumper.py +++ b/tests/unit/common/test_dumper.py @@ -1,21 +1,27 @@ """ Tests for the `dumper.py` file. """ + import csv import json import os -import pytest - from datetime import datetime from time import sleep +import pytest + from merlin.common.dumper import dump_handler + NUM_ROWS = 5 -CSV_INFO_TO_DUMP = {"row_num": [i for i in range(1, NUM_ROWS+1)], "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS+1)]} -JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS+1)} +CSV_INFO_TO_DUMP = { + "row_num": [i for i in range(1, NUM_ROWS + 1)], + "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS + 1)], +} +JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS + 1)} DUMP_HANDLER_DIR = "{temp_output_dir}/dump_handler" + def test_dump_handler_invalid_dump_file(): """ This is really testing the initialization of the Dumper class with an invalid file type. @@ -25,6 +31,7 @@ def test_dump_handler_invalid_dump_file(): dump_handler("bad_file.txt", CSV_INFO_TO_DUMP) assert "Invalid file type for bad_file.txt. Supported file types are: ['csv', 'json']" in str(excinfo.value) + def get_output_file(temp_dir: str, file_name: str): """ Helper function to get a full path to the temporary output file. @@ -38,6 +45,7 @@ def get_output_file(temp_dir: str, file_name: str): dump_file = f"{dump_dir}/{file_name}" return dump_file + def run_csv_dump_test(dump_file: str, fmode: str): """ Run the test for csv dump. @@ -52,16 +60,17 @@ def run_csv_dump_test(dump_file: str, fmode: str): reader = csv.reader(df) written_data = list(reader) - expected_rows = NUM_ROWS*2 if fmode == "a" else NUM_ROWS - assert len(written_data) == expected_rows+1 # Adding one because of the header row + expected_rows = NUM_ROWS * 2 if fmode == "a" else NUM_ROWS + assert len(written_data) == expected_rows + 1 # Adding one because of the header row for i, row in enumerate(written_data): assert len(row) == 2 # Check number of columns if i == 0: # Checking the header row assert row[0] == "row_num" assert row[1] == "other_info" else: # Checking the data rows - assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i%NUM_ROWS)-1]) - assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i%NUM_ROWS)-1]) + assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i % NUM_ROWS) - 1]) + assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i % NUM_ROWS) - 1]) + def test_dump_handler_csv_write(temp_output_dir: str): """ @@ -80,6 +89,7 @@ def test_dump_handler_csv_write(temp_output_dir: str): # Assert that everything ran properly run_csv_dump_test(dump_file, "w") + def test_dump_handler_csv_append(temp_output_dir: str): """ This is really testing the write method of the Dumper class with the file write mode set to append. @@ -93,13 +103,14 @@ def test_dump_handler_csv_append(temp_output_dir: str): # Run the first call to create the csv file dump_handler(dump_file, CSV_INFO_TO_DUMP) - + # Run the second call to append to the csv file dump_handler(dump_file, CSV_INFO_TO_DUMP) # Assert that everything ran properly run_csv_dump_test(dump_file, "a") + def test_dump_handler_json_write(temp_output_dir: str): """ This is really testing the write method of the Dumper class. @@ -120,6 +131,7 @@ def test_dump_handler_json_write(temp_output_dir: str): contents = json.load(df) assert contents == JSON_INFO_TO_DUMP + def test_dump_handler_json_append(temp_output_dir: str): """ This is really testing the write method of the Dumper class with the file write mode set to append. @@ -137,7 +149,7 @@ def test_dump_handler_json_append(temp_output_dir: str): dump_handler(dump_file, first_dump) # Sleep so we don't accidentally get the same timestamp - sleep(.5) + sleep(0.5) # Run the second call to append to the file timestamp_2 = str(datetime.now()) @@ -153,4 +165,4 @@ def test_dump_handler_json_append(temp_output_dir: str): assert timestamp_1 in keys assert timestamp_2 in keys assert contents[timestamp_1] == JSON_INFO_TO_DUMP - assert contents[timestamp_2] == JSON_INFO_TO_DUMP \ No newline at end of file + assert contents[timestamp_2] == JSON_INFO_TO_DUMP diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index d797f68c0..3e37cef84 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -1,6 +1,7 @@ """ Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. """ + import os import celery diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index cdb5b2f4f..d857b7ce5 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,6 +1,7 @@ """ Tests for the `sample_index.py` and `sample_index_factory.py` files. """ + import os import pytest diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py index c957ac105..b4cc252d5 100644 --- a/tests/unit/common/test_util_sampling.py +++ b/tests/unit/common/test_util_sampling.py @@ -1,6 +1,7 @@ """ Tests for the `util_sampling.py` file. """ + import numpy as np import pytest diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 8af1dda75..581b19488 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -1,6 +1,7 @@ """ Tests for the `broker.py` file. """ + import os from ssl import CERT_NONE from typing import Any, Dict diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index bd658bc66..64e56b7d9 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -1,6 +1,7 @@ """ Test the functionality of the Config object. """ + from copy import copy, deepcopy from types import SimpleNamespace diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py index aeb1da941..975e19ee4 100644 --- a/tests/unit/config/test_configfile.py +++ b/tests/unit/config/test_configfile.py @@ -1,6 +1,7 @@ """ Tests for the configfile.py module. """ + import getpass import os import shutil diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 314df6ce7..f49e3e897 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -1,6 +1,7 @@ """ Tests for the `results_backend.py` file. """ + import os from ssl import CERT_NONE from typing import Any, Dict diff --git a/tests/unit/server/test_RedisConfig.py b/tests/unit/server/test_RedisConfig.py index 12880d4d6..321d2f38a 100644 --- a/tests/unit/server/test_RedisConfig.py +++ b/tests/unit/server/test_RedisConfig.py @@ -4,13 +4,16 @@ This class is especially large so that's why these tests have been moved to their own file. """ + import filecmp import logging -import pytest from typing import Any +import pytest + from merlin.server.server_util import RedisConfig + class TestRedisConfig: """Tests for the RedisConfig class.""" @@ -73,10 +76,7 @@ def test_write(self, server_redis_conf_file: str, server_testing_dir: str): # Check that the contents of the copied file match the contents of the basic file assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) - @pytest.mark.parametrize("key, val, expected_return", [ - ("port", 1234, True), - ("invalid_key", "dummy_val", False) - ]) + @pytest.mark.parametrize("key, val, expected_return", [("port", 1234, True), ("invalid_key", "dummy_val", False)]) def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, expected_return: bool): """ Test the `set_config_value` method with valid and invalid keys. @@ -95,17 +95,20 @@ def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, else: assert not redis_config.changes_made() - @pytest.mark.parametrize("key, expected_val", [ - ("bind", "127.0.0.1"), - ("port", "6379"), - ("requirepass", "merlin_password"), - ("dir", "./"), - ("save", "300 100"), - ("dbfilename", "dump.rdb"), - ("appendfsync", "everysec"), - ("appendfilename", "appendonly.aof"), - ("invalid_key", None) - ]) + @pytest.mark.parametrize( + "key, expected_val", + [ + ("bind", "127.0.0.1"), + ("port", "6379"), + ("requirepass", "merlin_password"), + ("dir", "./"), + ("save", "300 100"), + ("dbfilename", "dump.rdb"), + ("appendfsync", "everysec"), + ("appendfilename", "appendonly.aof"), + ("invalid_key", None), + ], + ) def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_val: str): """ Test the `get_config_value` method with valid and invalid keys. @@ -117,18 +120,16 @@ def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_ redis_conf = RedisConfig(server_redis_conf_file) assert redis_conf.get_config_value(key) == expected_val - @pytest.mark.parametrize("ip_to_set", [ - "127.0.0.1", # Most common IP - "0.0.0.0", # Edge case (low) - "255.255.255.255", # Edge case (high) - "123.222.199.20", # Random valid IP - ]) - def test_set_ip_address_valid( - self, - caplog: "Fixture", # noqa: F821 - server_redis_conf_file: str, - ip_to_set: str - ): + @pytest.mark.parametrize( + "ip_to_set", + [ + "127.0.0.1", # Most common IP + "0.0.0.0", # Edge case (low) + "255.255.255.255", # Edge case (high) + "123.222.199.20", # Random valid IP + ], + ) + def test_set_ip_address_valid(self, caplog: "Fixture", server_redis_conf_file: str, ip_to_set: str): # noqa: F821 """ Test the `set_ip_address` method with valid ips. These should all return True and set the 'bind' value to whatever `ip_to_set` is. @@ -143,11 +144,14 @@ def test_set_ip_address_valid( assert f"Ipaddress is set to {ip_to_set}" in caplog.text, "Missing expected log message" assert redis_config.get_ip_address() == ip_to_set - @pytest.mark.parametrize("ip_to_set, expected_log", [ - (None, None), # No IP - ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 - ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist - ]) + @pytest.mark.parametrize( + "ip_to_set, expected_log", + [ + (None, None), # No IP + ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 + ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist + ], + ) def test_set_ip_address_invalid( self, caplog: "Fixture", # noqa: F821 @@ -174,12 +178,15 @@ def test_set_ip_address_invalid( if expected_log is not None: assert expected_log in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("port_to_set", [ - 6379, # Most common port - 1, # Edge case (low) - 65535, # Edge case (high) - 12345, # Random valid port - ]) + @pytest.mark.parametrize( + "port_to_set", + [ + 6379, # Most common port + 1, # Edge case (low) + 65535, # Edge case (high) + 12345, # Random valid port + ], + ) def test_set_port_valid( self, caplog: "Fixture", # noqa: F821 @@ -200,12 +207,15 @@ def test_set_port_valid( assert redis_config.get_port() == port_to_set assert f"Port is set to {port_to_set}" in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("port_to_set, expected_log", [ - (None, None), # No port - (0, "Invalid port given."), # Edge case (low) - (65536, "Invalid port given."), # Edge case (high) - ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist - ]) + @pytest.mark.parametrize( + "port_to_set, expected_log", + [ + (None, None), # No port + (0, "Invalid port given."), # Edge case (low) + (65536, "Invalid port given."), # Edge case (high) + ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist + ], + ) def test_set_port_invalid( self, caplog: "Fixture", # noqa: F821 @@ -232,10 +242,13 @@ def test_set_port_invalid( if expected_log is not None: assert expected_log in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("pass_to_set, expected_return", [ - ("valid_password", True), # Valid password - (None, False), # Invalid password - ]) + @pytest.mark.parametrize( + "pass_to_set, expected_return", + [ + ("valid_password", True), # Valid password + (None, False), # Invalid password + ], + ) def test_set_password( self, caplog: "Fixture", # noqa: F821 @@ -289,7 +302,7 @@ def test_set_directory_none(self, server_redis_conf_file: str): """ redis_config = RedisConfig(server_redis_conf_file) assert not redis_config.set_directory(None) - assert redis_config.get_config_value("dir") != None + assert redis_config.get_config_value("dir") is not None def test_set_directory_dir_unset( self, @@ -327,8 +340,10 @@ def test_set_snapshot_valid(self, caplog: "Fixture", server_redis_conf_file: str save_val = redis_conf.get_config_value("save").split() assert save_val[0] == str(snap_sec_to_set) assert save_val[1] == str(snap_changes_to_set) - expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " \ - f"Snapshot threshold is set to {snap_changes_to_set} changes" + expected_log = ( + f"Snapshot wait time is set to {snap_sec_to_set} seconds. " + f"Snapshot threshold is set to {snap_changes_to_set} changes" + ) assert expected_log in caplog.text, "Missing expected log message" def test_set_snapshot_just_seconds(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 @@ -432,11 +447,14 @@ def test_set_snapshot_file_dbfilename_unset(self, caplog: "Fixture", server_redi assert redis_conf.get_config_value("dbfilename") != filename assert "Unable to set snapshot_file name" in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("mode_to_set", [ - "always", - "everysec", - "no", - ]) + @pytest.mark.parametrize( + "mode_to_set", + [ + "always", + "everysec", + "no", + ], + ) def test_set_append_mode_valid( self, caplog: "Fixture", # noqa: F821 diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index c9b59e83e..909cb7cdf 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,12 +1,14 @@ """ Tests for the `server_util.py` module. """ + import filecmp import hashlib import os -import pytest from typing import Dict, Union +import pytest + from merlin.server.server_util import ( AppYaml, ContainerConfig, @@ -16,15 +18,19 @@ RedisUsers, ServerConfig, valid_ipv4, - valid_port + valid_port, ) -@pytest.mark.parametrize("valid_ip", [ - "0.0.0.0", - "127.0.0.1", - "14.105.200.58", - "255.255.255.255", -]) + +@pytest.mark.parametrize( + "valid_ip", + [ + "0.0.0.0", + "127.0.0.1", + "14.105.200.58", + "255.255.255.255", + ], +) def test_valid_ipv4_valid_ip(valid_ip: str): """ Test the `valid_ipv4` function with valid IPs. @@ -35,12 +41,16 @@ def test_valid_ipv4_valid_ip(valid_ip: str): """ assert valid_ipv4(valid_ip) -@pytest.mark.parametrize("invalid_ip", [ - "256.0.0.1", - "-1.0.0.1", - None, - "127.0.01", -]) + +@pytest.mark.parametrize( + "invalid_ip", + [ + "256.0.0.1", + "-1.0.0.1", + None, + "127.0.01", + ], +) def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): """ Test the `valid_ipv4` function with invalid IPs. @@ -52,11 +62,15 @@ def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): """ assert not valid_ipv4(invalid_ip) -@pytest.mark.parametrize("valid_input", [ - 1, - 433, - 65535, -]) + +@pytest.mark.parametrize( + "valid_input", + [ + 1, + 433, + 65535, + ], +) def test_valid_port_valid_input(valid_input: int): """ Test the `valid_port` function with valid port numbers. @@ -68,11 +82,15 @@ def test_valid_port_valid_input(valid_input: int): """ assert valid_port(valid_input) -@pytest.mark.parametrize("invalid_input", [ - -1, - 0, - 65536, -]) + +@pytest.mark.parametrize( + "invalid_input", + [ + -1, + 0, + 65536, + ], +) def test_valid_port_invalid_input(invalid_input: int): """ Test the `valid_port` function with invalid inputs. @@ -121,13 +139,16 @@ def test_init_with_missing_data(self): assert config.pass_file == ContainerConfig.PASSWORD_FILE assert config.user_file == ContainerConfig.USERS_FILE - @pytest.mark.parametrize("attr_name", [ - "image", - "config", - "pfile", - "pass_file", - "user_file", - ]) + @pytest.mark.parametrize( + "attr_name", + [ + "image", + "config", + "pfile", + "pass_file", + "user_file", + ], + ) def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): """ Tests that get_*_path methods construct the correct path. @@ -140,17 +161,20 @@ def test_get_path_methods(self, server_container_config_data: Dict[str, str], at expected_path = os.path.join(server_container_config_data["config_dir"], server_container_config_data[attr_name]) assert get_path_method() == expected_path - @pytest.mark.parametrize("getter_name, expected_attr", [ - ("get_format", "format"), - ("get_image_type", "image_type"), - ("get_image_name", "image"), - ("get_image_url", "url"), - ("get_config_name", "config"), - ("get_config_dir", "config_dir"), - ("get_pfile_name", "pfile"), - ("get_pass_file_name", "pass_file"), - ("get_user_file_name", "user_file"), - ]) + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_format", "format"), + ("get_image_type", "image_type"), + ("get_image_name", "image"), + ("get_image_url", "url"), + ("get_config_name", "config"), + ("get_config_dir", "config_dir"), + ("get_pfile_name", "pfile"), + ("get_pass_file_name", "pass_file"), + ("get_user_file_name", "user_file"), + ], + ) def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ Tests that all getter methods return the correct attribute values. @@ -216,12 +240,15 @@ def test_init_with_missing_data(self): assert config.stop_command == config.STOP_COMMAND assert config.pull_command == config.PULL_COMMAND - @pytest.mark.parametrize("getter_name, expected_attr", [ - ("get_command", "command"), - ("get_run_command", "run_command"), - ("get_stop_command", "stop_command"), - ("get_pull_command", "pull_command"), - ]) + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_command", "command"), + ("get_run_command", "run_command"), + ("get_stop_command", "stop_command"), + ("get_pull_command", "pull_command"), + ], + ) def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ Tests that all getter methods return the correct attribute values. @@ -257,10 +284,13 @@ def test_init_with_missing_data(self): assert config.status == incomplete_data["status"] assert config.kill == config.KILL_COMMAND - @pytest.mark.parametrize("getter_name, expected_attr", [ - ("get_status_command", "status"), - ("get_kill_command", "kill"), - ]) + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_status_command", "status"), + ("get_kill_command", "kill"), + ], + ) def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ Tests that all getter methods return the correct attribute values. @@ -304,7 +334,7 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] class TestRedisUsers: """ Tests for the RedisUsers class. - + TODO add integration test(s) for `apply_to_redis` method of this class. """ @@ -358,7 +388,7 @@ def test_get_user_dict(self): assert val == "on" else: assert val == test_dict[key] - + def test_set_password(self): """Test the `set_password` method of the User class.""" user = RedisUsers.User() diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 5a05e3599..fe7378540 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -1,6 +1,7 @@ """ Tests for the `merlin/examples/generator.py` module. """ + import os from typing import List diff --git a/tests/utils.py b/tests/utils.py index d883b83cd..0b408db54 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,7 @@ """ Utility functions for our test suite. """ + import os from typing import Dict From 2997de6624d1cebeaa98929e3f315948ee0721f9 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 14:09:18 -0700 Subject: [PATCH 039/201] parametrize setup examples tests --- tests/fixtures/examples.py | 20 + tests/unit/test_examples_generator.py | 572 ++++++++++---------------- 2 files changed, 248 insertions(+), 344 deletions(-) create mode 100644 tests/fixtures/examples.py diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py new file mode 100644 index 000000000..16a2f576d --- /dev/null +++ b/tests/fixtures/examples.py @@ -0,0 +1,20 @@ +""" +Fixtures specifically for help testing the modules in the examples/ directory. +""" + +import os +import pytest + +@pytest.fixture(scope="session") +def examples_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to the examples functionality. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for examples tests + """ + testing_dir = f"{temp_output_dir}/examples_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir \ No newline at end of file diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index fe7378540..3f0f2df9d 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -3,6 +3,7 @@ """ import os +import pytest from typing import List from tabulate import tabulate @@ -83,32 +84,27 @@ def test_gather_all_examples(): assert sorted(actual) == sorted(expected) -def test_write_example_dir(temp_output_dir: str): +def test_write_example_dir(examples_testing_dir: str): """ Test the `write_example` function with the src_path as a directory. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) dir_to_copy = f"{EXAMPLES_DIR}/feature_demo/" + dst_dir = f"{examples_testing_dir}/write_example_dir" + write_example(dir_to_copy, dst_dir) + assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(dst_dir)) - write_example(dir_to_copy, generator_dir) - assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(generator_dir)) - -def test_write_example_file(temp_output_dir: str): +def test_write_example_file(examples_testing_dir: str): """ Test the `write_example` function with the src_path as a file. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - - dst_path = f"{generator_dir}/flux_par.yaml" file_to_copy = f"{EXAMPLES_DIR}/flux/flux_par.yaml" - - write_example(file_to_copy, generator_dir) + dst_path = f"{examples_testing_dir}/flux_par.yaml" + write_example(file_to_copy, dst_path) assert os.path.exists(dst_path) @@ -174,6 +170,8 @@ def test_list_examples(): ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" actual = list_examples() + print(f"expected:\n{expected}") + print(f"actual:\n{actual}") assert actual == expected @@ -185,7 +183,7 @@ def test_setup_example_invalid_name(): assert setup_example("invalid_example_name", None) is None -def test_setup_example_no_outdir(temp_output_dir: str): +def test_setup_example_no_outdir(examples_testing_dir: str): """ Test the `setup_example` function with an invalid example name. This should create a directory with the example name (in this case hello) @@ -194,14 +192,12 @@ def test_setup_example_no_outdir(temp_output_dir: str): the `setup_example` function creates the hello/ subdirectory in a directory with the name of this test (setup_no_outdir). - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ cwd = os.getcwd() # Create the temp path to store this setup and move into that directory - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - setup_example_dir = os.path.join(generator_dir, "setup_no_outdir") + setup_example_dir = os.path.join(examples_testing_dir, "setup_no_outdir") create_dir(setup_example_dir) os.chdir(setup_example_dir) @@ -229,37 +225,226 @@ def test_setup_example_no_outdir(temp_output_dir: str): raise AssertionError from exc -def test_setup_example_outdir_exists(temp_output_dir: str): +def test_setup_example_outdir_exists(examples_testing_dir: str): """ Test the `setup_example` function with an output directory that already exists. This should just return None. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - - assert setup_example("hello", generator_dir) is None - - -##################################### -# Tests for setting up each example # -##################################### - - -def run_setup_example(temp_output_dir: str, example_name: str, example_files: List[str], expected_return: str): + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + assert setup_example("hello", examples_testing_dir) is None + + +@pytest.mark.parametrize( + "example_name, example_files, expected_return", + [ + ( + "feature_demo", + [ + ".gitignore", + "feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ], + "feature_demo", + ), + ( + "flux_local", + [ + "flux_local.yaml", + "flux_par_restart.yaml", + "flux_par.yaml", + "paper.yaml", + "requirements.txt", + "scripts/flux_info.py", + "scripts/hello_sleep.c", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/paper_workers.sbatch", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + "scripts/workers.bsub", + ], + "flux", + ), + ( + "lsf_par", + [ + "lsf_par_srun.yaml", + "lsf_par.yaml", + "scripts/hello.c", + "scripts/make_samples.py", + ], + "lsf", + ), + ( + "slurm_par", + [ + "slurm_par.yaml", + "slurm_par_restart.yaml", + "requirements.txt", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + ], + "slurm", + ), + ( + "hello", + [ + "hello_samples.yaml", + "hello.yaml", + "my_hello.yaml", + "requirements.txt", + "make_samples.py", + ], + "hello", + ), + ( + "hpc_demo", + [ + "hpc_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ], + "hpc_demo", + ), + ( + "iterative_demo", + [ + "iterative_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ], + "iterative_demo", + ), + ( + "null_spec", + [ + "null_spec.yaml", + "null_chain.yaml", + ".gitignore", + "Makefile", + "requirements.txt", + "scripts/aggregate_chain_output.sh", + "scripts/aggregate_output.sh", + "scripts/check_completion.sh", + "scripts/kill_all.sh", + "scripts/launch_chain_job.py", + "scripts/launch_jobs.py", + "scripts/make_samples.py", + "scripts/read_output_chain.py", + "scripts/read_output.py", + "scripts/search.sh", + "scripts/submit_chain.sbatch", + "scripts/submit.sbatch", + ], + "null_spec", + ), + ( + "openfoam_wf", + [ + "openfoam_wf.yaml", + "openfoam_wf_docker_template.yaml", + "README.md", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf", + ), + ( + "openfoam_wf_no_docker", + [ + "openfoam_wf_no_docker.yaml", + "openfoam_wf_no_docker_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf_no_docker", + ), + ( + "openfoam_wf_singularity", + [ + "openfoam_wf_singularity.yaml", + "openfoam_wf_singularity_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf_singularity", + ), + ( + "optimization_basic", + [ + "optimization_basic.yaml", + "requirements.txt", + "template_config.py", + "template_optimization.temp", + "scripts/collector.py", + "scripts/optimizer.py", + "scripts/test_functions.py", + "scripts/visualizer.py", + ], + "optimization", + ), + ( + "remote_feature_demo", + [ + ".gitignore", + "remote_feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ], + "remote_feature_demo", + ), + ("restart", ["restart.yaml", "scripts/make_samples.py"], "restart"), + ("restart_delay", ["restart_delay.yaml", "scripts/make_samples.py"], "restart_delay"), + ], +) +def test_setup_example(examples_testing_dir: str, example_name: str, example_files: List[str], expected_return: str): """ - Helper function to run tests for the `setup_example` function. + Run tests for the `setup_example` function. + Each test will consist of: + 1. The name of the example to setup + 2. A list of files that we're expecting to be setup + 3. The expected return value + Each test is a tuple in the parametrize decorator above this test function. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests :param example_name: The name of the example to setup :param example_files: A list of filenames that should be copied by setup_example :param expected_return: The expected return value from `setup_example` """ # Create the temp path to store this setup - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - setup_example_dir = os.path.join(generator_dir, f"setup_{example_name}") + setup_example_dir = os.path.join(examples_testing_dir, f"setup_{example_name}") # Ensure that the example name is returned actual = setup_example(example_name, setup_example_dir) @@ -271,317 +456,16 @@ def run_setup_example(temp_output_dir: str, example_name: str, example_files: Li assert os.path.exists(file) -def test_setup_example_feature_demo(temp_output_dir: str): - """ - Test the `setup_example` function for the feature_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "feature_demo" - example_files = [ - ".gitignore", - "feature_demo.yaml", - "requirements.txt", - "scripts/features.json", - "scripts/hello_world.py", - "scripts/pgen.py", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_flux(temp_output_dir: str): - """ - Test the `setup_example` function for the flux example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_files = [ - "flux_local.yaml", - "flux_par_restart.yaml", - "flux_par.yaml", - "paper.yaml", - "requirements.txt", - "scripts/flux_info.py", - "scripts/hello_sleep.c", - "scripts/hello.c", - "scripts/make_samples.py", - "scripts/paper_workers.sbatch", - "scripts/test_workers.sbatch", - "scripts/workers.sbatch", - "scripts/workers.bsub", - ] - - run_setup_example(temp_output_dir, "flux_local", example_files, "flux") - - -def test_setup_example_lsf(temp_output_dir: str): - """ - Test the `setup_example` function for the lsf example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - - # TODO should there be a workers.bsub for this example? - example_files = [ - "lsf_par_srun.yaml", - "lsf_par.yaml", - "scripts/hello.c", - "scripts/make_samples.py", - ] - - run_setup_example(temp_output_dir, "lsf_par", example_files, "lsf") - - -def test_setup_example_slurm(temp_output_dir: str): - """ - Test the `setup_example` function for the slurm example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_files = [ - "slurm_par.yaml", - "slurm_par_restart.yaml", - "requirements.txt", - "scripts/hello.c", - "scripts/make_samples.py", - "scripts/test_workers.sbatch", - "scripts/workers.sbatch", - ] - - run_setup_example(temp_output_dir, "slurm_par", example_files, "slurm") - - -def test_setup_example_hello(temp_output_dir: str): - """ - Test the `setup_example` function for the hello example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "hello" - example_files = [ - "hello_samples.yaml", - "hello.yaml", - "my_hello.yaml", - "requirements.txt", - "make_samples.py", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_hpc(temp_output_dir: str): - """ - Test the `setup_example` function for the hpc_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "hpc_demo" - example_files = [ - "hpc_demo.yaml", - "cumulative_sample_processor.py", - "faker_sample.py", - "sample_collector.py", - "sample_processor.py", - "requirements.txt", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_iterative(temp_output_dir: str): - """ - Test the `setup_example` function for the iterative_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "iterative_demo" - example_files = [ - "iterative_demo.yaml", - "cumulative_sample_processor.py", - "faker_sample.py", - "sample_collector.py", - "sample_processor.py", - "requirements.txt", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_null(temp_output_dir: str): - """ - Test the `setup_example` function for the null_spec example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "null_spec" - example_files = [ - "null_spec.yaml", - "null_chain.yaml", - ".gitignore", - "Makefile", - "requirements.txt", - "scripts/aggregate_chain_output.sh", - "scripts/aggregate_output.sh", - "scripts/check_completion.sh", - "scripts/kill_all.sh", - "scripts/launch_chain_job.py", - "scripts/launch_jobs.py", - "scripts/make_samples.py", - "scripts/read_output_chain.py", - "scripts/read_output.py", - "scripts/search.sh", - "scripts/submit_chain.sbatch", - "scripts/submit.sbatch", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_openfoam(temp_output_dir: str): - """ - Test the `setup_example` function for the openfoam_wf example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "openfoam_wf" - example_files = [ - "openfoam_wf.yaml", - "openfoam_wf_docker_template.yaml", - "README.md", - "requirements.txt", - "scripts/make_samples.py", - "scripts/blockMesh_template.txt", - "scripts/cavity_setup.sh", - "scripts/combine_outputs.py", - "scripts/learn.py", - "scripts/mesh_param_script.py", - "scripts/run_openfoam", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_openfoam_no_docker(temp_output_dir: str): - """ - Test the `setup_example` function for the openfoam_wf_no_docker example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "openfoam_wf_no_docker" - example_files = [ - "openfoam_wf_no_docker.yaml", - "openfoam_wf_no_docker_template.yaml", - "requirements.txt", - "scripts/make_samples.py", - "scripts/blockMesh_template.txt", - "scripts/cavity_setup.sh", - "scripts/combine_outputs.py", - "scripts/learn.py", - "scripts/mesh_param_script.py", - "scripts/run_openfoam", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_openfoam_singularity(temp_output_dir: str): - """ - Test the `setup_example` function for the openfoam_wf_singularity example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "openfoam_wf_singularity" - example_files = [ - "openfoam_wf_singularity.yaml", - "openfoam_wf_singularity_template.yaml", - "requirements.txt", - "scripts/make_samples.py", - "scripts/blockMesh_template.txt", - "scripts/cavity_setup.sh", - "scripts/combine_outputs.py", - "scripts/learn.py", - "scripts/mesh_param_script.py", - "scripts/run_openfoam", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_optimization(temp_output_dir: str): - """ - Test the `setup_example` function for the optimization example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_files = [ - "optimization_basic.yaml", - "requirements.txt", - "template_config.py", - "template_optimization.temp", - "scripts/collector.py", - "scripts/optimizer.py", - "scripts/test_functions.py", - "scripts/visualizer.py", - ] - - run_setup_example(temp_output_dir, "optimization_basic", example_files, "optimization") - - -def test_setup_example_remote_feature_demo(temp_output_dir: str): - """ - Test the `setup_example` function for the remote_feature_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "remote_feature_demo" - example_files = [ - ".gitignore", - "remote_feature_demo.yaml", - "requirements.txt", - "scripts/features.json", - "scripts/hello_world.py", - "scripts/pgen.py", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_restart(temp_output_dir: str): - """ - Test the `setup_example` function for the restart example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "restart" - example_files = ["restart.yaml", "scripts/make_samples.py"] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_restart_delay(temp_output_dir: str): - """ - Test the `setup_example` function for the restart_delay example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "restart_delay" - example_files = ["restart_delay.yaml", "scripts/make_samples.py"] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_simple_chain(temp_output_dir: str): +def test_setup_example_simple_chain(examples_testing_dir: str): """ Test the `setup_example` function for the simple_chain example. + This example just writes a single file so we can't run it in the `test_setup_example` test. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ # Create the temp path to store this setup - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - output_file = os.path.join(generator_dir, "simple_chain.yaml") + output_file = os.path.join(examples_testing_dir, "simple_chain.yaml") # Ensure that the example name is returned actual = setup_example("simple_chain", output_file) From 2f24577cdc53f40a270ab40b098f80e85cabb1fb Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 14:24:51 -0700 Subject: [PATCH 040/201] sort example output --- merlin/examples/generator.py | 2 +- tests/unit/test_examples_generator.py | 68 +++++++++++++-------------- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 285b946d8..63da74d78 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -60,7 +60,7 @@ def gather_example_dirs(): """Get all the example directories""" result = {} - for directory in os.listdir(EXAMPLES_DIR): + for directory in sorted(os.listdir(EXAMPLES_DIR)): result[directory] = directory return result diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 3f0f2df9d..7548c8a49 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -112,44 +112,39 @@ def test_list_examples(): """Test the `list_examples` function to see if it gives us all of the examples that we want.""" expected_headers = ["name", "description"] expected_rows = [ - [ - "openfoam_wf_no_docker", - "A parameter study that includes initializing, running,\n" - "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" - "without using docker.", - ], - [ - "optimization_basic", - "Design Optimization Template\n" - "To use,\n" - "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" - "2. Run the template_config file in current directory using `python template_config.py`\n" - "3. Merlin run as usual (merlin run optimization.yaml)\n" - "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" - "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", - ], ["feature_demo", "Run 10 hello worlds."], ["flux_local", "Run a scan through Merlin/Maestro"], ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], ["flux_par_restart", "A simple ensemble of parallel MPI jobs run by flux."], ["paper_flux", "Use flux to run single core MPI jobs and record timings."], - ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], - ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], - ["restart", "A simple ensemble of with restarts."], - ["restart_delay", "A simple ensemble of with restart delay times."], - ["simple_chain", "test to see that chains are not run in parallel"], - ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], - ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], - ["remote_feature_demo", "Run 10 hello worlds."], ["hello", "a very simple merlin workflow"], ["hello_samples", "a very simple merlin workflow, with samples"], ["hpc_demo", "Demo running a workflow on HPC machines"], + ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], + ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], + [ + "null_chain", + "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" + "May be used to measure overhead in merlin.\n" + "Iterates thru a chain of workflows.", + ], + [ + "null_spec", + "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + ], [ "openfoam_wf", "A parameter study that includes initializing, running,\n" "post-processing, collecting, learning and visualizing OpenFOAM runs\n" "using docker.", ], + [ + "openfoam_wf_no_docker", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" + "without using docker.", + ], [ "openfoam_wf_singularity", "A parameter study that includes initializing, running,\n" @@ -157,21 +152,24 @@ def test_list_examples(): "using singularity.", ], [ - "null_chain", - "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" - "May be used to measure overhead in merlin.\n" - "Iterates thru a chain of workflows.", - ], - [ - "null_spec", - "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + "optimization_basic", + "Design Optimization Template\n" + "To use,\n" + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" + "2. Run the template_config file in current directory using `python template_config.py`\n" + "3. Merlin run as usual (merlin run optimization.yaml)\n" + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", ], - ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ["remote_feature_demo", "Run 10 hello worlds."], + ["restart", "A simple ensemble of with restarts."], + ["restart_delay", "A simple ensemble of with restart delay times."], + ["simple_chain", "test to see that chains are not run in parallel"], + ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" actual = list_examples() - print(f"expected:\n{expected}") - print(f"actual:\n{actual}") assert actual == expected From 5e0a5f78903a371e5dcf93175a3a6ccd296bf016 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 15:27:09 -0700 Subject: [PATCH 041/201] ensure directory is changed back on no outdir test --- tests/unit/test_examples_generator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 7548c8a49..25432ffca 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -170,6 +170,8 @@ def test_list_examples(): ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" actual = list_examples() + print(f"actual:\n{actual}") + print(f"expected:\n{expected}") assert actual == expected @@ -221,6 +223,8 @@ def test_setup_example_no_outdir(examples_testing_dir: str): except AssertionError as exc: os.chdir(cwd) raise AssertionError from exc + finally: + os.chdir(cwd) def test_setup_example_outdir_exists(examples_testing_dir: str): From d8fa77c268c25a240900ab020063d6d05803ce3e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 16:54:24 -0700 Subject: [PATCH 042/201] sort the specs in examples output --- merlin/examples/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 63da74d78..c45cfb9ce 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -90,7 +90,7 @@ def list_examples(): for example_dir in gather_example_dirs(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") - for spec in specs: + for spec in sorted(specs): if "template" in spec: continue with open(spec) as f: # pylint: disable=C0103 From 8421d74ff33c094fdee94152ae5a7f4de6539304 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 16:56:45 -0700 Subject: [PATCH 043/201] fix lint issues --- tests/fixtures/examples.py | 4 +++- tests/unit/test_examples_generator.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py index 16a2f576d..7c4626e3e 100644 --- a/tests/fixtures/examples.py +++ b/tests/fixtures/examples.py @@ -3,8 +3,10 @@ """ import os + import pytest + @pytest.fixture(scope="session") def examples_testing_dir(temp_output_dir: str) -> str: """ @@ -17,4 +19,4 @@ def examples_testing_dir(temp_output_dir: str) -> str: if not os.path.exists(testing_dir): os.mkdir(testing_dir) - return testing_dir \ No newline at end of file + return testing_dir diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 25432ffca..7d4d879fb 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -3,9 +3,9 @@ """ import os -import pytest from typing import List +import pytest from tabulate import tabulate from merlin.examples.generator import ( From 0543ae48623128efaba0de00f83ed90a27c73b9b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 10 Jun 2024 09:47:35 -0700 Subject: [PATCH 044/201] start writing tests for server config --- merlin/examples/generator.py | 1 - merlin/server/server_config.py | 4 +-- tests/unit/server/test_server_config.py | 43 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 tests/unit/server/test_server_config.py diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index c45cfb9ce..725448bec 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -146,5 +146,4 @@ def setup_example(name, outdir): LOG.info(f"Copying example '{name}' to {outdir}") write_example(src_path, outdir) - print(f"example: {example}") return example diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index f4d5d5174..b0b91f892 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -92,8 +92,8 @@ def generate_password(length, pass_command: str = None) -> str: :return:: string value with given length """ if pass_command: - process = subprocess.run(pass_command.split(), shell=True, stdout=subprocess.PIPE) - return process.stdout + process = subprocess.run(pass_command, shell=True, capture_output=True, text=True) + return process.stdout.strip() characters = list(string.ascii_letters + string.digits + "!@#$%^&*()") diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py new file mode 100644 index 000000000..058e77fcf --- /dev/null +++ b/tests/unit/server/test_server_config.py @@ -0,0 +1,43 @@ +""" +Tests for the `server_config.py` module. +""" + +import string + +from merlin.server.server_config import ( + PASSWORD_LENGTH, + check_process_file_format, + config_merlin_server, + create_server_config, + dump_process_file, + generate_password, + get_server_status, + parse_redis_output, + pull_process_file, + pull_server_config, + pull_server_image, +) + + +def test_generate_password_no_pass_command(): + """ + Test the `generate_password` function with no password command. + This should generate a password of 256 (PASSWORD_LENGTH) random ASCII characters. + """ + generated_password = generate_password(PASSWORD_LENGTH) + assert len(generated_password) == PASSWORD_LENGTH + valid_ascii_chars = string.ascii_letters + string.digits + "!@#$%^&*()" + for ch in generated_password: + assert ch in valid_ascii_chars + + +def test_generate_password_with_pass_command(): + """ + Test the `generate_password` function with no password command. + This should generate a password of 256 (PASSWORD_LENGTH) random ASCII characters. + """ + test_pass = "test-password" + generated_password = generate_password(0, pass_command=f"echo {test_pass}") + assert generated_password == test_pass + + From 3b0ccde1175390fecd9c4a144a24314d8e692727 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 11 Dec 2023 15:30:05 -0800 Subject: [PATCH 045/201] add pytest coverage library and add sample_index coverage --- .gitignore | 3 +- merlin/common/sample_index.py | 2 +- requirements/dev.txt | 1 + tests/unit/common/test_sample_index.py | 672 ++++++++++++++++++------- 4 files changed, 508 insertions(+), 170 deletions(-) diff --git a/.gitignore b/.gitignore index c22521934..cec577a85 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,9 @@ flux.out slurm*.out docs/build/ -# Tox files +# Test files .tox/* +.coverage # Jupyter jupyter/.ipynb_checkpoints diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index c7808bd3b..caea6ad6e 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -225,8 +225,8 @@ def __setitem__(self, full_address, sub_tree): # Replace if we already have something at this address. if delete_me is not None: - self.children.__delitem__(full_address) SampleIndex.check_valid_addresses_for_insertion(full_address, sub_tree) + self.children.__delitem__(full_address) self.children[full_address] = sub_tree return raise KeyError diff --git a/requirements/dev.txt b/requirements/dev.txt index 3695c6164..ab5962119 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,6 +5,7 @@ dep-license flake8 isort pytest +pytest-cov pylint twine sphinx>=2.0.0 diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index c693827f0..1237c52a1 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,178 +1,514 @@ import os +import pytest import shutil from contextlib import suppress +from merlin.common.sample_index import SampleIndex, uniform_directories, new_dir from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy -TEST_DIR = "UNIT_TEST_SPACE" - - -def clear_test_tree(): - with suppress(FileNotFoundError): - shutil.rmtree(TEST_DIR) - - -def clear(func): - def wrapper(): - clear_test_tree() - func() - clear_test_tree() - - return wrapper - - -@clear -def test_index_file_writing(): - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) - indx.write_directories() - indx.write_multiple_sample_index_files() - indx2 = read_hierarchy(TEST_DIR) - assert indx2.get_path_to_sample(123000123) == indx.get_path_to_sample(123000123) - - -def test_bundle_retrieval(): - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) - expected = f"{TEST_DIR}/0/0/0/samples0-10000.ext" - result = indx.get_path_to_sample(123) - assert expected == result - - expected = f"{TEST_DIR}/0/0/0/samples10000-20000.ext" - result = indx.get_path_to_sample(10000) - assert expected == result - - expected = f"{TEST_DIR}/1/2/3/samples123000000-123010000.ext" - result = indx.get_path_to_sample(123000123) - assert expected == result - - -def test_start_sample_id(): - expected = """: DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10 - 0: BUNDLE 0 MIN 203 MAX 213 - 1: BUNDLE 1 MIN 213 MAX 223 - 2: BUNDLE 2 MIN 223 MAX 233 - 3: BUNDLE 3 MIN 233 MAX 243 - 4: BUNDLE 4 MIN 243 MAX 253 - 5: BUNDLE 5 MIN 253 MAX 263 - 6: BUNDLE 6 MIN 263 MAX 273 - 7: BUNDLE 7 MIN 273 MAX 283 - 8: BUNDLE 8 MIN 283 MAX 293 - 9: BUNDLE 9 MIN 293 MAX 303 -""" - idx203 = create_hierarchy(100, 10, start_sample_id=203) - assert expected == str(idx203) - - -@clear -def test_directory_writing(): - path = os.path.join(TEST_DIR) - indx = create_hierarchy(2, 1, [1], root=path) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE 0 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: BUNDLE 1 MIN 1 MAX 2 -""" - assert expected == str(indx) - indx.write_directories() - assert os.path.isdir(f"{TEST_DIR}/0") - assert os.path.isdir(f"{TEST_DIR}/1") - indx.write_multiple_sample_index_files() - - clear_test_tree() - - path = os.path.join(TEST_DIR) - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000], root=path) - indx.write_directories() - path = indx.get_path_to_sample(123000123) - assert os.path.exists(os.path.dirname(path)) - assert path != TEST_DIR - path = indx.get_path_to_sample(10000000000) - assert path == TEST_DIR - - clear_test_tree() - - path = os.path.join(TEST_DIR) - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=path) - indx.write_directories() - - -def test_directory_path(): - indx = create_hierarchy(20, 1, [20, 5, 1], root="") - leaves = indx.make_directory_string() - expected_leaves = "0/0/0 0/0/1 0/0/2 0/0/3 0/0/4 0/1/0 0/1/1 0/1/2 0/1/3 0/1/4 0/2/0 0/2/1 0/2/2 0/2/3 0/2/4 0/3/0 0/3/1 0/3/2 0/3/3 0/3/4" - assert leaves == expected_leaves - all_dirs = indx.make_directory_string(just_leaf_directories=False) - expected_all_dirs = " 0 0/0 0/0/0 0/0/1 0/0/2 0/0/3 0/0/4 0/1 0/1/0 0/1/1 0/1/2 0/1/3 0/1/4 0/2 0/2/0 0/2/1 0/2/2 0/2/3 0/2/4 0/3 0/3/0 0/3/1 0/3/2 0/3/3 0/3/4" - assert all_dirs == expected_all_dirs - - -@clear -def test_subhierarchy_insertion(): - indx = create_hierarchy(2, 1, [1], root=TEST_DIR) - print("Writing directories") - indx.write_directories() - indx.write_multiple_sample_index_files() - print("reading heirarchy") - top = read_hierarchy(os.path.abspath(TEST_DIR)) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE -1 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: BUNDLE -1 MIN 1 MAX 2 -""" - assert str(top) == expected - print("creating sub_heirarchy") - sub_h = create_hierarchy(100, 10, address="1.0") - print("inserting sub_heirarchy") - top["1.0"] = sub_h - print(str(indx)) - print("after insertion") - print(str(top)) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE -1 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10 - 1.0.0: BUNDLE 0 MIN 0 MAX 10 - 1.0.1: BUNDLE 1 MIN 10 MAX 20 - 1.0.2: BUNDLE 2 MIN 20 MAX 30 - 1.0.3: BUNDLE 3 MIN 30 MAX 40 - 1.0.4: BUNDLE 4 MIN 40 MAX 50 - 1.0.5: BUNDLE 5 MIN 50 MAX 60 - 1.0.6: BUNDLE 6 MIN 60 MAX 70 - 1.0.7: BUNDLE 7 MIN 70 MAX 80 - 1.0.8: BUNDLE 8 MIN 80 MAX 90 - 1.0.9: BUNDLE 9 MIN 90 MAX 100 -""" - assert str(top) == expected - - -def test_sample_index(): - """Run through some basic testing of the SampleIndex class.""" +def test_uniform_directories(): + """ + Test the `uniform_directories` function with different inputs. + """ + # Create the tests and the expected outputs tests = [ - (10, 1, []), - (10, 3, []), - (11, 2, [5]), - (10, 3, [3]), - (10, 3, [1]), - (10, 1, [3]), - (10, 3, [1, 3]), - (10, 1, [2]), - (1000, 100, [500]), - (1000, 50, [500, 100]), - (1000000000, 100000132, []), + # SMALL SAMPLE SIZE + (10, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (10, 1, 2), + (10, 2, 100), + (10, 2, 2), + # MEDIUM SAMPLE SIZE + (10000, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (10000, 1, 5), + (10000, 5, 100), + (10000, 5, 10), + # LARGE SAMPLE SIZE + (1000000000, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (1000000000, 1, 5), + (1000000000, 5, 100), + (1000000000, 5, 10), ] + expected_outputs = [ + # SMALL SAMPLE SIZE + [1], + [8, 4, 2, 1], + [2], + [8, 4, 2], + # MEDIUM SAMPLE SIZE + [100, 1], + [3125, 625, 125, 25, 5, 1], + [500, 5], + [5000, 500, 50, 5], + # LARGE SAMPLE SIZE + [100000000, 1000000, 10000, 100, 1], + [244140625, 48828125, 9765625, 1953125, 390625, 78125, 15625, 3125, 625, 125, 25, 5, 1], + [500000000, 5000000, 50000, 500, 5], + [500000000, 50000000, 5000000, 500000, 50000, 5000, 500, 50, 5], + ] + + # Run the tests and compare outputs + for i, test in enumerate(tests): + actual = uniform_directories(num_samples=test[0], bundle_size=test[1], level_max_dirs=test[2]) + assert actual == expected_outputs[i] + + +def test_new_dir(temp_output_dir: str): + """ + Test the `new_dir` function. This will test a valid path and also raising an OSError during + creation. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Test basic functionality + test_path = f"{os.getcwd()}/test_new_dir" + new_dir(test_path) + assert os.path.exists(test_path) + + # Test OSError functionality + new_dir(test_path) + + + +class TestSampleIndex: + """ + These tests focus on testing the SampleIndex class used for creating the + sample hierarchy. + + NOTE to see output of creating any hierarchy, change `write_all_hierarchies` to True. + The results of each hierarchy will be written to: + /tmp/`whoami`/pytest/pytest-of-`whoami`/pytest-current/integration_outfiles_current/test_sample_index/ + """ + + write_all_hierarchies = False + + def get_working_dir(self, test_workspace: str): + """ + This method is called for every test to get a unique workspace in the temporary + directory for the test output. + + :param test_workspace: The unique name for this workspace + (all tests use their unique test name for this value usually) + """ + return f"{os.getcwd()}/test_sample_index/{test_workspace}" + + def write_hierarchy_for_debug(self, indx: SampleIndex): + """ + This method is for debugging purposes. It will cause all tests that don't write + hierarchies to write them so the output can be investigated. + + :param indx: The `SampleIndex` object to write the hierarchy for + """ + if self.write_all_hierarchies: + indx.write_directories() + indx.write_multiple_sample_index_files() + + def test_invalid_children(self): + """ + This will test that an invalid type for the `children` argument will raise + an error. + """ + tests = [ + ["a", "b", "c"], + True, + "a b c", + ] + for test in tests: + with pytest.raises(TypeError): + SampleIndex(0, 10, test, "name") + + def test_is_parent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_parent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_parent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if parent of leaf is recognized + assert indx.is_parent_of_leaf is False + assert indx.children["0"].is_parent_of_leaf is True + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"] + assert leaf_node.is_parent_of_leaf is False + + def test_is_grandparent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_grandparent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if grandparent of leaf is recognized + assert indx.is_grandparent_of_leaf is True + assert indx.children["0"].is_grandparent_of_leaf is False + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"] + assert leaf_node.is_grandparent_of_leaf is False + + def test_is_great_grandparent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_great_grandparent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_great_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [5, 1], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if great grandparent of leaf is recognized + assert indx.is_great_grandparent_of_leaf is True + assert indx.children["0"].is_great_grandparent_of_leaf is False + assert indx.children["0"].children["0.0"].is_great_grandparent_of_leaf is False + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"].children["0.0.0"] + assert leaf_node.is_great_grandparent_of_leaf is False + + def test_traverse_bundle(self, temp_output_dir: str): + """ + Test the `traverse_bundle` method to make sure it's just returning leaves. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Ensure all nodes in the traversal are leaves + for _, node in indx.traverse_bundles(): + assert node.is_leaf + + def test_getitem(self, temp_output_dir: str): + """ + Test the `__getitem__` magic method. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test getting that requesting the root returns itself + assert indx[""] == indx + + # Test a valid address + assert indx["0"] == indx.children["0"] + + # Test an invalid address + with pytest.raises(KeyError): + indx["10"] + + def test_setitem(self, temp_output_dir: str): + """ + Test the `__setitem__` magic method. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + invalid_indx = SampleIndex(1, 3, {}, "invalid_indx") + + # Ensure that trying to change the root raises an error + with pytest.raises(KeyError): + indx[""] = invalid_indx + + # Ensure we can't just add a new subtree to a level + with pytest.raises(KeyError): + indx["10"] = invalid_indx + + # Test that invalid subtrees are caught + with pytest.raises(TypeError): + indx["0"] = invalid_indx + + # Test a valid set operation + dummy_indx = SampleIndex(0, 1, {}, "dummy_indx", leafid=0, address="0.0") + indx["0"]["0.0"] = dummy_indx + + + def test_index_file_writing(self, temp_output_dir: str): + """ + Test the functionality of writing multiple index files. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_index_file_writing") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + indx.write_directories() + indx.write_multiple_sample_index_files() + indx2 = read_hierarchy(working_dir) + assert indx2.get_path_to_sample(123000123) == indx.get_path_to_sample(123000123) + + def test_directory_writing_small(self, temp_output_dir: str): + """ + Test that writing a small directory functions properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the directory and ensure it has the correct format + working_dir = self.get_working_dir("test_directory_writing_small/") + indx = create_hierarchy(2, 1, [1], root=working_dir) + expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ + " 0.0: BUNDLE 0 MIN 0 MAX 1\n" \ + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ + " 1.0: BUNDLE 1 MIN 1 MAX 2\n" \ + + assert expected == str(indx) + + # Write the directories and ensure the paths are actually written + indx.write_directories() + assert os.path.isdir(f"{working_dir}/0") + assert os.path.isdir(f"{working_dir}/1") + indx.write_multiple_sample_index_files() + + def test_directory_writing_large(self, temp_output_dir: str): + """ + Test that writing a large directory functions properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_directory_writing_large") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + indx.write_directories() + path = indx.get_path_to_sample(123000123) + assert os.path.exists(os.path.dirname(path)) + assert path != working_dir + path = indx.get_path_to_sample(10000000000) + assert path == working_dir + + def test_bundle_retrieval(self, temp_output_dir: str): + """ + Test the functionality to get a bundle of samples when providing a sample id to find. + This will test a large sample hierarchy to ensure this scales properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the hierarchy + working_dir = self.get_working_dir("test_bundle_retrieval") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test for a small sample id + expected = f"{working_dir}/0/0/0/samples0-10000.ext" + result = indx.get_path_to_sample(123) + assert expected == result + + # Test for a mid size sample id + expected = f"{working_dir}/0/0/0/samples10000-20000.ext" + result = indx.get_path_to_sample(10000) + assert expected == result + + # Test for a large sample id + expected = f"{working_dir}/1/2/3/samples123000000-123010000.ext" + result = indx.get_path_to_sample(123000123) + assert expected == result + + def test_start_sample_id(self, temp_output_dir: str): + """ + Test creating a hierarchy using a starting sample id. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_start_sample_id") + expected = ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" \ + " 0: BUNDLE 0 MIN 203 MAX 213\n" \ + " 1: BUNDLE 1 MIN 213 MAX 223\n" \ + " 2: BUNDLE 2 MIN 223 MAX 233\n" \ + " 3: BUNDLE 3 MIN 233 MAX 243\n" \ + " 4: BUNDLE 4 MIN 243 MAX 253\n" \ + " 5: BUNDLE 5 MIN 253 MAX 263\n" \ + " 6: BUNDLE 6 MIN 263 MAX 273\n" \ + " 7: BUNDLE 7 MIN 273 MAX 283\n" \ + " 8: BUNDLE 8 MIN 283 MAX 293\n" \ + " 9: BUNDLE 9 MIN 293 MAX 303\n" \ + + idx203 = create_hierarchy(100, 10, start_sample_id=203, root=working_dir) + self.write_hierarchy_for_debug(idx203) + + assert expected == str(idx203) + + def test_make_directory_string(self, temp_output_dir: str): + """ + Test the `make_directory_string` method of `SampleIndex`. This will check + both the normal functionality where we just request paths to the leaves and + also the inverse functionality where we request all paths that are not leaves. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Creating the hierarchy + working_dir = self.get_working_dir("test_make_directory_string") + indx = create_hierarchy(20, 1, [20, 5, 1], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Testing normal functionality (just leaf directories) + leaves = indx.make_directory_string() + expected_leaves_list = [ + f"{working_dir}/0/0/0", + f"{working_dir}/0/0/1", + f"{working_dir}/0/0/2", + f"{working_dir}/0/0/3", + f"{working_dir}/0/0/4", + f"{working_dir}/0/1/0", + f"{working_dir}/0/1/1", + f"{working_dir}/0/1/2", + f"{working_dir}/0/1/3", + f"{working_dir}/0/1/4", + f"{working_dir}/0/2/0", + f"{working_dir}/0/2/1", + f"{working_dir}/0/2/2", + f"{working_dir}/0/2/3", + f"{working_dir}/0/2/4", + f"{working_dir}/0/3/0", + f"{working_dir}/0/3/1", + f"{working_dir}/0/3/2", + f"{working_dir}/0/3/3", + f"{working_dir}/0/3/4", + ] + expected_leaves = " ".join(expected_leaves_list) + assert leaves == expected_leaves + + # Testing no leaf functionality + all_dirs = indx.make_directory_string(just_leaf_directories=False) + expected_all_dirs_list = [ + working_dir, + f"{working_dir}/0", + f"{working_dir}/0/0", + f"{working_dir}/0/0/0", + f"{working_dir}/0/0/1", + f"{working_dir}/0/0/2", + f"{working_dir}/0/0/3", + f"{working_dir}/0/0/4", + f"{working_dir}/0/1", + f"{working_dir}/0/1/0", + f"{working_dir}/0/1/1", + f"{working_dir}/0/1/2", + f"{working_dir}/0/1/3", + f"{working_dir}/0/1/4", + f"{working_dir}/0/2", + f"{working_dir}/0/2/0", + f"{working_dir}/0/2/1", + f"{working_dir}/0/2/2", + f"{working_dir}/0/2/3", + f"{working_dir}/0/2/4", + f"{working_dir}/0/3", + f"{working_dir}/0/3/0", + f"{working_dir}/0/3/1", + f"{working_dir}/0/3/2", + f"{working_dir}/0/3/3", + f"{working_dir}/0/3/4" + ] + expected_all_dirs = " ".join(expected_all_dirs_list) + assert all_dirs == expected_all_dirs + + def test_subhierarchy_insertion(self, temp_output_dir: str): + """ + Test that a subhierarchy can be inserted into our `SampleIndex` properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the hierarchy and read it + working_dir = self.get_working_dir("test_subhierarchy_insertion") + indx = create_hierarchy(2, 1, [1], root=working_dir) + indx.write_directories() + indx.write_multiple_sample_index_files() + top = read_hierarchy(os.path.abspath(working_dir)) + + # Compare results + expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ + " 1.0: BUNDLE -1 MIN 1 MAX 2\n" \ + + assert str(top) == expected + + # Create and insert the sub hierarchy + sub_h = create_hierarchy(100, 10, address="1.0") + top["1.0"] = sub_h + + # Compare results + expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ + " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" \ + " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" \ + " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" \ + " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" \ + " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" \ + " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" \ + " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" \ + " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" \ + " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" \ + " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" \ + " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" \ + + assert str(top) == expected + + def test_sample_index_creation_and_insertion(self, temp_output_dir: str): + """ + Run through some basic testing of the SampleIndex class. This will try + creating hierarchies of different sizes and inserting subhierarchies of + different sizes as well. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Define the tests for hierarchies of varying sizes + tests = [ + (10, 1, []), + (10, 3, []), + (11, 2, [5]), + (10, 3, [3]), + (10, 3, [1]), + (10, 1, [3]), + (10, 3, [1, 3]), + (10, 1, [2]), + (1000, 100, [500]), + (1000, 50, [500, 100]), + (1000000000, 100000132, []), + ] + + # Run all the tests we defined above + for i, args in enumerate(tests): + working_dir = self.get_working_dir(f"test_sample_index_creation_and_insertion/{i}") + + # Put at root address of "0" to guarantee insertion at "0.1" later is valid + idx = create_hierarchy(args[0], args[1], args[2], address="0", root=working_dir) + self.write_hierarchy_for_debug(idx) - for args in tests: - print(f"############ TEST {args[0]} {args[1]} {args[2]} ###########") - # put at root address of "0" to guarantee insertion at "0.1" later is valid - idx = create_hierarchy(args[0], args[1], args[2], address="0") - print(str(idx)) - try: - idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") - print("successful set") - print(str(idx)) - except KeyError as error: - print(error) - assert False + # Inserting hierarchy at 0.1 + try: + idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") + except KeyError as error: + assert False From 4ad3729525e93c8dbffd00a89ff2bfe79f7e5df0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 09:31:22 -0800 Subject: [PATCH 046/201] run fix style and add module header --- tests/unit/common/test_sample_index.py | 100 +++++++++++++------------ 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index 1237c52a1..296783273 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,9 +1,11 @@ +""" +Tests for the `sample_index.py` and `sample_index_factory.py` files. +""" import os + import pytest -import shutil -from contextlib import suppress -from merlin.common.sample_index import SampleIndex, uniform_directories, new_dir +from merlin.common.sample_index import SampleIndex, new_dir, uniform_directories from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy @@ -70,7 +72,6 @@ def test_new_dir(temp_output_dir: str): new_dir(test_path) - class TestSampleIndex: """ These tests focus on testing the SampleIndex class used for creating the @@ -228,7 +229,7 @@ def test_setitem(self, temp_output_dir: str): working_dir = self.get_working_dir("test_is_grandparent_of_leaf") indx = create_hierarchy(10, 1, [2], root=working_dir) self.write_hierarchy_for_debug(indx) - + invalid_indx = SampleIndex(1, 3, {}, "invalid_indx") # Ensure that trying to change the root raises an error @@ -247,7 +248,6 @@ def test_setitem(self, temp_output_dir: str): dummy_indx = SampleIndex(0, 1, {}, "dummy_indx", leafid=0, address="0.0") indx["0"]["0.0"] = dummy_indx - def test_index_file_writing(self, temp_output_dir: str): """ Test the functionality of writing multiple index files. @@ -272,12 +272,13 @@ def test_directory_writing_small(self, temp_output_dir: str): # Create the directory and ensure it has the correct format working_dir = self.get_working_dir("test_directory_writing_small/") indx = create_hierarchy(2, 1, [1], root=working_dir) - expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ - " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ - " 0.0: BUNDLE 0 MIN 0 MAX 1\n" \ - " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ - " 1.0: BUNDLE 1 MIN 1 MAX 2\n" \ - + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE 0 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: BUNDLE 1 MIN 1 MAX 2\n" + ) assert expected == str(indx) # Write the directories and ensure the paths are actually written @@ -338,18 +339,19 @@ def test_start_sample_id(self, temp_output_dir: str): temporary output path for our tests """ working_dir = self.get_working_dir("test_start_sample_id") - expected = ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" \ - " 0: BUNDLE 0 MIN 203 MAX 213\n" \ - " 1: BUNDLE 1 MIN 213 MAX 223\n" \ - " 2: BUNDLE 2 MIN 223 MAX 233\n" \ - " 3: BUNDLE 3 MIN 233 MAX 243\n" \ - " 4: BUNDLE 4 MIN 243 MAX 253\n" \ - " 5: BUNDLE 5 MIN 253 MAX 263\n" \ - " 6: BUNDLE 6 MIN 263 MAX 273\n" \ - " 7: BUNDLE 7 MIN 273 MAX 283\n" \ - " 8: BUNDLE 8 MIN 283 MAX 293\n" \ - " 9: BUNDLE 9 MIN 293 MAX 303\n" \ - + expected = ( + ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" + " 0: BUNDLE 0 MIN 203 MAX 213\n" + " 1: BUNDLE 1 MIN 213 MAX 223\n" + " 2: BUNDLE 2 MIN 223 MAX 233\n" + " 3: BUNDLE 3 MIN 233 MAX 243\n" + " 4: BUNDLE 4 MIN 243 MAX 253\n" + " 5: BUNDLE 5 MIN 253 MAX 263\n" + " 6: BUNDLE 6 MIN 263 MAX 273\n" + " 7: BUNDLE 7 MIN 273 MAX 283\n" + " 8: BUNDLE 8 MIN 283 MAX 293\n" + " 9: BUNDLE 9 MIN 293 MAX 303\n" + ) idx203 = create_hierarchy(100, 10, start_sample_id=203, root=working_dir) self.write_hierarchy_for_debug(idx203) @@ -424,7 +426,7 @@ def test_make_directory_string(self, temp_output_dir: str): f"{working_dir}/0/3/1", f"{working_dir}/0/3/2", f"{working_dir}/0/3/3", - f"{working_dir}/0/3/4" + f"{working_dir}/0/3/4", ] expected_all_dirs = " ".join(expected_all_dirs_list) assert all_dirs == expected_all_dirs @@ -444,12 +446,13 @@ def test_subhierarchy_insertion(self, temp_output_dir: str): top = read_hierarchy(os.path.abspath(working_dir)) # Compare results - expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ - " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ - " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ - " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ - " 1.0: BUNDLE -1 MIN 1 MAX 2\n" \ - + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: BUNDLE -1 MIN 1 MAX 2\n" + ) assert str(top) == expected # Create and insert the sub hierarchy @@ -457,22 +460,23 @@ def test_subhierarchy_insertion(self, temp_output_dir: str): top["1.0"] = sub_h # Compare results - expected = ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" \ - " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" \ - " 0.0: BUNDLE -1 MIN 0 MAX 1\n" \ - " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" \ - " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" \ - " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" \ - " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" \ - " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" \ - " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" \ - " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" \ - " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" \ - " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" \ - " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" \ - " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" \ - " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" \ - + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" + " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" + " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" + " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" + " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" + " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" + " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" + " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" + " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" + " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" + " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" + ) assert str(top) == expected def test_sample_index_creation_and_insertion(self, temp_output_dir: str): @@ -510,5 +514,5 @@ def test_sample_index_creation_and_insertion(self, temp_output_dir: str): # Inserting hierarchy at 0.1 try: idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") - except KeyError as error: + except KeyError: assert False From 089e2bdf7eba1b9639343c833c97c6fb0149087d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 09:31:43 -0800 Subject: [PATCH 047/201] add tests for encryption modules --- merlin/common/security/encrypt.py | 9 +- tests/conftest.py | 37 ++++++++ tests/encryption_manager.py | 49 ++++++++++ tests/unit/common/test_encryption.py | 129 +++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 tests/encryption_manager.py create mode 100644 tests/unit/common/test_encryption.py diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index b1932cd28..ad42d79d9 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -52,11 +52,10 @@ def _get_key_path(): except AttributeError: key_filepath = "~/.merlin/encrypt_data_key" - try: - key_filepath = os.path.abspath(os.path.expanduser(key_filepath)) - except KeyError as e: - raise ValueError("Error! No password provided for RabbitMQ") from e - return key_filepath + if key_filepath is None: + raise ValueError("Error! No password provided for RabbitMQ") + + return os.path.abspath(os.path.expanduser(key_filepath)) def _gen_key(key_path): diff --git a/tests/conftest.py b/tests/conftest.py index bea07f64c..8dcf4daf6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,6 +53,8 @@ fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) ] +from tests.encryption_manager import EncryptionManager + class RedisServerError(Exception): """ @@ -216,3 +218,38 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl with CeleryTestWorkersManager(celery_app) as workers_manager: workers_manager.launch_workers(worker_info) yield + + +@pytest.fixture(scope="session") +def encryption_output_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name + """ + Get a temporary output directory for our encryption tests. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + encryption_dir = f"{temp_output_dir}/encryption_tests" + os.mkdir(encryption_dir) + return encryption_dir + + +@pytest.fixture(scope="session") +def test_encryption_key() -> bytes: + """An encryption key to be used for tests that need it""" + return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" + + +@pytest.fixture(scope="class") +def use_fake_encrypt_data_key(encryption_output_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name + """ + Create a fake encrypt data key to use for these tests. This will save the + current data key so we can set it back to what it was prior to running + the tests. + + :param encryption_output_dir: The path to the temporary output directory we'll be using for this test run + """ + # Use a context manager to ensure cleanup runs even if an error occurs + with EncryptionManager(encryption_output_dir, test_encryption_key) as encrypt_manager: + # Set the fake encryption key + encrypt_manager.set_fake_key() + # Yield control to the tests + yield diff --git a/tests/encryption_manager.py b/tests/encryption_manager.py new file mode 100644 index 000000000..883b1a184 --- /dev/null +++ b/tests/encryption_manager.py @@ -0,0 +1,49 @@ +""" +Module to define functionality for managing encryption settings +while running our test suite. +""" +import os +from types import TracebackType +from typing import Type + +from merlin.config.configfile import CONFIG + + +class EncryptionManager: + """ + A class to handle safe setup and teardown of encryption tests. + """ + + def __init__(self, temp_output_dir: str, test_encryption_key: bytes): + self.temp_output_dir = temp_output_dir + self.key_path = os.path.abspath(os.path.expanduser(f"{self.temp_output_dir}/encrypt_data_key")) + self.test_encryption_key = test_encryption_key + self.orig_results_backend = CONFIG.results_backend + + def __enter__(self): + """This magic method is necessary for allowing this class to be sued as a context manager""" + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + This will always run at the end of a context with statement, even if an error is raised. + It's a safe way to ensure all of our encryption settings at the start of the tests are reset. + """ + self.reset_encryption_settings() + + def set_fake_key(self): + """ + Create a fake encrypt data key to use for tests. This will save the fake encryption key to + our temporary output directory located at: + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/encryption_tests/ + """ + with open(self.key_path, "w") as key_file: + key_file.write(self.test_encryption_key.decode("utf-8")) + + CONFIG.results_backend.encryption_key = self.key_path + + def reset_encryption_settings(self): + """ + Reset the encryption settings to what they were prior to running our encryption tests. + """ + CONFIG.results_backend = self.orig_results_backend diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py new file mode 100644 index 000000000..6daa53817 --- /dev/null +++ b/tests/unit/common/test_encryption.py @@ -0,0 +1,129 @@ +""" +Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. +""" +import os + +import celery +import pytest + +from merlin.common.security.encrypt import _gen_key, _get_key, _get_key_path, decrypt, encrypt +from merlin.common.security.encrypt_backend_traffic import _decrypt_decode, _encrypt_encode, set_backend_funcs +from merlin.config.configfile import CONFIG + + +class TestEncryption: + """ + This class will house all tests necessary for our encryption modules. + """ + + def test_encrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + """ + Test that our encryption function is encrypting the bytes that we're + passing to it. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + """ + str_to_encrypt = b"super secret string shhh" + encrypted_str = encrypt(str_to_encrypt) + for word in str_to_encrypt.decode("utf-8").split(" "): + assert word not in encrypted_str.decode("utf-8") + + def test_decrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + """ + Test that our decryption function is decrypting the bytes that we're + passing to it. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + """ + # This is the output of the bytes from the encrypt test + str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" + decrypted_str = decrypt(str_to_decrypt) + assert decrypted_str == b"super secret string shhh" + + def test_get_key_path(self, use_fake_encrypt_data_key: "fixture"): # noqa F821 + """ + Test the `_get_key_path` function. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + """ + # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which + # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) + user = os.getlogin() + actual_default = _get_key_path() + assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encryption_tests/encrypt_data_key") + + # Test with having the encryption key set to None + temp = CONFIG.results_backend.encryption_key + CONFIG.results_backend.encryption_key = None + with pytest.raises(ValueError) as excinfo: + _get_key_path() + assert "Error! No password provided for RabbitMQ" in str(excinfo.value) + CONFIG.results_backend.encryption_key = temp + + # Test with having the entire results_backend wiped from CONFIG + orig_results_backend = CONFIG.results_backend + CONFIG.results_backend = None + actual_no_results_backend = _get_key_path() + assert actual_no_results_backend == os.path.abspath(os.path.expanduser("~/.merlin/encrypt_data_key")) + CONFIG.results_backend = orig_results_backend + + def test_gen_key(self, encryption_output_dir: str): + """ + Test the `_gen_key` function. + + :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + """ + # Create the file but don't put anything in it + key_gen_test_file = f"{encryption_output_dir}/key_gen_test" + with open(key_gen_test_file, "w"): + pass + + # Ensure nothing is in the file + with open(key_gen_test_file, "r") as key_gen_file: + key_gen_contents = key_gen_file.read() + assert key_gen_contents == "" + + # Run the test and then check to make sure the file is now populated + _gen_key(key_gen_test_file) + with open(key_gen_test_file, "r") as key_gen_file: + key_gen_contents = key_gen_file.read() + assert key_gen_contents != "" + + def test_get_key(self, use_fake_encrypt_data_key: str, encryption_output_dir: str, test_encryption_key: bytes): + """ + Test the `_get_key` function. + + :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + :param test_encryption_key: A fixture to establish a fixed encryption key for testing + """ + # Test the default functionality + actual_default = _get_key() + assert actual_default == test_encryption_key + + # Modify the permission of the key file so that it can't be read by anyone + # (we're purposefully trying to raise an IOError) + key_path = f"{encryption_output_dir}/encrypt_data_key" + orig_file_permissions = os.stat(key_path).st_mode + os.chmod(key_path, 0o222) + with pytest.raises(IOError): + _get_key() + os.chmod(key_path, orig_file_permissions) + + # Reset the key value to our test value since the IOError test will rewrite the key + with open(key_path, "w") as key_file: + key_file.write(test_encryption_key.decode("utf-8")) + + def test_set_backend_funcs(self): + """ + Test the `set_backend_funcs` function. + """ + # Make sure these values haven't been set yet + assert celery.backends.base.Backend.encode != _encrypt_encode + assert celery.backends.base.Backend.decode != _decrypt_decode + + set_backend_funcs() + + # Ensure the new functions have been set + assert celery.backends.base.Backend.encode == _encrypt_encode + assert celery.backends.base.Backend.decode == _decrypt_decode From 2a3bc69ee4114e19b4dd99f1d8aed3c67129ca58 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 10:52:34 -0800 Subject: [PATCH 048/201] add unit tests for util_sampling --- merlin/common/util_sampling.py | 1 + tests/unit/common/test_util_sampling.py | 44 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/unit/common/test_util_sampling.py diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 0a6c585cf..1396016c7 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -35,6 +35,7 @@ import numpy as np +# TODO should we move this to merlin-spellbook? def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False): """Scale samples to new limits, either log10 or linearly. diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py new file mode 100644 index 000000000..0fd77739f --- /dev/null +++ b/tests/unit/common/test_util_sampling.py @@ -0,0 +1,44 @@ +""" +Tests for the `util_sampling.py` file. +""" +import numpy as np +import pytest + +from merlin.common.util_sampling import scale_samples + + +class TestUtilSampling: + """ + This class will hold all of the tests for the `util_sampling.py` file. + """ + + def test_scale_samples_basic(self): + """Test basic functionality without logging""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(-1, 1), (2, 6)] + result = scale_samples(samples_norm, limits) + expected_result = np.array([[-0.6, 3.6], [0.2, 5.2]]) + np.testing.assert_array_almost_equal(result, expected_result) + + def test_scale_samples_logarithmic(self): + """Test functionality with log enabled""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(1, 5), (1, 100)] + result = scale_samples(samples_norm, limits, do_log=[False, True]) + expected_result = np.array([[1.8, 6.309573], [3.4, 39.810717]]) + np.testing.assert_array_almost_equal(result, expected_result) + + def test_scale_samples_invalid_input(self): + """Test that function raises ValueError for invalid input""" + with pytest.raises(ValueError): + # Invalid input: samples_norm should be a 2D array + scale_samples([0.2, 0.4, 0.6], [(1, 5), (2, 6)]) + + def test_scale_samples_with_custom_limits_norm(self): + """Test functionality with custom limits_norm""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(1, 5), (2, 6)] + limits_norm = (-1, 1) + result = scale_samples(samples_norm, limits, limits_norm=limits_norm) + expected_result = np.array([[3.4, 4.8], [4.2, 5.6]]) + np.testing.assert_array_almost_equal(result, expected_result) \ No newline at end of file From 30fd21d930af9ddd19fd3d3deb7c3f8654e43047 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 10:54:33 -0800 Subject: [PATCH 049/201] run fix-style and fix typo --- tests/unit/common/test_util_sampling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py index 0fd77739f..c957ac105 100644 --- a/tests/unit/common/test_util_sampling.py +++ b/tests/unit/common/test_util_sampling.py @@ -13,7 +13,7 @@ class TestUtilSampling: """ def test_scale_samples_basic(self): - """Test basic functionality without logging""" + """Test basic functionality""" samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) limits = [(-1, 1), (2, 6)] result = scale_samples(samples_norm, limits) @@ -41,4 +41,4 @@ def test_scale_samples_with_custom_limits_norm(self): limits_norm = (-1, 1) result = scale_samples(samples_norm, limits, limits_norm=limits_norm) expected_result = np.array([[3.4, 4.8], [4.2, 5.6]]) - np.testing.assert_array_almost_equal(result, expected_result) \ No newline at end of file + np.testing.assert_array_almost_equal(result, expected_result) From 76b3a55ff5b57ba3df32c960cb012992cb150938 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 11:52:48 -0800 Subject: [PATCH 050/201] create directory for context managers and fix issue with an encryption test --- tests/conftest.py | 5 +++-- tests/context_managers/__init__.py | 0 .../celery_workers_manager.py} | 5 +++-- tests/{ => context_managers}/encryption_manager.py | 0 tests/unit/common/test_encryption.py | 6 ++++++ 5 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 tests/context_managers/__init__.py rename tests/{celery_test_workers.py => context_managers/celery_workers_manager.py} (98%) rename tests/{ => context_managers}/encryption_manager.py (100%) diff --git a/tests/conftest.py b/tests/conftest.py index 8dcf4daf6..12d9eac4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,7 +53,8 @@ fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) ] -from tests.encryption_manager import EncryptionManager +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.context_managers.encryption_manager import EncryptionManager class RedisServerError(Exception): @@ -215,7 +216,7 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl # (basically just add in concurrency value to worker_queue_map) worker_info = {worker_name: {"concurrency": 1, "queues": [queue]} for worker_name, queue in worker_queue_map.items()} - with CeleryTestWorkersManager(celery_app) as workers_manager: + with CeleryWorkersManager(celery_app) as workers_manager: workers_manager.launch_workers(worker_info) yield diff --git a/tests/context_managers/__init__.py b/tests/context_managers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/celery_test_workers.py b/tests/context_managers/celery_workers_manager.py similarity index 98% rename from tests/celery_test_workers.py rename to tests/context_managers/celery_workers_manager.py index ad81d30e6..2b4c22094 100644 --- a/tests/celery_test_workers.py +++ b/tests/context_managers/celery_workers_manager.py @@ -40,9 +40,9 @@ from typing import Dict, List, Type from celery import Celery +from merlin.config.configfile import CONFIG - -class CeleryTestWorkersManager: +class CeleryWorkersManager: """ A class to handle the setup and teardown of celery workers. This should be treated as a context and used with python's @@ -201,6 +201,7 @@ def launch_workers(self, worker_info: Dict[str, Dict]): :param worker_info: A dict of worker info with the form {"worker_name": {"concurrency": , "queues": }} """ + # CONFIG.results_backend.encryption_key = "~/.merlin/encrypt_data_key" for worker_name, worker_settings in worker_info.items(): self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) diff --git a/tests/encryption_manager.py b/tests/context_managers/encryption_manager.py similarity index 100% rename from tests/encryption_manager.py rename to tests/context_managers/encryption_manager.py diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 6daa53817..6f978ddfe 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -118,6 +118,9 @@ def test_set_backend_funcs(self): """ Test the `set_backend_funcs` function. """ + orig_encode = celery.backends.base.Backend.encode + orig_decode = celery.backends.base.Backend.decode + # Make sure these values haven't been set yet assert celery.backends.base.Backend.encode != _encrypt_encode assert celery.backends.base.Backend.decode != _decrypt_decode @@ -127,3 +130,6 @@ def test_set_backend_funcs(self): # Ensure the new functions have been set assert celery.backends.base.Backend.encode == _encrypt_encode assert celery.backends.base.Backend.decode == _decrypt_decode + + celery.backends.base.Backend.encode = orig_encode + celery.backends.base.Backend.decode = orig_decode From fa33cb27424706d2d70f19b5f32d8841527677df Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Dec 2023 17:31:41 -0800 Subject: [PATCH 051/201] add a context manager for spinning up/down the redis server --- tests/conftest.py | 83 ++------------ .../celery_workers_manager.py | 3 +- tests/context_managers/encryption_manager.py | 2 +- tests/context_managers/server_manager.py | 105 ++++++++++++++++++ 4 files changed, 116 insertions(+), 77 deletions(-) create mode 100644 tests/context_managers/server_manager.py diff --git a/tests/conftest.py b/tests/conftest.py index 12d9eac4d..584ca1e55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,7 +38,6 @@ from typing import Dict import pytest -import redis from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature @@ -55,18 +54,7 @@ from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.encryption_manager import EncryptionManager - - -class RedisServerError(Exception): - """ - Exception to signal that the server wasn't pinged properly. - """ - - -class ServerInitError(Exception): - """ - Exception to signal that there was an error initializing the server. - """ +from tests.context_managers.server_manager import RedisServerManager @pytest.fixture(scope="session") @@ -91,73 +79,20 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def redis_pass() -> str: - """ - This fixture represents the password to the merlin test server. - - :returns: The redis password for our test server - """ - return "merlin-test-server" - - -@pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name - """ - This fixture will initialize the merlin server (i.e. create all the files we'll - need to start up a local redis server). It will return the path to the directory - containing the files needed for the server to start up. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :param redis_pass: The password to the test redis server that we'll create here - :returns: The path to the merlin_server directory with the server configurations - """ - # Initialize the setup for the local redis server - # We'll also set the password to 'merlin-test-server' so it'll be easy to shutdown if there's an issue - subprocess.run(f"merlin server init; merlin server config -pwd {redis_pass}", shell=True, capture_output=True, text=True) - - # Check that the merlin server was initialized properly - server_dir = f"{temp_output_dir}/merlin_server" - if not os.path.exists(server_dir): - raise ServerInitError("The merlin server was not initialized properly.") - - return server_dir - - -@pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name,unused-argument +def redis_server(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. - :param merlin_server_dir: The directory to the merlin test server configuration. - This will not be used here but we need the server configurations before we can - start the server. - :param redis_pass: The raw redis password stored in the redis.pass file + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :yields: The local redis server uri """ - # Start the local redis server - try: - # Need to set LC_ALL='C' before starting the server or else redis causes a failure - subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) - except subprocess.TimeoutExpired: - pass - - # Ensure the server started properly - host = "localhost" - port = 6379 - database = 0 - username = "default" - redis_client = redis.Redis(host=host, port=port, db=database, password=redis_pass, username=username) - if not redis_client.ping(): - raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") - - # Hand over the redis server url to any other fixtures/tests that need it - redis_server_uri = f"redis://{username}:{redis_pass}@{host}:{port}/{database}" - yield redis_server_uri - - # Kill the server; don't run this until all tests are done (accomplished with 'yield' above) - kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) - assert "Merlin server terminated." in kill_process.stderr + with RedisServerManager(temp_output_dir) as redis_server_manager: + redis_server_manager.initialize_server() + redis_server_manager.start_server() + # Yield the redis_server uri to any fixtures/tests that may need it + yield redis_server_manager.redis_server_uri + # The server will be stopped once this context reaches the end of it's execution here @pytest.fixture(scope="session") diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 2b4c22094..0acdee130 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -40,7 +40,7 @@ from typing import Dict, List, Type from celery import Celery -from merlin.config.configfile import CONFIG + class CeleryWorkersManager: """ @@ -201,7 +201,6 @@ def launch_workers(self, worker_info: Dict[str, Dict]): :param worker_info: A dict of worker info with the form {"worker_name": {"concurrency": , "queues": }} """ - # CONFIG.results_backend.encryption_key = "~/.merlin/encrypt_data_key" for worker_name, worker_settings in worker_info.items(): self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) diff --git a/tests/context_managers/encryption_manager.py b/tests/context_managers/encryption_manager.py index 883b1a184..84b2e4a1e 100644 --- a/tests/context_managers/encryption_manager.py +++ b/tests/context_managers/encryption_manager.py @@ -21,7 +21,7 @@ def __init__(self, temp_output_dir: str, test_encryption_key: bytes): self.orig_results_backend = CONFIG.results_backend def __enter__(self): - """This magic method is necessary for allowing this class to be sued as a context manager""" + """This magic method is necessary for allowing this class to be used as a context manager""" return self def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py new file mode 100644 index 000000000..d373c1f1c --- /dev/null +++ b/tests/context_managers/server_manager.py @@ -0,0 +1,105 @@ +""" +Module to define functionality for managing the containerized +server used for testing. +""" +import os +import signal +import subprocess +from types import TracebackType +from typing import Type + +import redis +import yaml + + +class RedisServerError(Exception): + """ + Exception to signal that the server wasn't pinged properly. + """ + + +class ServerInitError(Exception): + """ + Exception to signal that there was an error initializing the server. + """ + + +class RedisServerManager: + """ + A class to handle the setup and teardown of a containerized redis server. + This should be treated as a context and used with python's built-in 'with' + statement. If you use it without this statement, beware that the processes + spun up here may never be stopped. + """ + + def __init__(self, temp_output_dir: str): + self._redis_pass = "merlin-test-server" + self.server_dir = f"{temp_output_dir}/merlin_server" + self.host = "localhost" + self.port = 6379 + self.database = 0 + self.username = "default" + self.redis_server_uri = f"redis://{self.username}:{self._redis_pass}@{self.host}:{self.port}/{self.database}" + + def __enter__(self): + """This magic method is necessary for allowing this class to be used as a context manager""" + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + This will always run at the end of a context with statement, even if an error is raised. + It's a safe way to ensure all of our server gets stopped no matter what. + """ + self.stop_server() + + def initialize_server(self): + """ + Initialize the setup for the local redis server. We'll write the folder to: + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ + We'll set the password to be 'merlin-test-server' so it'll be easy to shutdown if necessary + """ + subprocess.run( + f"merlin server init; merlin server config -pwd {self._redis_pass}", shell=True, capture_output=True, text=True + ) + + # Check that the merlin server was initialized properly + if not os.path.exists(self.server_dir): + raise ServerInitError("The merlin server was not initialized properly.") + + def start_server(self): + """Attempt to start the local redis server.""" + try: + # Need to set LC_ALL='C' before starting the server or else redis causes a failure + subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) + except subprocess.TimeoutExpired: + pass + + # Ensure the server started properly + redis_client = redis.Redis( + host=self.host, port=self.port, db=self.database, password=self._redis_pass, username=self.username + ) + if not redis_client.ping(): + raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + + def stop_server(self): + """Stop the server.""" + # Attempt to stop the server gracefully with `merlin server` + kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) + + # Check that the server was terminated + if "Merlin server terminated." not in kill_process.stderr: + # If it wasn't, try to kill the process by using the pid stored in a file created by `merlin server` + try: + with open(f"{self.server_dir}/merlin_server.pf", "r") as process_file: + server_process_info = yaml.load(process_file, yaml.Loader) + os.kill(int(server_process_info["image_pid"]), signal.SIGKILL) + # If the file can't be found then let's make sure there's even a redis-server process running + except FileNotFoundError as exc: + process_query = subprocess.run("ps ux", shell=True, text=True, capture_output=True) + # If there is a file running we didn't start it in this test run so we can't kill it + if "redis-server" in process_query.stdout: + raise RedisServerError( + "Found an active redis server but cannot stop it since there is no process file (merlin_server.pf). " + "Did you start this server before running tests?" + ) from exc + # No else here. If there's no redis-server process found then there's nothing to stop From 78a298a9f1eb3fcf21175ff0d5a5624b6df844d6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 13 Dec 2023 14:42:41 -0800 Subject: [PATCH 052/201] fix issue with path in one test --- tests/unit/common/test_sample_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index 296783273..cdb5b2f4f 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -64,7 +64,7 @@ def test_new_dir(temp_output_dir: str): temporary output path for our tests """ # Test basic functionality - test_path = f"{os.getcwd()}/test_new_dir" + test_path = f"{os.getcwd()}/test_sample_index/test_new_dir" new_dir(test_path) assert os.path.exists(test_path) From 5b6878cb00fa9790ce802b0a5df5ea29f4c310c8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 13 Dec 2023 14:44:12 -0800 Subject: [PATCH 053/201] rework CONFIG functionality for testing --- merlin/config/__init__.py | 30 ++++++ tests/conftest.py | 99 +++++++++++++------- tests/context_managers/encryption_manager.py | 49 ---------- tests/context_managers/server_manager.py | 29 +++++- tests/unit/common/test_encryption.py | 31 +++--- 5 files changed, 135 insertions(+), 103 deletions(-) delete mode 100644 tests/context_managers/encryption_manager.py diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 41645e249..fe6f4a000 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -31,6 +31,7 @@ """ Used to store the application configuration. """ +from copy import copy from types import SimpleNamespace from typing import Dict, List, Optional @@ -56,6 +57,35 @@ def __init__(self, app_dict): self.results_backend: Optional[SimpleNamespace] self.load_app_into_namespaces(app_dict) + def __copy__(self): + """ + A magic method to allow this class to be copied with copy(instance_of_Config). + """ + cls = self.__class__ + result = cls.__new__(cls) + copied_attrs = { + "celery": copy(self.__dict__["celery"]), + "broker": copy(self.__dict__["broker"]), + "results_backend": copy(self.__dict__["results_backend"]), + } + result.__dict__.update(copied_attrs) + return result + + def __str__(self): + """ + A magic method so we can print the CONFIG class. + """ + formatted_str = "config:" + attrs = {"celery": self.celery, "broker": self.broker, "results_backend": self.results_backend} + for name, attr in attrs.items(): + if attr is not None: + items = (f" {k}: {v!r}" for k, v in attr.__dict__.items()) + joined_items = "\n".join(items) + formatted_str += f"\n {name}: \n{joined_items}" + else: + formatted_str += f"\n {name}: \n None" + return formatted_str + def load_app_into_namespaces(self, app_dict: Dict) -> None: """ Makes the application dictionary into a namespace, sets the attributes of the Config from the namespace values. diff --git a/tests/conftest.py b/tests/conftest.py index 584ca1e55..16ff529b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,21 +28,25 @@ # SOFTWARE. ############################################################################### """ -This module contains pytest fixtures to be used throughout the entire -integration test suite. +This module contains pytest fixtures to be used throughout the entire test suite. """ import os -import subprocess from glob import glob +import yaml +from copy import copy from time import sleep -from typing import Dict +from typing import Any, Dict import pytest from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature -from tests.celery_test_workers import CeleryTestWorkersManager +from merlin.config.configfile import CONFIG +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.context_managers.server_manager import RedisServerManager + +REDIS_PASS = "merlin-test-server" ####################################### @@ -52,11 +56,6 @@ fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) ] -from tests.context_managers.celery_workers_manager import CeleryWorkersManager -from tests.context_managers.encryption_manager import EncryptionManager -from tests.context_managers.server_manager import RedisServerManager - - @pytest.fixture(scope="session") def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: """ @@ -79,15 +78,27 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def redis_server(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name +def merlin_server_dir(temp_output_dir: str) -> str: + """ + The path to the merlin_server directory that will be created by the `redis_server` fixture. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture + """ + return f"{temp_output_dir}/merlin_server" + + +@pytest.fixture(scope="session") +def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # pylint: disable=redefined-outer-name """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ - with RedisServerManager(temp_output_dir) as redis_server_manager: + with RedisServerManager(merlin_server_dir, REDIS_PASS, test_encryption_key) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() # Yield the redis_server uri to any fixtures/tests that may need it @@ -157,35 +168,53 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl @pytest.fixture(scope="session") -def encryption_output_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name +def test_encryption_key() -> bytes: """ - Get a temporary output directory for our encryption tests. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + An encryption key to be used for tests that need it. + + :returns: The test encryption key """ - encryption_dir = f"{temp_output_dir}/encryption_tests" - os.mkdir(encryption_dir) - return encryption_dir + return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" @pytest.fixture(scope="session") -def test_encryption_key() -> bytes: - """An encryption key to be used for tests that need it""" - return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" +def app_yaml(merlin_server_dir: str, redis_server: str) -> Dict[str, Any]: # pylint: disable=redefined-outer-name + """ + Load in the app.yaml file generated by starting the redis server. + :param merlin_server_dir: The directory to the merlin test server configuration + :param redis_server: The fixture that starts up the redis server + :returns: The contents of the app.yaml file created by starting the redis server + """ + with open(f"{merlin_server_dir}/app.yaml", "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + return app_yaml -@pytest.fixture(scope="class") -def use_fake_encrypt_data_key(encryption_output_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name + +@pytest.fixture(scope="function") +def config(app_yaml: str): # pylint: disable=redefined-outer-name """ - Create a fake encrypt data key to use for these tests. This will save the - current data key so we can set it back to what it was prior to running - the tests. + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object. This will modify the CONFIG object to use static test values + that shouldn't change. - :param encryption_output_dir: The path to the temporary output directory we'll be using for this test run + :param app_yaml: The contents of the app.yaml created by starting the containerized redis server """ - # Use a context manager to ensure cleanup runs even if an error occurs - with EncryptionManager(encryption_output_dir, test_encryption_key) as encrypt_manager: - # Set the fake encryption key - encrypt_manager.set_fake_key() - # Yield control to the tests - yield + global CONFIG + orig_config = copy(CONFIG) + + CONFIG.broker.password = app_yaml["broker"]["password"] + CONFIG.broker.port = app_yaml["broker"]["port"] + CONFIG.broker.server = app_yaml["broker"]["server"] + CONFIG.broker.username = app_yaml["broker"]["username"] + CONFIG.broker.vhost = app_yaml["broker"]["vhost"] + + CONFIG.results_backend.password = app_yaml["results_backend"]["password"] + CONFIG.results_backend.port = app_yaml["results_backend"]["port"] + CONFIG.results_backend.server = app_yaml["results_backend"]["server"] + CONFIG.results_backend.username = app_yaml["results_backend"]["username"] + CONFIG.results_backend.encryption_key = app_yaml["results_backend"]["encryption_key"] + + yield + + CONFIG = orig_config diff --git a/tests/context_managers/encryption_manager.py b/tests/context_managers/encryption_manager.py deleted file mode 100644 index 84b2e4a1e..000000000 --- a/tests/context_managers/encryption_manager.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Module to define functionality for managing encryption settings -while running our test suite. -""" -import os -from types import TracebackType -from typing import Type - -from merlin.config.configfile import CONFIG - - -class EncryptionManager: - """ - A class to handle safe setup and teardown of encryption tests. - """ - - def __init__(self, temp_output_dir: str, test_encryption_key: bytes): - self.temp_output_dir = temp_output_dir - self.key_path = os.path.abspath(os.path.expanduser(f"{self.temp_output_dir}/encrypt_data_key")) - self.test_encryption_key = test_encryption_key - self.orig_results_backend = CONFIG.results_backend - - def __enter__(self): - """This magic method is necessary for allowing this class to be used as a context manager""" - return self - - def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): - """ - This will always run at the end of a context with statement, even if an error is raised. - It's a safe way to ensure all of our encryption settings at the start of the tests are reset. - """ - self.reset_encryption_settings() - - def set_fake_key(self): - """ - Create a fake encrypt data key to use for tests. This will save the fake encryption key to - our temporary output directory located at: - /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/encryption_tests/ - """ - with open(self.key_path, "w") as key_file: - key_file.write(self.test_encryption_key.decode("utf-8")) - - CONFIG.results_backend.encryption_key = self.key_path - - def reset_encryption_settings(self): - """ - Reset the encryption settings to what they were prior to running our encryption tests. - """ - CONFIG.results_backend = self.orig_results_backend diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index d373c1f1c..9a10e0cbf 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -32,9 +32,10 @@ class RedisServerManager: spun up here may never be stopped. """ - def __init__(self, temp_output_dir: str): - self._redis_pass = "merlin-test-server" - self.server_dir = f"{temp_output_dir}/merlin_server" + def __init__(self, server_dir: str, redis_pass: str, test_encryption_key: bytes): + self._redis_pass = redis_pass + self._test_encryption_key = test_encryption_key + self.server_dir = server_dir self.host = "localhost" self.port = 6379 self.database = 0 @@ -66,6 +67,26 @@ def initialize_server(self): if not os.path.exists(self.server_dir): raise ServerInitError("The merlin server was not initialized properly.") + def _create_fake_encryption_key(self): + """ + For testing we'll use a specific encryption key. We'll create a file for that and + save it to the app.yaml created for testing. + """ + # Create a fake encryption key file for testing purposes + encryption_file = f"{self.server_dir}/encrypt_data_key" + with open(encryption_file, "w") as key_file: + key_file.write(self._test_encryption_key.decode("utf-8")) + + # Load up the app.yaml that was created by starting the server + server_app_yaml = f"{self.server_dir}/app.yaml" + with open(server_app_yaml, "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + + # Modify the path to the encryption key and then save it + app_yaml["results_backend"]["encryption_key"] = encryption_file + with open(server_app_yaml, "w") as app_yaml_file: + yaml.dump(app_yaml, app_yaml_file) + def start_server(self): """Attempt to start the local redis server.""" try: @@ -81,6 +102,8 @@ def start_server(self): if not redis_client.ping(): raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + self._create_fake_encryption_key() + def stop_server(self): """Stop the server.""" # Attempt to stop the server gracefully with `merlin server` diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 6f978ddfe..012c5c540 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -16,41 +16,40 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + def test_encrypt(self, config: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, use_fake_encrypt_data_key: "fixture"): # noqa: F821 + def test_decrypt(self, config: "fixture"): # noqa: F821 """ - Test that our decryption function is decrypting the bytes that we're - passing to it. + Test that our decryption function is decrypting the bytes that we're passing to it. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, use_fake_encrypt_data_key: "fixture"): # noqa F821 + def test_get_key_path(self, config: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) user = os.getlogin() actual_default = _get_key_path() - assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encryption_tests/encrypt_data_key") + assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encrypt_data_key") # Test with having the encryption key set to None temp = CONFIG.results_backend.encryption_key @@ -67,14 +66,14 @@ def test_get_key_path(self, use_fake_encrypt_data_key: "fixture"): # noqa F821 assert actual_no_results_backend == os.path.abspath(os.path.expanduser("~/.merlin/encrypt_data_key")) CONFIG.results_backend = orig_results_backend - def test_gen_key(self, encryption_output_dir: str): + def test_gen_key(self, temp_output_dir: str): """ Test the `_gen_key` function. - :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + :param temp_output_dir: The path to the temporary output directory for this test run """ # Create the file but don't put anything in it - key_gen_test_file = f"{encryption_output_dir}/key_gen_test" + key_gen_test_file = f"{temp_output_dir}/key_gen_test" with open(key_gen_test_file, "w"): pass @@ -89,13 +88,13 @@ def test_gen_key(self, encryption_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, use_fake_encrypt_data_key: str, encryption_output_dir: str, test_encryption_key: bytes): + def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, config: "fixture"): # noqa: F821 """ Test the `_get_key` function. - :param use_fake_encrypt_data_key: A fixture to set up a fake encryption key for testing - :param encryption_output_dir: A fixture to create a temporary output directory for our encryption tests + :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing + :param config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() @@ -103,7 +102,7 @@ def test_get_key(self, use_fake_encrypt_data_key: str, encryption_output_dir: st # Modify the permission of the key file so that it can't be read by anyone # (we're purposefully trying to raise an IOError) - key_path = f"{encryption_output_dir}/encrypt_data_key" + key_path = f"{merlin_server_dir}/encrypt_data_key" orig_file_permissions = os.stat(key_path).st_mode os.chmod(key_path, 0o222) with pytest.raises(IOError): From 7aaa832fe7f53bc1cdc2842fa39ef31f5e0d08a2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 11:52:08 -0800 Subject: [PATCH 054/201] refactor config fixture so it doesn't depend on redis server to be started --- tests/conftest.py | 159 ++++++++++++++++++----- tests/context_managers/server_manager.py | 25 +--- tests/unit/common/test_encryption.py | 19 +-- 3 files changed, 135 insertions(+), 68 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 16ff529b4..47db35133 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,7 +46,81 @@ from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager -REDIS_PASS = "merlin-test-server" +SERVER_PASS = "merlin-test-server" + + +####################################### +#### Helper Functions for Fixtures #### +####################################### + + +def create_pass_file(pass_filepath: str): + """ + Check if a password file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the password to the file. + + :param pass_filepath: The path to the password file that we need to check for/create + """ + if not os.path.exists(pass_filepath): + with open(pass_filepath, "w") as pass_file: + pass_file.write(SERVER_PASS) + + +def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_filepath: str = None): + """ + Check if an encryption file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the encryption key to the file. If an app.yaml + filepath has been passed to this function then we'll need to update it so that the encryption + key points to the `key_filepath`. + + :param key_filepath: The path to the file that will store our encryption key + :param encryption_key: An encryption key to be used for testing + :param app_yaml_filepath: A path to the app.yaml file that needs to be updated + """ + if not os.path.exists(key_filepath): + with open(key_filepath, "w") as key_file: + key_file.write(encryption_key.decode("utf-8")) + + if app_yaml_filepath is not None: + # Load up the app.yaml that was created by starting the server + with open(app_yaml_filepath, "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + + # Modify the path to the encryption key and then save it + app_yaml["results_backend"]["encryption_key"] = key_filepath + with open(app_yaml_filepath, "w") as app_yaml_file: + yaml.dump(app_yaml, app_yaml_file) + + +def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): + """ + Given configuration options for the broker and results_backend, update + the CONFIG object. + + :param broker: A dict of the configuration settings for the broker + :param results_backend: A dict of configuration settings for the results_backend + """ + global CONFIG + + # Set the broker configuration for testing + CONFIG.broker.password = broker["password"] + CONFIG.broker.port = broker["port"] + CONFIG.broker.server = broker["server"] + CONFIG.broker.username = broker["username"] + CONFIG.broker.vhost = broker["vhost"] + CONFIG.broker.name = broker["name"] + + # Set the results_backend configuration for testing + CONFIG.results_backend.password = results_backend["password"] + CONFIG.results_backend.port = results_backend["port"] + CONFIG.results_backend.server = results_backend["server"] + CONFIG.results_backend.username = results_backend["username"] + CONFIG.results_backend.encryption_key = results_backend["encryption_key"] + + +####################################### +######### Fixture Definitions ######### +####################################### ####################################### @@ -85,7 +159,10 @@ def merlin_server_dir(temp_output_dir: str) -> str: :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture """ - return f"{temp_output_dir}/merlin_server" + server_dir = f"{temp_output_dir}/merlin_server" + if not os.path.exists(server_dir): + os.mkdir(server_dir) + return server_dir @pytest.fixture(scope="session") @@ -98,9 +175,10 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ - with RedisServerManager(merlin_server_dir, REDIS_PASS, test_encryption_key) as redis_server_manager: + with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() + create_encryption_file(f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml") # Yield the redis_server uri to any fixtures/tests that may need it yield redis_server_manager.redis_server_uri # The server will be stopped once this context reaches the end of it's execution here @@ -171,50 +249,61 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl def test_encryption_key() -> bytes: """ An encryption key to be used for tests that need it. - + :returns: The test encryption key """ return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" -@pytest.fixture(scope="session") -def app_yaml(merlin_server_dir: str, redis_server: str) -> Dict[str, Any]: # pylint: disable=redefined-outer-name - """ - Load in the app.yaml file generated by starting the redis server. - - :param merlin_server_dir: The directory to the merlin test server configuration - :param redis_server: The fixture that starts up the redis server - :returns: The contents of the app.yaml file created by starting the redis server - """ - with open(f"{merlin_server_dir}/app.yaml", "r") as app_yaml_file: - app_yaml = yaml.load(app_yaml_file, yaml.Loader) - return app_yaml - - @pytest.fixture(scope="function") -def config(app_yaml: str): # pylint: disable=redefined-outer-name +def redis_config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name """ This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object. This will modify the CONFIG object to use static test values - that shouldn't change. + that uses the CONFIG object with a Redis broker and results_backend. - :param app_yaml: The contents of the app.yaml created by starting the containerized redis server + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: An encryption key to be used for testing """ global CONFIG - orig_config = copy(CONFIG) - CONFIG.broker.password = app_yaml["broker"]["password"] - CONFIG.broker.port = app_yaml["broker"]["port"] - CONFIG.broker.server = app_yaml["broker"]["server"] - CONFIG.broker.username = app_yaml["broker"]["username"] - CONFIG.broker.vhost = app_yaml["broker"]["vhost"] - - CONFIG.results_backend.password = app_yaml["results_backend"]["password"] - CONFIG.results_backend.port = app_yaml["results_backend"]["port"] - CONFIG.results_backend.server = app_yaml["results_backend"]["server"] - CONFIG.results_backend.username = app_yaml["results_backend"]["username"] - CONFIG.results_backend.encryption_key = app_yaml["results_backend"]["encryption_key"] + # Create a copy of the CONFIG option so we can reset it after the test + orig_config = copy(CONFIG) + # Create a password file and encryption key file (if they don't already exist) + pass_file = f"{merlin_server_dir}/redis.pass" + key_file = f"{merlin_server_dir}/encrypt_data_key" + create_pass_file(pass_file) + create_encryption_file(key_file, test_encryption_key) + + # Create the broker and results_backend configuration to use + broker = { + "cert_reqs": "none", + "password": pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + "name": "redis", + } + + results_backend = { + "cert_reqs": "none", + "db_num": 0, + "encryption_key": key_file, + "password": pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "name": "redis", + } + + # Set the configuration + set_config(broker, results_backend) + + # Go run the tests yield - CONFIG = orig_config + # Reset the configuration + CONFIG.celery = orig_config.celery + CONFIG.broker = orig_config.broker + CONFIG.results_backend = orig_config.results_backend diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index 9a10e0cbf..ea6a731ff 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -32,9 +32,8 @@ class RedisServerManager: spun up here may never be stopped. """ - def __init__(self, server_dir: str, redis_pass: str, test_encryption_key: bytes): + def __init__(self, server_dir: str, redis_pass: str): self._redis_pass = redis_pass - self._test_encryption_key = test_encryption_key self.server_dir = server_dir self.host = "localhost" self.port = 6379 @@ -67,26 +66,6 @@ def initialize_server(self): if not os.path.exists(self.server_dir): raise ServerInitError("The merlin server was not initialized properly.") - def _create_fake_encryption_key(self): - """ - For testing we'll use a specific encryption key. We'll create a file for that and - save it to the app.yaml created for testing. - """ - # Create a fake encryption key file for testing purposes - encryption_file = f"{self.server_dir}/encrypt_data_key" - with open(encryption_file, "w") as key_file: - key_file.write(self._test_encryption_key.decode("utf-8")) - - # Load up the app.yaml that was created by starting the server - server_app_yaml = f"{self.server_dir}/app.yaml" - with open(server_app_yaml, "r") as app_yaml_file: - app_yaml = yaml.load(app_yaml_file, yaml.Loader) - - # Modify the path to the encryption key and then save it - app_yaml["results_backend"]["encryption_key"] = encryption_file - with open(server_app_yaml, "w") as app_yaml_file: - yaml.dump(app_yaml, app_yaml_file) - def start_server(self): """Attempt to start the local redis server.""" try: @@ -102,8 +81,6 @@ def start_server(self): if not redis_client.ping(): raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") - self._create_fake_encryption_key() - def stop_server(self): """Stop the server.""" # Attempt to stop the server gracefully with `merlin server` diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 012c5c540..d65d201f2 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -1,6 +1,7 @@ """ Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. """ +import getpass import os import celery @@ -16,38 +17,38 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, config: "fixture"): # noqa: F821 + def test_encrypt(self, redis_config: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, config: "fixture"): # noqa: F821 + def test_decrypt(self, redis_config: "fixture"): # noqa: F821 """ Test that our decryption function is decrypting the bytes that we're passing to it. - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, config: "fixture"): # noqa: F821 + def test_get_key_path(self, redis_config: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) - user = os.getlogin() + user = getpass.getuser() actual_default = _get_key_path() assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encrypt_data_key") @@ -88,13 +89,13 @@ def test_gen_key(self, temp_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, config: "fixture"): # noqa: F821 + def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_config: "fixture"): # noqa: F821 """ Test the `_get_key` function. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing - :param config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() From acb6d43a9210ab203eb4eca9d90ac313cd40bffd Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 14:24:52 -0800 Subject: [PATCH 055/201] split CONFIG fixtures into rabbit and redis configs, run fix-style --- merlin/config/__init__.py | 1 - tests/conftest.py | 130 +++++++++++++++++---------- tests/unit/common/test_encryption.py | 4 +- 3 files changed, 85 insertions(+), 50 deletions(-) diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index fe6f4a000..14a37343d 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -32,7 +32,6 @@ Used to store the application configuration. """ from copy import copy - from types import SimpleNamespace from typing import Dict, List, Optional diff --git a/tests/conftest.py b/tests/conftest.py index 47db35133..175adbcd6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,12 +32,12 @@ """ import os from glob import glob -import yaml from copy import copy from time import sleep -from typing import Any, Dict +from typing import Dict import pytest +import yaml from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature @@ -46,9 +46,18 @@ from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager + SERVER_PASS = "merlin-test-server" +####################################### +# Loading in Module Specific Fixtures # +####################################### +pytest_plugins = [ + fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) +] + + ####################################### #### Helper Functions for Fixtures #### ####################################### @@ -85,7 +94,7 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi # Load up the app.yaml that was created by starting the server with open(app_yaml_filepath, "r") as app_yaml_file: app_yaml = yaml.load(app_yaml_file, yaml.Loader) - + # Modify the path to the encryption key and then save it app_yaml["results_backend"]["encryption_key"] = key_filepath with open(app_yaml_filepath, "w") as app_yaml_file: @@ -100,8 +109,6 @@ def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): :param broker: A dict of the configuration settings for the broker :param results_backend: A dict of configuration settings for the results_backend """ - global CONFIG - # Set the broker configuration for testing CONFIG.broker.password = broker["password"] CONFIG.broker.port = broker["port"] @@ -122,14 +129,6 @@ def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): ######### Fixture Definitions ######### ####################################### - -####################################### -# Loading in Module Specific Fixtures # -####################################### -pytest_plugins = [ - fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) -] - @pytest.fixture(scope="session") def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: """ @@ -152,7 +151,7 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str) -> str: +def merlin_server_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name """ The path to the merlin_server directory that will be created by the `redis_server` fixture. @@ -178,7 +177,9 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() - create_encryption_file(f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml") + create_encryption_file( + f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml" + ) # Yield the redis_server uri to any fixtures/tests that may need it yield redis_server_manager.redis_server_uri # The server will be stopped once this context reaches the end of it's execution here @@ -256,49 +257,42 @@ def test_encryption_key() -> bytes: @pytest.fixture(scope="function") -def redis_config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name +def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name """ - This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a Redis broker and results_backend. + DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. + This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` + fixtures. It sets up the CONFIG object but leaves certain broker settings unset. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: An encryption key to be used for testing """ - global CONFIG + # global CONFIG # Create a copy of the CONFIG option so we can reset it after the test orig_config = copy(CONFIG) - # Create a password file and encryption key file (if they don't already exist) - pass_file = f"{merlin_server_dir}/redis.pass" + # Create an encryption key file (if it doesn't already exist) key_file = f"{merlin_server_dir}/encrypt_data_key" - create_pass_file(pass_file) create_encryption_file(key_file, test_encryption_key) - # Create the broker and results_backend configuration to use - broker = { - "cert_reqs": "none", - "password": pass_file, - "port": 6379, - "server": "127.0.0.1", - "username": "default", - "vhost": "host4testing", - "name": "redis", - } - - results_backend = { - "cert_reqs": "none", - "db_num": 0, - "encryption_key": key_file, - "password": pass_file, - "port": 6379, - "server": "127.0.0.1", - "username": "default", - "name": "redis", - } - - # Set the configuration - set_config(broker, results_backend) + # Set the broker configuration for testing + CONFIG.broker.password = "password path not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.port = "port not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.name = "name not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.server = "127.0.0.1" + CONFIG.broker.username = "default" + CONFIG.broker.vhost = "host4testing" + CONFIG.broker.cert_reqs = "none" + + # Set the results_backend configuration for testing + CONFIG.results_backend.password = f"{merlin_server_dir}/redis.pass" + CONFIG.results_backend.port = 6379 + CONFIG.results_backend.server = "127.0.0.1" + CONFIG.results_backend.username = "default" + CONFIG.results_backend.cert_reqs = "none" + CONFIG.results_backend.encryption_key = key_file + CONFIG.results_backend.db_num = 0 + CONFIG.results_backend.name = "redis" # Go run the tests yield @@ -307,3 +301,47 @@ def redis_config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: CONFIG.celery = orig_config.celery CONFIG.broker = orig_config.broker CONFIG.results_backend = orig_config.results_backend + + +@pytest.fixture(scope="function") +def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a Redis broker and results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + # global CONFIG + + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + + CONFIG.broker.password = pass_file + CONFIG.broker.port = 6379 + CONFIG.broker.name = "redis" + + yield + + +@pytest.fixture(scope="function") +def rabbit_config( + merlin_server_dir: str, config: "fixture" +): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a RabbitMQ broker and Redis results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + # global CONFIG + + pass_file = f"{merlin_server_dir}/rabbit.pass" + create_pass_file(pass_file) + + CONFIG.broker.password = pass_file + CONFIG.broker.port = 5671 + CONFIG.broker.name = "rabbitmq" + + yield diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index d65d201f2..6392cf8da 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -1,7 +1,6 @@ """ Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. """ -import getpass import os import celery @@ -48,9 +47,8 @@ def test_get_key_path(self, redis_config: "fixture"): # noqa: F821 """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) - user = getpass.getuser() actual_default = _get_key_path() - assert actual_default.startswith(f"/tmp/{user}/") and actual_default.endswith("/encrypt_data_key") + assert actual_default.startswith("/tmp/") and actual_default.endswith("/encrypt_data_key") # Test with having the encryption key set to None temp = CONFIG.results_backend.encryption_key From c1bfc6ab25a93b345c66d40d755f83a57824bac0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 14:25:15 -0800 Subject: [PATCH 056/201] add unit tests for broker.py --- merlin/config/broker.py | 14 +- tests/unit/config/test_broker.py | 549 +++++++++++++++++++++++++++++++ 2 files changed, 553 insertions(+), 10 deletions(-) create mode 100644 tests/unit/config/test_broker.py diff --git a/merlin/config/broker.py b/merlin/config/broker.py index dc8131c28..fd33ba2e5 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -85,13 +85,13 @@ def get_rabbit_connection(include_password, conn="amqps"): password_filepath = CONFIG.broker.password LOG.debug(f"Broker: password filepath = {password_filepath}") password_filepath = os.path.abspath(expanduser(password_filepath)) - except KeyError as e: # pylint: disable=C0103 - raise ValueError("Broker: No password provided for RabbitMQ") from e + except (AttributeError, KeyError) as exc: + raise ValueError("Broker: No password provided for RabbitMQ") from exc try: password = read_file(password_filepath) - except IOError as e: # pylint: disable=C0103 - raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from e + except IOError as exc: + raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from exc try: port = CONFIG.broker.port @@ -205,12 +205,6 @@ def get_connection_string(include_password=True): except AttributeError: broker = "" - try: - config_path = CONFIG.celery.certs - config_path = os.path.abspath(os.path.expanduser(config_path)) - except AttributeError: - config_path = None - if broker not in BROKERS: raise ValueError(f"Error: {broker} is not a supported broker.") return _sort_valid_broker(broker, include_password) diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py new file mode 100644 index 000000000..9d4760f3e --- /dev/null +++ b/tests/unit/config/test_broker.py @@ -0,0 +1,549 @@ +""" +Tests for the `broker.py` file. +""" +import os +from ssl import CERT_NONE +from typing import Any, Dict + +import pytest + +from merlin.config.broker import ( + RABBITMQ_CONNECTION, + REDISSOCK_CONNECTION, + get_connection_string, + get_rabbit_connection, + get_redis_connection, + get_redissock_connection, + get_ssl_config, + read_file, +) +from merlin.config.configfile import CONFIG +from tests.conftest import SERVER_PASS, create_pass_file + + +def test_read_file(merlin_server_dir: str): + """ + Test the `read_file` function. We'll start up our containerized redis server + so that we have a password file to read here. + + :param merlin_server_dir: The directory to the merlin test server configuration + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + actual = read_file(pass_file) + assert actual == SERVER_PASS + + +def test_get_connection_string_invalid_broker(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: + ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "invalid_broker" + with pytest.raises(ValueError): + get_connection_string() + + +def test_get_connection_string_no_broker(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function without a broker name value in the CONFIG object. This + should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.name + with pytest.raises(ValueError): + get_connection_string() + + +def test_get_connection_string_simple(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function in the simplest way that we can. This function + will automatically check for a broker url and if it finds one in the CONFIG object it will just + return the value it finds. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + test_url = "test_url" + CONFIG.broker.url = test_url + actual = get_connection_string() + assert actual == test_url + + +def test_get_ssl_config_no_broker(redis_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function without a broker. This should return False. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.name + assert not get_ssl_config() + + +class TestRabbitBroker: + """ + This class will house all tests necessary for our broker module when using a + rabbit broker. + """ + + def run_get_rabbit_connection(self, expected_vals: Dict[str, Any], include_password: bool, conn: str): + """ + Helper method to run the tests for the `get_rabbit_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"conn": "", + "vhost": "host4testing", + "username": "default", + "password": "", + "server": "127.0.0.1", + "port": } + :param include_password: If True, include the password in the output. Otherwise don't. + :param conn: The connection type to pass in (either amqp or amqps) + """ + expected = RABBITMQ_CONNECTION.format(**expected_vals) + actual = get_rabbit_connection(include_password=include_password, conn=conn) + assert actual == expected + + def test_get_rabbit_connection(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_dont_include_password(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function but set include_password to False. This should * out the + password + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": "******", + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=False, conn=conn) + + def test_get_rabbit_connection_no_port_amqp(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use + 5672 as the port since we're using amqp as the connection. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + CONFIG.broker.name = "amqp" + conn = "amqp" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5672, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_no_port_amqps(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use + 5671 as the port since we're using amqps as the connection. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_no_password(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no password file set. This should raise a ValueError. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.password + with pytest.raises(ValueError) as excinfo: + get_rabbit_connection(True) + assert "Broker: No password provided for RabbitMQ" in str(excinfo.value) + + def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with an invalid password filepath. + This should raise a ValueError. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.password = "invalid_filepath" + expanded_filepath = os.path.abspath(os.path.expanduser(CONFIG.broker.password)) + with pytest.raises(ValueError) as excinfo: + get_rabbit_connection(True) + assert f"Broker: RabbitMQ password file {expanded_filepath} does not exist" in str(excinfo.value) + + def run_get_connection_string(self, expected_vals: Dict[str, Any]): + """ + Helper method to run the tests for the `get_connection_string`. + + :param expected_vals: A dict of expected values for this test. Format: + {"conn": "", + "vhost": "host4testing", + "username": "default", + "password": "", + "server": "127.0.0.1", + "port": } + """ + expected = RABBITMQ_CONNECTION.format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_connection_string_rabbitmq(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rabbitmq as the broker. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "conn": "amqps", + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_connection_string(expected_vals) + + def test_get_connection_string_amqp(self, rabbit_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with amqp as the broker. + + :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + CONFIG.broker.name = "amqp" + expected_vals = { + "conn": "amqp", + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5672, + } + self.run_get_connection_string(expected_vals) + + +class TestRedisBroker: + """ + This class will house all tests necessary for our broker module when using a + redis broker. + """ + + def run_get_redissock_connection(self, expected_vals: Dict[str, str]): + """ + Helper method to run the tests for the `get_redissock_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"db_num": "", "path": ""} + """ + expected = REDISSOCK_CONNECTION.format(**expected_vals) + actual = get_redissock_connection() + assert actual == expected + + def test_get_redissock_connection(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with both a db_num and a broker path set. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Create and store a fake path and db_num for testing + test_path = "/fake/path/to/broker" + test_db_num = "45" + CONFIG.broker.path = test_path + CONFIG.broker.db_num = test_db_num + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": test_db_num, "path": test_path} + self.run_get_redissock_connection(expected_vals) + + def test_get_redissock_connection_no_db(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with a broker path set but no db num. + This should default the db_num to 0. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Create and store a fake path for testing + test_path = "/fake/path/to/broker" + CONFIG.broker.path = test_path + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": 0, "path": test_path} + self.run_get_redissock_connection(expected_vals) + + def test_get_redissock_connection_no_path(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with a db num set but no broker path. + This should raise an AttributeError since there will be no path value to read from + in `CONFIG.broker`. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.db_num = "45" + with pytest.raises(AttributeError): + get_redissock_connection() + + def test_get_redissock_connection_no_path_nor_db(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with neither a broker path nor a db num set. + This should raise an AttributeError since there will be no path value to read from + in `CONFIG.broker`. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + with pytest.raises(AttributeError): + get_redissock_connection() + + def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_password: bool, use_ssl: bool): + """ + Helper method to run the tests for the `get_redis_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"urlbase": "", "spass": "", "server": "127.0.0.1", "port": , "db_num": } + :param include_password: If True, include the password in the output. Otherwise don't. + :param use_ssl: If True, use ssl for the connection. Otherwise don't. + """ + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) + assert expected == actual + + def test_get_redis_connection(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_no_port(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the port setting from the CONFIG object. This should still run and give us + port = 6379. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_with_db(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after adding the db_num setting to the CONFIG object. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + test_db_num = "45" + CONFIG.broker.db_num = test_db_num + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": test_db_num, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_no_username(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the username setting from the CONFIG object. This should still run and give us + username = ''. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.username + expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_invalid_pass_file(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after changing the permissions of the password file so it can't be opened. This should still + run and give us password = CONFIG.broker.password. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Capture the initial permissions of the password file so we can reset them + orig_file_permissions = os.stat(CONFIG.broker.password).st_mode + + # Change the permissions of the password file so it can't be read + os.chmod(CONFIG.broker.password, 0o222) + + try: + # Run the test + expected_vals = { + "urlbase": "redis", + "spass": f"default:{CONFIG.broker.password}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + except AssertionError as exc: + # If this test failed, make sure to reset the permissions in case other tests need to read this file + os.chmod(CONFIG.broker.password, orig_file_permissions) + raise AssertionError from exc + + os.chmod(CONFIG.broker.password, orig_file_permissions) + + def test_get_redis_connection_dont_include_password(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function without including the password. This should place 6 *s + where the password would normally be placed in spass. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) + + def test_get_redis_connection_use_ssl(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "rediss", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) + + def test_get_redis_connection_no_password(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the password setting from the CONFIG object. This should still run and give us + spass = ''. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.password + expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_connection_string_redis(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis as the broker (this is what our CONFIG + is set to by default with the redis_config fixture). + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert expected == actual + + def test_get_connection_string_rediss(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rediss (with two 's') as the broker. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "rediss" + expected_vals = { + "urlbase": "rediss", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert expected == actual + + def test_get_connection_string_redis_socket(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis+socket as the broker. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Change our broker + CONFIG.broker.name = "redis+socket" + + # Create and store a fake path and db_num for testing + test_path = "/fake/path/to/broker" + test_db_num = "45" + CONFIG.broker.path = test_path + CONFIG.broker.db_num = test_db_num + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": test_db_num, "path": test_path} + expected = REDISSOCK_CONNECTION.format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_ssl_config_redis(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). + This should return False. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert not get_ssl_config() + + def test_get_ssl_config_rediss(self, redis_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss (with two 's') as the broker. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "rediss" + expected = {"ssl_cert_reqs": CERT_NONE} + actual = get_ssl_config() + assert actual == expected From 33002919702349b99ccfcdc6a76863f6177c2648 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 15:31:57 -0800 Subject: [PATCH 057/201] add unit tests for the Config object --- merlin/config/__init__.py | 4 +- setup.cfg | 5 + tests/conftest.py | 4 +- tests/unit/config/test_config_object.py | 149 ++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/unit/config/test_config_object.py diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 14a37343d..7bd8028fb 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -80,9 +80,9 @@ def __str__(self): if attr is not None: items = (f" {k}: {v!r}" for k, v in attr.__dict__.items()) joined_items = "\n".join(items) - formatted_str += f"\n {name}: \n{joined_items}" + formatted_str += f"\n {name}:\n{joined_items}" else: - formatted_str += f"\n {name}: \n None" + formatted_str += f"\n {name}:\n None" return formatted_str def load_app_into_namespaces(self, app_dict: Dict) -> None: diff --git a/setup.cfg b/setup.cfg index a000df59a..0eaa116ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,3 +26,8 @@ max-line-length = 127 files=best_practices,test ignore_missing_imports=true + +[coverage:run] +omit = + merlin/ascii.py + merlin/config/celeryconfig.py diff --git a/tests/conftest.py b/tests/conftest.py index 175adbcd6..0f1aad0c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -326,8 +326,8 @@ def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylin @pytest.fixture(scope="function") def rabbit_config( - merlin_server_dir: str, config: "fixture" -): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a RabbitMQ broker and Redis results_backend. diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py new file mode 100644 index 000000000..bd658bc66 --- /dev/null +++ b/tests/unit/config/test_config_object.py @@ -0,0 +1,149 @@ +""" +Test the functionality of the Config object. +""" +from copy import copy, deepcopy +from types import SimpleNamespace + +from merlin.config import Config + + +class TestConfig: + """ + Class for testing the Config object. We'll store a valid `app_dict` + as an attribute here so that each test doesn't have to redefine it + each time. + """ + + app_dict = { + "celery": {"override": {"visibility_timeout": 86400}}, + "broker": { + "cert_reqs": "none", + "name": "rabbitmq", + "password": "/path/to/pass_file", + "port": 5671, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + }, + "results_backend": { + "cert_reqs": "none", + "db_num": 0, + "name": "rediss", + "password": "/path/to/pass_file", + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + "encryption_key": "/path/to/encryption_key", + }, + } + + def test_config_creation(self): + """ + Test the creation of the Config object. This should create nested namespaces + for each key in the `app_dict` variable and save them to their respective + attributes in the object. + """ + config = Config(self.app_dict) + + # Create the nested namespace for celery and compare result + override_namespace = SimpleNamespace(**self.app_dict["celery"]["override"]) + updated_celery_dict = deepcopy(self.app_dict) + updated_celery_dict["celery"]["override"] = override_namespace + celery_namespace = SimpleNamespace(**updated_celery_dict["celery"]) + assert config.celery == celery_namespace + + # Broker and Results Backend are easier since there's no nested namespace here + assert config.broker == SimpleNamespace(**self.app_dict["broker"]) + assert config.results_backend == SimpleNamespace(**self.app_dict["results_backend"]) + + def test_config_creation_no_celery(self): + """ + Test the creation of the Config object without the celery key. This should still + work and just not set anything for the celery attribute. + """ + + # Copy the celery section so we can restore it later and then delete it + celery_section = copy(self.app_dict["celery"]) + del self.app_dict["celery"] + config = Config(self.app_dict) + + # Broker and Results Backend are the only things loaded here + assert config.broker == SimpleNamespace(**self.app_dict["broker"]) + assert config.results_backend == SimpleNamespace(**self.app_dict["results_backend"]) + + # Ensure the celery attribute is not loaded + assert "celery" not in dir(config) + + # Reset celery section in case other tests use it after this + self.app_dict["celery"] = celery_section + + def test_config_copy(self): + """ + Test the `__copy__` magic method of the Config object. Here we'll make sure + each attribute was copied properly but the ids should be different. + """ + orig_config = Config(self.app_dict) + copied_config = copy(orig_config) + + assert orig_config.celery == copied_config.celery + assert orig_config.broker == copied_config.broker + assert orig_config.results_backend == copied_config.results_backend + + assert id(orig_config) != id(copied_config) + + def test_config_str(self): + """ + Test the `__str__` magic method of the Config object. This should just give us + a formatted string of the attributes in the object. + """ + config = Config(self.app_dict) + + # Test normal printing + actual = config.__str__() + expected = ( + "config:\n" + " celery:\n" + " override: namespace(visibility_timeout=86400)\n" + " broker:\n" + " cert_reqs: 'none'\n" + " name: 'rabbitmq'\n" + " password: '/path/to/pass_file'\n" + " port: 5671\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " results_backend:\n" + " cert_reqs: 'none'\n" + " db_num: 0\n" + " name: 'rediss'\n" + " password: '/path/to/pass_file'\n" + " port: 6379\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " encryption_key: '/path/to/encryption_key'" + ) + + assert actual == expected + + # Test printing with one section set to None + config.results_backend = None + actual_with_none = config.__str__() + expected_with_none = ( + "config:\n" + " celery:\n" + " override: namespace(visibility_timeout=86400)\n" + " broker:\n" + " cert_reqs: 'none'\n" + " name: 'rabbitmq'\n" + " password: '/path/to/pass_file'\n" + " port: 5671\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " results_backend:\n" + " None" + ) + + assert actual_with_none == expected_with_none From 58a30443aab1e0c48891dd8f1ae85f6ac7e11d7a Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 14 Dec 2023 15:32:08 -0800 Subject: [PATCH 058/201] update CHANGELOG --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b4427b1..8c325328c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,8 +95,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - this required adding a decent amount of test files to help with the tests; these can be found under the tests/unit/study/status_test_files directory - Pytest fixtures in the `conftest.py` file of the integration test suite - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command -- Tests for the `celeryadapter.py` module -- New CeleryTestWorkersManager context to help with starting/stopping workers for tests +- Coverage to the test suite. This includes adding tests for: + - `merlin/common/` + - `merlin/config/` + - `celeryadapter.py` +- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures + - RedisServerManager: context to help with starting/stopping a redis server for tests + - CeleryWorkersManager: context to help with starting/stopping workers for tests +- Ability to copy and print the `Config` object from `merlin/config/__init__.py` ### Changed - Reformatted the entire `merlin status` command From 9f16701e0cf13a9625f10274645d063cb2c116f2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 08:54:15 -0800 Subject: [PATCH 059/201] make CONFIG fixtures more flexible for tests --- tests/conftest.py | 79 ++++++++++++++---- tests/unit/common/test_encryption.py | 16 ++-- tests/unit/config/test_broker.py | 118 +++++++++++++-------------- 3 files changed, 132 insertions(+), 81 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0f1aad0c4..1c996958d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -256,6 +256,21 @@ def test_encryption_key() -> bytes: return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" +####################################### +########### CONFIG Fixtures ########### +####################################### +# These are intended to be used # +# either by themselves or together # +# For example, you can use a rabbit # +# broker config and a redis results # +# backend config together # +####################################### +############ !!!WARNING!!! ############ +# DO NOT USE THE `config` FIXTURE # +# IN A TEST; IT HAS UNSET VALUES # +####################################### + + @pytest.fixture(scope="function") def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name """ @@ -266,7 +281,6 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: An encryption key to be used for testing """ - # global CONFIG # Create a copy of the CONFIG option so we can reset it after the test orig_config = copy(CONFIG) @@ -276,23 +290,24 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab create_encryption_file(key_file, test_encryption_key) # Set the broker configuration for testing - CONFIG.broker.password = "password path not yet set" # This will be updated in `redis_config` or `rabbit_config` - CONFIG.broker.port = "port not yet set" # This will be updated in `redis_config` or `rabbit_config` - CONFIG.broker.name = "name not yet set" # This will be updated in `redis_config` or `rabbit_config` + CONFIG.broker.password = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.port = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.name = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` CONFIG.broker.server = "127.0.0.1" CONFIG.broker.username = "default" CONFIG.broker.vhost = "host4testing" CONFIG.broker.cert_reqs = "none" # Set the results_backend configuration for testing - CONFIG.results_backend.password = f"{merlin_server_dir}/redis.pass" - CONFIG.results_backend.port = 6379 + CONFIG.results_backend.password = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` + CONFIG.results_backend.name = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` CONFIG.results_backend.server = "127.0.0.1" CONFIG.results_backend.username = "default" CONFIG.results_backend.cert_reqs = "none" CONFIG.results_backend.encryption_key = key_file CONFIG.results_backend.db_num = 0 - CONFIG.results_backend.name = "redis" # Go run the tests yield @@ -304,7 +319,7 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab @pytest.fixture(scope="function") -def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +def redis_broker_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis broker and results_backend. @@ -312,8 +327,6 @@ def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylin :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - # global CONFIG - pass_file = f"{merlin_server_dir}/redis.pass" create_pass_file(pass_file) @@ -325,18 +338,35 @@ def redis_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylin @pytest.fixture(scope="function") -def rabbit_config( +def redis_results_backend_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a Redis results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.port = 6379 + CONFIG.results_backend.name = "redis" + + yield + + +@pytest.fixture(scope="function") +def rabbit_broker_config( merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument ): """ This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a RabbitMQ broker and Redis results_backend. + that uses the CONFIG object with a RabbitMQ broker. :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - # global CONFIG - pass_file = f"{merlin_server_dir}/rabbit.pass" create_pass_file(pass_file) @@ -345,3 +375,24 @@ def rabbit_config( CONFIG.broker.name = "rabbitmq" yield + + +@pytest.fixture(scope="function") +def mysql_results_backend_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a MySQL results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/mysql.pass" + create_pass_file(pass_file) + + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.name = "mysql" + CONFIG.results_backend.dbname = "test_mysql_db" + + yield diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 6392cf8da..d0069f09e 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -16,34 +16,34 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, redis_config: "fixture"): # noqa: F821 + def test_encrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, redis_config: "fixture"): # noqa: F821 + def test_decrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 """ Test that our decryption function is decrypting the bytes that we're passing to it. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, redis_config: "fixture"): # noqa: F821 + def test_get_key_path(self, redis_results_backend_config: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) @@ -87,13 +87,13 @@ def test_gen_key(self, temp_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_config: "fixture"): # noqa: F821 + def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture"): # noqa: F821 """ Test the `_get_key` function. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 9d4760f3e..490b47649 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -34,37 +34,37 @@ def test_read_file(merlin_server_dir: str): assert actual == SERVER_PASS -def test_get_connection_string_invalid_broker(redis_config: "fixture"): # noqa: F821 +def test_get_connection_string_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "invalid_broker" with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_no_broker(redis_config: "fixture"): # noqa: F821 +def test_get_connection_string_no_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function without a broker name value in the CONFIG object. This should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_simple(redis_config: "fixture"): # noqa: F821 +def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function in the simplest way that we can. This function will automatically check for a broker url and if it finds one in the CONFIG object it will just return the value it finds. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_url = "test_url" CONFIG.broker.url = test_url @@ -72,11 +72,11 @@ def test_get_connection_string_simple(redis_config: "fixture"): # noqa: F821 assert actual == test_url -def test_get_ssl_config_no_broker(redis_config: "fixture"): # noqa: F821 +def test_get_ssl_config_no_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function without a broker. This should return False. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name assert not get_ssl_config() @@ -106,11 +106,11 @@ def run_get_rabbit_connection(self, expected_vals: Dict[str, Any], include_passw actual = get_rabbit_connection(include_password=include_password, conn=conn) assert actual == expected - def test_get_rabbit_connection(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ conn = "amqps" expected_vals = { @@ -123,12 +123,12 @@ def test_get_rabbit_connection(self, rabbit_config: "fixture"): # noqa: F821 } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) - def test_get_rabbit_connection_dont_include_password(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_dont_include_password(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function but set include_password to False. This should * out the password - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ conn = "amqps" expected_vals = { @@ -141,12 +141,12 @@ def test_get_rabbit_connection_dont_include_password(self, rabbit_config: "fixtu } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=False, conn=conn) - def test_get_rabbit_connection_no_port_amqp(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_no_port_amqp(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use 5672 as the port since we're using amqp as the connection. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port CONFIG.broker.name = "amqp" @@ -161,12 +161,12 @@ def test_get_rabbit_connection_no_port_amqp(self, rabbit_config: "fixture"): # } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) - def test_get_rabbit_connection_no_port_amqps(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_no_port_amqps(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use 5671 as the port since we're using amqps as the connection. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port conn = "amqps" @@ -180,23 +180,23 @@ def test_get_rabbit_connection_no_port_amqps(self, rabbit_config: "fixture"): # } self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) - def test_get_rabbit_connection_no_password(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_no_password(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with no password file set. This should raise a ValueError. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.password with pytest.raises(ValueError) as excinfo: get_rabbit_connection(True) assert "Broker: No password provided for RabbitMQ" in str(excinfo.value) - def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_rabbit_connection` function with an invalid password filepath. This should raise a ValueError. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.password = "invalid_filepath" expanded_filepath = os.path.abspath(os.path.expanduser(CONFIG.broker.password)) @@ -220,11 +220,11 @@ def run_get_connection_string(self, expected_vals: Dict[str, Any]): actual = get_connection_string() assert actual == expected - def test_get_connection_string_rabbitmq(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_connection_string_rabbitmq(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rabbitmq as the broker. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "conn": "amqps", @@ -236,11 +236,11 @@ def test_get_connection_string_rabbitmq(self, rabbit_config: "fixture"): # noqa } self.run_get_connection_string(expected_vals) - def test_get_connection_string_amqp(self, rabbit_config: "fixture"): # noqa: F821 + def test_get_connection_string_amqp(self, rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with amqp as the broker. - :param rabbit_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port CONFIG.broker.name = "amqp" @@ -272,11 +272,11 @@ def run_get_redissock_connection(self, expected_vals: Dict[str, str]): actual = get_redissock_connection() assert actual == expected - def test_get_redissock_connection(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with both a db_num and a broker path set. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path and db_num for testing test_path = "/fake/path/to/broker" @@ -288,12 +288,12 @@ def test_get_redissock_connection(self, redis_config: "fixture"): # noqa: F821 expected_vals = {"db_num": test_db_num, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_db(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a broker path set but no db num. This should default the db_num to 0. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path for testing test_path = "/fake/path/to/broker" @@ -303,25 +303,25 @@ def test_get_redissock_connection_no_db(self, redis_config: "fixture"): # noqa: expected_vals = {"db_num": 0, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_path(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a db num set but no broker path. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.db_num = "45" with pytest.raises(AttributeError): get_redissock_connection() - def test_get_redissock_connection_no_path_nor_db(self, redis_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with neither a broker path nor a db num set. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ with pytest.raises(AttributeError): get_redissock_connection() @@ -339,11 +339,11 @@ def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_passwo actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) assert expected == actual - def test_get_redis_connection(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -354,13 +354,13 @@ def test_get_redis_connection(self, redis_config: "fixture"): # noqa: F821 } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_port(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the port setting from the CONFIG object. This should still run and give us port = 6379. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port expected_vals = { @@ -372,12 +372,12 @@ def test_get_redis_connection_no_port(self, redis_config: "fixture"): # noqa: F } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_with_db(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after adding the db_num setting to the CONFIG object. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_db_num = "45" CONFIG.broker.db_num = test_db_num @@ -390,25 +390,25 @@ def test_get_redis_connection_with_db(self, redis_config: "fixture"): # noqa: F } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_username(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_username(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the username setting from the CONFIG object. This should still run and give us username = ''. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.username expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_invalid_pass_file(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after changing the permissions of the password file so it can't be opened. This should still run and give us password = CONFIG.broker.password. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Capture the initial permissions of the password file so we can reset them orig_file_permissions = os.stat(CONFIG.broker.password).st_mode @@ -433,21 +433,21 @@ def test_get_redis_connection_invalid_pass_file(self, redis_config: "fixture"): os.chmod(CONFIG.broker.password, orig_file_permissions) - def test_get_redis_connection_dont_include_password(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_dont_include_password(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function without including the password. This should place 6 *s where the password would normally be placed in spass. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) - def test_get_redis_connection_use_ssl(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "rediss", @@ -458,24 +458,24 @@ def test_get_redis_connection_use_ssl(self, redis_config: "fixture"): # noqa: F } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) - def test_get_redis_connection_no_password(self, redis_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_password(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the password setting from the CONFIG object. This should still run and give us spass = ''. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.password expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_connection_string_redis(self, redis_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis as the broker (this is what our CONFIG - is set to by default with the redis_config fixture). + is set to by default with the redis_broker_config fixture). - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -488,11 +488,11 @@ def test_get_connection_string_redis(self, redis_config: "fixture"): # noqa: F8 actual = get_connection_string() assert expected == actual - def test_get_connection_string_rediss(self, redis_config: "fixture"): # noqa: F821 + def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rediss (with two 's') as the broker. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected_vals = { @@ -506,11 +506,11 @@ def test_get_connection_string_rediss(self, redis_config: "fixture"): # noqa: F actual = get_connection_string() assert expected == actual - def test_get_connection_string_redis_socket(self, redis_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis+socket as the broker. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Change our broker CONFIG.broker.name = "redis+socket" @@ -527,21 +527,21 @@ def test_get_connection_string_redis_socket(self, redis_config: "fixture"): # n actual = get_connection_string() assert actual == expected - def test_get_ssl_config_redis(self, redis_config: "fixture"): # noqa: F821 + def test_get_ssl_config_redis(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). This should return False. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert not get_ssl_config() - def test_get_ssl_config_rediss(self, redis_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss(self, redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss (with two 's') as the broker. This should return a dict of cert reqs with ssl.CERT_NONE as the value. - :param redis_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected = {"ssl_cert_reqs": CERT_NONE} From 17be237c9acb6395b2f4472d67ece6d3072c44b6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 12:55:05 -0800 Subject: [PATCH 060/201] add tests for results_backend.py --- merlin/config/results_backend.py | 6 + tests/conftest.py | 21 + tests/unit/config/test_results_backend.py | 570 ++++++++++++++++++++++ 3 files changed, 597 insertions(+) create mode 100644 tests/unit/config/test_results_backend.py diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 259e249a6..893c52f04 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -236,6 +236,12 @@ def get_mysql(certs_path=None, mysql_certs=None, include_password=True): mysql_config["password"] = "******" mysql_config["server"] = server + # Ensure the ssl_key, ssl_ca, and ssl_cert keys are all set + if mysql_certs == MYSQL_CONFIG_FILENAMES: + for key, cert_file in mysql_certs.items(): + if key not in mysql_config: + mysql_config[key] = os.path.join(certs_path, cert_file) + return MYSQL_CONNECTION_STRING.format(**mysql_config) diff --git a/tests/conftest.py b/tests/conftest.py index 1c996958d..9cbe1ec69 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,6 +48,11 @@ SERVER_PASS = "merlin-test-server" +CERT_FILES = { + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem", +} ####################################### @@ -101,6 +106,20 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi yaml.dump(app_yaml, app_yaml_file) +def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): + """ + Check if cert files already exist and if they don't then create them. + + :param cert_filepath: The path to the cert files + :param cert_files: A dict of certification files to create + """ + for cert_file in cert_files.values(): + full_cert_filepath = f"{cert_filepath}/{cert_file}" + if not os.path.exists(full_cert_filepath): + with open(full_cert_filepath, "w"): + pass + + def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): """ Given configuration options for the broker and results_backend, update @@ -391,6 +410,8 @@ def mysql_results_backend_config( pass_file = f"{merlin_server_dir}/mysql.pass" create_pass_file(pass_file) + create_cert_files(merlin_server_dir, CERT_FILES) + CONFIG.results_backend.password = pass_file CONFIG.results_backend.name = "mysql" CONFIG.results_backend.dbname = "test_mysql_db" diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py new file mode 100644 index 000000000..3531a83a2 --- /dev/null +++ b/tests/unit/config/test_results_backend.py @@ -0,0 +1,570 @@ +""" +Tests for the `results_backend.py` file. +""" +import os +import pytest +from ssl import CERT_NONE +from typing import Any, Dict + +from merlin.config.configfile import CONFIG +from merlin.config.results_backend import ( + MYSQL_CONFIG_FILENAMES, + MYSQL_CONNECTION_STRING, + SQLITE_CONNECTION_STRING, + get_backend_password, + get_connection_string, + get_mysql, + get_mysql_config, + get_redis, + get_ssl_config +) +from tests.conftest import CERT_FILES, SERVER_PASS, create_cert_files, create_pass_file + +RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" + + +def test_get_backend_password_pass_file_in_merlin(): + """ + Test the `get_backend_password` function with the password file in the ~/.merlin/ + directory. We'll create a dummy file in this directory and delete it once the test + is done. + """ + + # Check if the .merlin directory exists and create it if it doesn't + remove_merlin_dir_after_test = False + path_to_merlin_dir = os.path.expanduser("~/.merlin") + if not os.path.exists(path_to_merlin_dir): + remove_merlin_dir_after_test = True + os.mkdir(path_to_merlin_dir) + + # Create the test password file + pass_filename = "test.pass" + full_pass_filepath = f"{path_to_merlin_dir}/{pass_filename}" + create_pass_file(full_pass_filepath) + + try: + # Run the test + assert get_backend_password(pass_filename) == SERVER_PASS + # Cleanup + os.remove(full_pass_filepath) + if remove_merlin_dir_after_test: + os.rmdir(path_to_merlin_dir) + except AssertionError as exc: + # If the test fails, make sure we clean up the files/dirs created + os.remove(full_pass_filepath) + if remove_merlin_dir_after_test: + os.rmdir(path_to_merlin_dir) + raise AssertionError from exc + + +def test_get_backend_password_pass_file_not_in_merlin(temp_output_dir: str): + """ + Test the `get_backend_password` function with the password file not in the ~/.merlin/ + directory. By using the `temp_output_dir` fixture, our cwd will be the temporary directory. + We'll create a password file in the this directory for this test and have `get_backend_password` + read from that. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + pass_file = "test.pass" + create_pass_file(pass_file) + + assert get_backend_password(pass_file) == SERVER_PASS + + +def test_get_backend_password_directly_pass_password(): + """ + Test the `get_backend_password` function by passing the password directly to this + function instead of a password file. + """ + assert get_backend_password(SERVER_PASS) == SERVER_PASS + + +def test_get_backend_password_using_certs_path(temp_output_dir: str): + """ + Test the `get_backend_password` function with certs_path set to our temporary testing path. + We'll create a password file in the temporary directory for this test and have `get_backend_password` + read from that. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + pass_filename = "test_certs.pass" + test_dir = RESULTS_BACKEND_DIR.format(temp_output_dir=temp_output_dir) + if not os.path.exists(test_dir): + os.mkdir(test_dir) + full_pass_filepath = f"{test_dir}/{pass_filename}" + create_pass_file(full_pass_filepath) + + assert get_backend_password(pass_filename, certs_path=test_dir) == SERVER_PASS + + +def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with no results_backend set. This should return False. + NOTE: we're using the config fixture here to make sure values are reset after this test finishes. + We won't actually use anything from the config fixture. + + :param config: A fixture to set up the CONFIG object for us + """ + del CONFIG.results_backend.name + assert get_ssl_config() is False + + +def test_get_connection_string_no_results_backend(config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with no results_backend set. + This should raise a ValueError. + NOTE: we're using the config fixture here to make sure values are reset after this test finishes. + We won't actually use anything from the config fixture. + + :param config: A fixture to set up the CONFIG object for us + """ + del CONFIG.results_backend.name + with pytest.raises(ValueError) as excinfo: + get_connection_string() + + assert "'' is not a supported results backend" in str(excinfo.value) + + +class TestRedisResultsBackend: + """ + This class will house all tests necessary for our results_backend module when using a + redis results_backend. + """ + + def run_get_redis( + self, + expected_vals: Dict[str, Any], + certs_path: str = None, + include_password: bool = True, + ssl: bool = False, + ): + """ + Helper method for running tests for the `get_redis` function. + + :param expected_vals: A dict of expected values for this test. Format: + {"urlbase": "redis", + "spass": "", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0} + :param certs_path: A string denoting the path to the certification files + :param include_password: If True, include the password in the output. Otherwise don't. + :param ssl: If True, use ssl. Otherwise, don't. + """ + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_redis(certs_path=certs_path, include_password=include_password, ssl=ssl) + assert actual == expected + + def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with default functionality. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_dont_include_password(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with the password hidden. This should * out the password. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:******@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=False, ssl=False) + + def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with ssl enabled. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "rediss", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=True) + + def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no port in our CONFIG object. This should default to port=6379. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.port + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no db_num in our CONFIG object. This should default to db_num=0. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.db_num + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no username in our CONFIG object. This should default to username=''. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.username + expected_vals = { + "urlbase": "redis", + "spass": f":{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no password filepath in our CONFIG object. This should default to spass=''. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.password + expected_vals = { + "urlbase": "redis", + "spass": "", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function. We'll run this after changing the permissions of the password file so it + can't be opened. This should still run and give us password=CONFIG.results_backend.password. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + + # Capture the initial permissions of the password file so we can reset them + orig_file_permissions = os.stat(CONFIG.results_backend.password).st_mode + + # Change the permissions of the password file so it can't be read + os.chmod(CONFIG.results_backend.password, 0o222) + + try: + # Run the test + expected_vals = { + "urlbase": "redis", + "spass": f"default:{CONFIG.results_backend.password}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + os.chmod(CONFIG.results_backend.password, orig_file_permissions) + except AssertionError as exc: + # If this test failed, make sure to reset the permissions in case other tests need to read this file + os.chmod(CONFIG.results_backend.password, orig_file_permissions) + raise AssertionError from exc + + def test_get_ssl_config_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with redis as the results_backend. This should return False since + ssl requires using rediss (with two 's'). + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config() is False + + def test_get_ssl_config_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss as the results_backend. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "rediss" + assert get_ssl_config() == {"ssl_cert_reqs": CERT_NONE} + + def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss as the results_backend and no cert_reqs set. + This should return True. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.cert_reqs + CONFIG.results_backend.name = "rediss" + assert get_ssl_config() is True + + def test_get_connection_string_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis as the results_backend. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_connection_string_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rediss as the results_backend. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "rediss" + expected_vals = { + "urlbase": "rediss", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + +class TestMySQLResultsBackend: + """ + This class will house all tests necessary for our results_backend module when using a + MySQL results_backend. + NOTE: You'll notice a lot of these tests are setting CONFIG.results_backend.name to be + "invalid". This is so that we can get by the first if statement in the `get_mysql_config` + function. + """ + + def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql_config` function with the certs dict getting set and returned. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected = {} + for key, cert_file in CERT_FILES.items(): + expected[key] = f"{merlin_server_dir}/{cert_file}" + actual = get_mysql_config(merlin_server_dir, CERT_FILES) + assert actual == expected + + def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql_config` function with mysql_ssl being found. This should just return the ssl value that's found. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_mysql_config(None, None) == {"cert_reqs": CERT_NONE} + + def test_get_mysql_config_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql_config` function with no mysql certs dict. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + assert get_mysql_config(merlin_server_dir, {}) == {} + + def test_get_mysql_config_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql_config` function with an invalid certs path. This should return False. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "invalid" + assert get_mysql_config("invalid/path", CERT_FILES) is False + + def run_get_mysql(self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool): + """ + Helper method for running tests for the `get_mysql` function. + + :param expected_vals: A dict of expected values for this test. Format: + {"cert_reqs": cert reqs dict, + "user": "default", + "password": "", + "server": "127.0.0.1", + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem"} + :param certs_path: A string denoting the path to the certification files + :param mysql_certs: A dict of cert files + :param include_password: If True, include the password in the output. Otherwise don't. + """ + expected = MYSQL_CONNECTION_STRING.format(**expected_vals) + actual = get_mysql(certs_path=certs_path, mysql_certs=mysql_certs, include_password=include_password) + assert actual == expected + + def test_get_mysql(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function with default behavior. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + for key, cert_file in CERT_FILES.items(): + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True) + + def test_get_mysql_dont_include_password(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function but set include_password to False. This should * out the password. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": "******", + "server": "127.0.0.1", + } + for key, cert_file in CERT_FILES.items(): + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False) + + def test_get_mysql_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function with no mysql_certs passed in. This should use default config filenames so we'll + have to create these default files. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + + create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + + for key, cert_file in MYSQL_CONFIG_FILENAMES.items(): + # Password file is already is already set in expected_vals dict + if key == "password": + continue + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=None, include_password=True) + + def test_get_mysql_no_server(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql` function with no server set. This should raise a TypeError. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.server = False + with pytest.raises(TypeError) as excinfo: + get_mysql() + assert f"Results backend: server False does not have a configuration" in str(excinfo.value) + + def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql` function with an invalid certs_path. This should raise a TypeError. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "invalid" + with pytest.raises(TypeError) as excinfo: + get_mysql(certs_path="invalid_path", mysql_certs=CERT_FILES) + err_msg = f"""The connection information for MySQL could not be set, cannot find:\n + {CERT_FILES}\ncheck the celery/certs path or set the ssl information in the app.yaml file.""" + assert err_msg in str(excinfo.value) + + def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with mysql as the results_backend. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config() == {"cert_reqs": CERT_NONE} + + def test_get_ssl_config_mysql_celery_check(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with mysql as the results_backend and celery_check set. + This should return False. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config(celery_check=True) is False + + def test_get_connection_string_mysql(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_connection_string` function with MySQL as the results_backend. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.celery.certs = merlin_server_dir + + create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + for key, cert_file in MYSQL_CONFIG_FILENAMES.items(): + # Password file is already is already set in expected_vals dict + if key == "password": + continue + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + + assert MYSQL_CONNECTION_STRING.format(**expected_vals) == get_connection_string(include_password=True) + + def test_get_connection_string_sqlite(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with sqlite as the results_backend. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "sqlite" + assert get_connection_string() == SQLITE_CONNECTION_STRING From d54f75023c3c5cb120be0b1aab9afc61a0b13414 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 13:09:17 -0800 Subject: [PATCH 061/201] fix lint issues for most recent changes --- tests/conftest.py | 18 ++++++++---- tests/unit/common/test_encryption.py | 4 ++- tests/unit/config/test_results_backend.py | 36 +++++++++++++++-------- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9cbe1ec69..5a8a567b2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,7 +110,7 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): """ Check if cert files already exist and if they don't then create them. - :param cert_filepath: The path to the cert files + :param cert_filepath: The path to the cert files :param cert_files: A dict of certification files to create """ for cert_file in cert_files.values(): @@ -318,9 +318,13 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab CONFIG.broker.cert_reqs = "none" # Set the results_backend configuration for testing - CONFIG.results_backend.password = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.password = ( + None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + ) CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` - CONFIG.results_backend.name = None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + CONFIG.results_backend.name = ( + None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + ) CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` CONFIG.results_backend.server = "127.0.0.1" CONFIG.results_backend.username = "default" @@ -338,7 +342,9 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disab @pytest.fixture(scope="function") -def redis_broker_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +def redis_broker_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis broker and results_backend. @@ -357,7 +363,9 @@ def redis_broker_config(merlin_server_dir: str, config: "fixture"): # noqa: F82 @pytest.fixture(scope="function") -def redis_results_backend_config(merlin_server_dir: str, config: "fixture"): # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +def redis_results_backend_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis results_backend. diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index d0069f09e..d797f68c0 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -87,7 +87,9 @@ def test_gen_key(self, temp_output_dir: str): key_gen_contents = key_gen_file.read() assert key_gen_contents != "" - def test_get_key(self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_key( + self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture" # noqa: F821 + ): """ Test the `_get_key` function. diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 3531a83a2..80ce05657 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -2,10 +2,11 @@ Tests for the `results_backend.py` file. """ import os -import pytest from ssl import CERT_NONE from typing import Any, Dict +import pytest + from merlin.config.configfile import CONFIG from merlin.config.results_backend import ( MYSQL_CONFIG_FILENAMES, @@ -16,10 +17,11 @@ get_mysql, get_mysql_config, get_redis, - get_ssl_config + get_ssl_config, ) from tests.conftest import CERT_FILES, SERVER_PASS, create_cert_files, create_pass_file + RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" @@ -36,7 +38,7 @@ def test_get_backend_password_pass_file_in_merlin(): if not os.path.exists(path_to_merlin_dir): remove_merlin_dir_after_test = True os.mkdir(path_to_merlin_dir) - + # Create the test password file pass_filename = "test.pass" full_pass_filepath = f"{path_to_merlin_dir}/{pass_filename}" @@ -179,7 +181,7 @@ def test_get_redis_dont_include_password(self, redis_results_backend_config: "fi """ expected_vals = { "urlbase": "redis", - "spass": f"default:******@", + "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0, @@ -372,7 +374,7 @@ class TestMySQLResultsBackend: def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 """ - Test the `get_mysql_config` function with the certs dict getting set and returned. + Test the `get_mysql_config` function with the certs dict getting set and returned. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here :param merlin_server_dir: The directory that has the test certification files @@ -392,9 +394,11 @@ def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixtur """ assert get_mysql_config(None, None) == {"cert_reqs": CERT_NONE} - def test_get_mysql_config_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + def test_get_mysql_config_no_mysql_certs( + self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 + ): """ - Test the `get_mysql_config` function with no mysql certs dict. + Test the `get_mysql_config` function with no mysql certs dict. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here :param merlin_server_dir: The directory that has the test certification files @@ -411,7 +415,9 @@ def test_get_mysql_config_invalid_certs_path(self, mysql_results_backend_config: CONFIG.results_backend.name = "invalid" assert get_mysql_config("invalid/path", CERT_FILES) is False - def run_get_mysql(self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool): + def run_get_mysql( + self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool + ): """ Helper method for running tests for the `get_mysql` function. @@ -447,9 +453,13 @@ def test_get_mysql(self, mysql_results_backend_config: "fixture", merlin_server_ } for key, cert_file in CERT_FILES.items(): expected_vals[key] = f"{merlin_server_dir}/{cert_file}" - self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True) + self.run_get_mysql( + expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True + ) - def test_get_mysql_dont_include_password(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + def test_get_mysql_dont_include_password( + self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 + ): """ Test the `get_mysql` function but set include_password to False. This should * out the password. @@ -465,7 +475,9 @@ def test_get_mysql_dont_include_password(self, mysql_results_backend_config: "fi } for key, cert_file in CERT_FILES.items(): expected_vals[key] = f"{merlin_server_dir}/{cert_file}" - self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False) + self.run_get_mysql( + expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False + ) def test_get_mysql_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 """ @@ -502,7 +514,7 @@ def test_get_mysql_no_server(self, mysql_results_backend_config: "fixture"): # CONFIG.results_backend.server = False with pytest.raises(TypeError) as excinfo: get_mysql() - assert f"Results backend: server False does not have a configuration" in str(excinfo.value) + assert "Results backend: server False does not have a configuration" in str(excinfo.value) def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 """ From 8e9d1e2e5f121ddf18e3f1e1a79f845f8c8c2944 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 16:31:19 -0800 Subject: [PATCH 062/201] fix filename issue in setup.cfg and move celeryadapter tests to integration suite --- setup.cfg | 2 +- tests/{unit/study => integration}/test_celeryadapter.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{unit/study => integration}/test_celeryadapter.py (100%) diff --git a/setup.cfg b/setup.cfg index 0eaa116ea..6b4278799 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,5 +29,5 @@ ignore_missing_imports=true [coverage:run] omit = - merlin/ascii.py + merlin/ascii_art.py merlin/config/celeryconfig.py diff --git a/tests/unit/study/test_celeryadapter.py b/tests/integration/test_celeryadapter.py similarity index 100% rename from tests/unit/study/test_celeryadapter.py rename to tests/integration/test_celeryadapter.py From 6dae836dc8fd8c137c0584fe968d0f267b16ce5f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 18 Dec 2023 16:32:53 -0800 Subject: [PATCH 063/201] add ssl filepaths to mysql config object --- tests/conftest.py | 17 ++++++++++++++++- tests/unit/config/test_results_backend.py | 17 +++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5a8a567b2..4de1ae7f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,8 +31,9 @@ This module contains pytest fixtures to be used throughout the entire test suite. """ import os -from glob import glob +import shutil from copy import copy +from glob import glob from time import sleep from typing import Dict @@ -120,6 +121,17 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): pass +def create_app_yaml(app_yaml_filepath: str): + """ + Create a dummy app.yaml file at `app_yaml_filepath`. + + :param app_yaml_filepath: The location to create an app.yaml file at + """ + full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" + if not os.path.exists(full_app_yaml_filepath): + shutil.copy(f"{os.path.dirname(__file__)}/dummy_app.yaml", full_app_yaml_filepath) + + def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): """ Given configuration options for the broker and results_backend, update @@ -423,5 +435,8 @@ def mysql_results_backend_config( CONFIG.results_backend.password = pass_file CONFIG.results_backend.name = "mysql" CONFIG.results_backend.dbname = "test_mysql_db" + CONFIG.results_backend.keyfile = CERT_FILES["ssl_key"] + CONFIG.results_backend.certfile = CERT_FILES["ssl_cert"] + CONFIG.results_backend.ca_certs = CERT_FILES["ssl_ca"] yield diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 80ce05657..59e53a5ae 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -386,13 +386,16 @@ def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture actual = get_mysql_config(merlin_server_dir, CERT_FILES) assert actual == expected - def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture"): # noqa: F821 + def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 """ Test the `get_mysql_config` function with mysql_ssl being found. This should just return the ssl value that's found. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - assert get_mysql_config(None, None) == {"cert_reqs": CERT_NONE} + expected = {key: f"{temp_output_dir}/{cert_file}" for key, cert_file in CERT_FILES.items()} + expected["cert_reqs"] = CERT_NONE + assert get_mysql_config(None, None) == expected def test_get_mysql_config_no_mysql_certs( self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 @@ -529,14 +532,17 @@ def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixtu {CERT_FILES}\ncheck the celery/certs path or set the ssl information in the app.yaml file.""" assert err_msg in str(excinfo.value) - def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 """ Test the `get_ssl_config` function with mysql as the results_backend. This should return a dict of cert reqs with ssl.CERT_NONE as the value. :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - assert get_ssl_config() == {"cert_reqs": CERT_NONE} + expected = {key: f"{temp_output_dir}/{cert_file}" for key, cert_file in CERT_FILES.items()} + expected["cert_reqs"] = CERT_NONE + assert get_ssl_config() == expected def test_get_ssl_config_mysql_celery_check(self, mysql_results_backend_config: "fixture"): # noqa: F821 """ @@ -557,6 +563,9 @@ def test_get_connection_string_mysql(self, mysql_results_backend_config: "fixtur CONFIG.celery.certs = merlin_server_dir create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + CONFIG.results_backend.keyfile = MYSQL_CONFIG_FILENAMES["ssl_key"] + CONFIG.results_backend.certfile = MYSQL_CONFIG_FILENAMES["ssl_cert"] + CONFIG.results_backend.ca_certs = MYSQL_CONFIG_FILENAMES["ssl_ca"] expected_vals = { "cert_reqs": CERT_NONE, From 89caa21e1dbfcdf28d61f351b1c57404bd01da95 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 09:24:17 -0800 Subject: [PATCH 064/201] add unit tests for configfile.py --- tests/conftest.py | 12 - tests/unit/config/dummy_app.yaml | 33 + tests/unit/config/old_test_configfile.py | 97 --- tests/unit/config/old_test_results_backend.py | 67 -- tests/unit/config/test_configfile.py | 705 ++++++++++++++++++ 5 files changed, 738 insertions(+), 176 deletions(-) create mode 100644 tests/unit/config/dummy_app.yaml delete mode 100644 tests/unit/config/old_test_configfile.py delete mode 100644 tests/unit/config/old_test_results_backend.py create mode 100644 tests/unit/config/test_configfile.py diff --git a/tests/conftest.py b/tests/conftest.py index 4de1ae7f5..529b7cce3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,7 +31,6 @@ This module contains pytest fixtures to be used throughout the entire test suite. """ import os -import shutil from copy import copy from glob import glob from time import sleep @@ -121,17 +120,6 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): pass -def create_app_yaml(app_yaml_filepath: str): - """ - Create a dummy app.yaml file at `app_yaml_filepath`. - - :param app_yaml_filepath: The location to create an app.yaml file at - """ - full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" - if not os.path.exists(full_app_yaml_filepath): - shutil.copy(f"{os.path.dirname(__file__)}/dummy_app.yaml", full_app_yaml_filepath) - - def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): """ Given configuration options for the broker and results_backend, update diff --git a/tests/unit/config/dummy_app.yaml b/tests/unit/config/dummy_app.yaml new file mode 100644 index 000000000..966156566 --- /dev/null +++ b/tests/unit/config/dummy_app.yaml @@ -0,0 +1,33 @@ +broker: + cert_reqs: none + name: redis + password: redis.pass + port: '6379' + server: 127.0.0.1 + username: default + vhost: host4gunny +celery: + override: + visibility_timeout: 86400 +container: + config: redis.conf + config_dir: ./merlin_server/ + format: singularity + image: redis_latest.sif + image_type: redis + pass_file: redis.pass + pfile: merlin_server.pf + url: docker://redis + user_file: redis.users +process: + kill: kill {pid} + status: pgrep -P {pid} +results_backend: + cert_reqs: none + db_num: 0 + encryption_key: encrypt_data_key + name: redis + password: redis.pass + port: '6379' + server: 127.0.0.1 + username: default \ No newline at end of file diff --git a/tests/unit/config/old_test_configfile.py b/tests/unit/config/old_test_configfile.py deleted file mode 100644 index 39139ec11..000000000 --- a/tests/unit/config/old_test_configfile.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Tests for the configfile module.""" - -import os -import shutil -import tempfile -import unittest -from getpass import getuser - -from merlin.config import configfile - -from .utils import mkfile - - -CONFIG_FILE_CONTENTS = """ -celery: - certs: path/to/celery/config/files - -broker: - name: rabbitmq - username: testuser - password: rabbit.password # The filename that contains the password. - server: jackalope.llnl.gov - -results_backend: - name: mysql - dbname: testuser - username: mlsi - password: mysql.password # The filename that contains the password. - server: rabbit.llnl.gov - -""" - - -class TestFindConfigFile(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.appfile = mkfile(self.tmpdir, "app.yaml") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_tempdir(self): - self.assertTrue(os.path.isdir(self.tmpdir)) - - def test_find_config_file(self): - """ - Given the path to a vaild config file, find and return the full - filepath. - """ - path = configfile.find_config_file(path=self.tmpdir) - expected = os.path.join(self.tmpdir, self.appfile) - self.assertEqual(path, expected) - - def test_find_config_file_error(self): - """Given an invalid path, return None.""" - invalid = "invalid/path" - expected = None - - path = configfile.find_config_file(path=invalid) - self.assertEqual(path, expected) - - -class TestConfigFile(unittest.TestCase): - """Unit tests for loading the config file.""" - - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.configfile = mkfile(self.tmpdir, "app.yaml", content=CONFIG_FILE_CONTENTS) - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_get_config(self): - """ - Given the directory path to a valid merlin config file, then - `get_config` should find the merlin config file and load the YAML - contents to a dictionary. - """ - expected = { - "broker": { - "name": "rabbitmq", - "password": "rabbit.password", - "server": "jackalope.llnl.gov", - "username": "testuser", - "vhost": getuser(), - }, - "celery": {"certs": "path/to/celery/config/files"}, - "results_backend": { - "dbname": "testuser", - "name": "mysql", - "password": "mysql.password", - "server": "rabbit.llnl.gov", - "username": "mlsi", - }, - } - - self.assertDictEqual(configfile.get_config(self.tmpdir), expected) diff --git a/tests/unit/config/old_test_results_backend.py b/tests/unit/config/old_test_results_backend.py deleted file mode 100644 index 638f13eb8..000000000 --- a/tests/unit/config/old_test_results_backend.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Tests for the results_backend module.""" - -import os -import shutil -import tempfile -import unittest - -from merlin.config import results_backend - -from .utils import mkfile - - -class TestResultsBackend(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - - # Create test files. - self.tmpfile1 = mkfile(self.tmpdir, "mysql_test1.txt") - self.tmpfile2 = mkfile(self.tmpdir, "mysql_test2.txt") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_mysql_config(self): - """ - Given the path to a directory containing the MySQL cert files and a - dictionary of files to look for, then find and return the full path to - all the certs. - """ - certs = {"test1": "mysql_test1.txt", "test2": "mysql_test2.txt"} - - # This will just be the above dictionary with the full file paths. - expected = { - "test1": os.path.join(self.tmpdir, certs["test1"]), - "test2": os.path.join(self.tmpdir, certs["test2"]), - } - results = results_backend.get_mysql_config(self.tmpdir, certs) - self.assertDictEqual(results, expected) - - def test_mysql_config_no_files(self): - """ - Given the path to a directory containing the MySQL cert files and - an empty dictionary, then `get_mysql_config` should return an empty - dictionary. - """ - files = {} - result = results_backend.get_mysql_config(self.tmpdir, files) - self.assertEqual(result, {}) - - -class TestConfingMysqlErrorPath(unittest.TestCase): - """ - Test `get_mysql_config` against cases were the given path does not exist. - """ - - def test_mysql_config_false(self): - """ - Given a path that does not exist, then `get_mysql_config` should return - False. - """ - path = "invalid/path" - - # We don't need the dictionary populated for this test. The function - # should return False before trying to process the dictionary. - certs = {} - result = results_backend.get_mysql_config(path, certs) - self.assertFalse(result) diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py new file mode 100644 index 000000000..5d635e79b --- /dev/null +++ b/tests/unit/config/test_configfile.py @@ -0,0 +1,705 @@ +""" +Tests for the configfile.py module. +""" +import getpass +import os +import shutil +import ssl +from copy import copy, deepcopy + +import pytest +import yaml + +from merlin.config.configfile import ( + CONFIG, + default_config_info, + find_config_file, + get_cert_file, + get_config, + get_ssl_entries, + is_debug, + load_config, + load_default_celery, + load_default_user_names, + load_defaults, + merge_sslmap, + process_ssl_map, +) +from tests.conftest import CERT_FILES + + +CONFIGFILE_DIR = "{temp_output_dir}/test_configfile" +COPIED_APP_FILENAME = "app_copy.yaml" +DUMMY_APP_FILEPATH = f"{os.path.dirname(__file__)}/dummy_app.yaml" + + +def create_configfile_dir(temp_output_dir: str): + """ + Create the configfile dir if it doesn't exist yet. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + full_configfile_dirpath = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + if not os.path.exists(full_configfile_dirpath): + os.mkdir(full_configfile_dirpath) + + +def create_app_yaml(app_yaml_filepath: str): + """ + Create a dummy app.yaml file at `app_yaml_filepath`. + + :param app_yaml_filepath: The location to create an app.yaml file at + """ + full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" + if not os.path.exists(full_app_yaml_filepath): + shutil.copy(DUMMY_APP_FILEPATH, full_app_yaml_filepath) + + +def test_load_config(temp_output_dir: str): + """ + Test the `load_config` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: + expected = yaml.load(dummy_app_file, yaml.Loader) + + actual = load_config(f"{configfile_dir}/app.yaml") + assert actual == expected + + +def test_load_config_invalid_file(): + """ + Test the `load_config` function with an invalid filepath. + """ + assert load_config("invalid/filepath") is None + + +def test_find_config_file_valid_path(temp_output_dir: str): + """ + Test the `find_config_file` function with passing a valid path in. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + assert find_config_file(configfile_dir) == f"{configfile_dir}/app.yaml" + + +def test_find_config_file_invalid_path(): + """ + Test the `find_config_file` function with passing an invalid path in. + """ + assert find_config_file("invalid/path") is None + + +def test_find_config_file_local_path(temp_output_dir: str): + """ + Test the `find_config_file` function by having it find a local (in our cwd) app.yaml file. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + # Move into the configfile directory and run the test + os.chdir(configfile_dir) + try: + assert find_config_file() == f"{os.getcwd()}/app.yaml" + except AssertionError as exc: + # Move back to the temp output directory even if the test fails + os.chdir(temp_output_dir) + raise AssertionError from exc + + # Move back to the temp output directory + os.chdir(temp_output_dir) + + +def test_find_config_file_merlin_home_path(temp_output_dir: str): + """ + Test the `find_config_file` function by having it find an app.yaml file in our merlin directory. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + merlin_home = os.path.expanduser("~/.merlin") + if not os.path.exists(merlin_home): + os.mkdir(merlin_home) + create_app_yaml(merlin_home) + assert find_config_file() == f"{merlin_home}/app.yaml" + + +def check_for_and_move_app_yaml(dir_to_check: str) -> bool: + """ + Check for any app.yaml files in `dir_to_check`. If one is found, rename it. + Return True if an app.yaml was found, false otherwise. + + :param dir_to_check: The directory to search for an app.yaml in + :returns: True if an app.yaml was found. False otherwise. + """ + for filename in os.listdir(dir_to_check): + full_path = os.path.join(dir_to_check, filename) + if os.path.isfile(full_path) and filename == "app.yaml": + os.rename(full_path, f"{dir_to_check}/{COPIED_APP_FILENAME}") + return True + return False + + +def test_find_config_file_no_path(temp_output_dir: str): + """ + Test the `find_config_file` function by making it unable to find any app.yaml path. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Rename any app.yaml in the cwd + cwd_path = os.getcwd() + cwd_had_app_yaml = check_for_and_move_app_yaml(cwd_path) + + # Rename any app.yaml in the merlin home directory + merlin_home_dir = os.path.expanduser("~/.merlin") + merlin_home_had_app_yaml = check_for_and_move_app_yaml(merlin_home_dir) + + try: + assert find_config_file() is None + except AssertionError as exc: + # Reset the cwd app.yaml even if the test fails + if cwd_had_app_yaml: + os.rename(f"{cwd_path}/{COPIED_APP_FILENAME}", f"{cwd_path}/app.yaml") + + # Reset the merlin home app.yaml even if the test fails + if merlin_home_had_app_yaml: + os.rename(f"{merlin_home_dir}/{COPIED_APP_FILENAME}", f"{merlin_home_dir}/app.yaml") + + raise AssertionError from exc + + # Reset the cwd app.yaml + if cwd_had_app_yaml: + os.rename(f"{cwd_path}/{COPIED_APP_FILENAME}", f"{cwd_path}/app.yaml") + + # Reset the merlin home app.yaml + if merlin_home_had_app_yaml: + os.rename(f"{merlin_home_dir}/{COPIED_APP_FILENAME}", f"{merlin_home_dir}/app.yaml") + + +def test_load_default_user_names_nothing_to_load(): + """ + Test the `load_default_user_names` function with nothing to load. In other words, in this + test the config dict will have a username and vhost already set for the broker. We'll + create the dict then make a copy of it to test against after calling the function. + """ + actual_config = {"broker": {"username": "default", "vhost": "host4testing"}} + expected_config = deepcopy(actual_config) + assert actual_config is not expected_config + + load_default_user_names(actual_config) + + # Ensure that nothing was modified after our call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_user_names_no_username(): + """ + Test the `load_default_user_names` function with no username. In other words, in this + test the config dict will have vhost already set for the broker but not a username. + """ + expected_config = {"broker": {"username": getpass.getuser(), "vhost": "host4testing"}} + actual_config = {"broker": {"vhost": "host4testing"}} + load_default_user_names(actual_config) + + # Ensure that the username was set in the call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_user_names_no_vhost(): + """ + Test the `load_default_user_names` function with no vhost. In other words, in this + test the config dict will have username already set for the broker but not a vhost. + """ + expected_config = {"broker": {"username": "default", "vhost": getpass.getuser()}} + actual_config = {"broker": {"username": "default"}} + load_default_user_names(actual_config) + + # Ensure that the vhost was set in the call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_celery_nothing_to_load(): + """ + Test the `load_default_celery` function with nothing to load. In other words, in this + test the config dict will have a celery entry containing omit_queue_tag, queue_tag, and + override. We'll create the dict then make a copy of it to test against after calling + the function. + """ + actual_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + expected_config = deepcopy(actual_config) + assert actual_config is not expected_config + + load_default_celery(actual_config) + + # Ensure that nothing was modified after our call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_omit_queue_tag(): + """ + Test the `load_default_celery` function with no omit_queue_tag. The function should + create a default entry of False for this. + """ + actual_config = {"celery": {"queue_tag": "[merlin]_", "override": None}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the omit_queue_tag was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_queue_tag(): + """ + Test the `load_default_celery` function with no queue_tag. The function should + create a default entry of '[merlin]_' for this. + """ + actual_config = {"celery": {"omit_queue_tag": False, "override": None}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the queue_tag was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_override(): + """ + Test the `load_default_celery` function with no override. The function should + create a default entry of None for this. + """ + actual_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_"}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the override was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_celery_block(): + """ + Test the `load_default_celery` function with no celery block. The function should + create a default entry of + {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} for this. + """ + actual_config = {} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the celery block was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_defaults(): + """ + Test that the `load_defaults` function loads the user names and the celery block properly. + """ + actual_config = {"broker": {}} + expected_config = { + "broker": {"username": getpass.getuser(), "vhost": getpass.getuser()}, + "celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}, + } + load_defaults(actual_config) + + assert actual_config == expected_config + + +def test_get_config(temp_output_dir: str): + """ + Test the `get_config` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + + # Load up the contents of the dummy app.yaml file that we copied + with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: + expected = yaml.load(dummy_app_file, yaml.Loader) + + # Add in default settings that should be added + expected["celery"]["omit_queue_tag"] = False + expected["celery"]["queue_tag"] = "[merlin]_" + + actual = get_config(configfile_dir) + + assert actual == expected + + +def test_get_config_invalid_path(): + """ + Test the `get_config` function with an invalid path. This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + get_config("invalid/path") + + assert "Cannot find a merlin config file!" in str(excinfo.value) + + +def test_is_debug_no_merlin_debug(): + """ + Test the `is_debug` function without having MERLIN_DEBUG in the environment. + This should return False. + """ + + # Delete the current val of MERLIN_DEBUG and store it (if there is one) + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + del os.environ["MERLIN_DEBUG"] + reset_merlin_debug = True + + # Run the test + try: + assert is_debug() is False + except AssertionError as exc: + # Make sure to reset the value of MERLIN_DEBUG even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + raise AssertionError from exc + + # Reset the value of MERLIN_DEBUG + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + + +def test_is_debug_with_merlin_debug(): + """ + Test the `is_debug` function with having MERLIN_DEBUG in the environment. + This should return True. + """ + + # Grab the current value of MERLIN_DEBUG if there is one + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ and int(os.environ["MERLIN_DEBUG"]) != 1: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + reset_merlin_debug = True + + # Set the MERLIN_DEBUG value to be 1 + os.environ["MERLIN_DEBUG"] = "1" + + try: + assert is_debug() is True + except AssertionError as exc: + # Make sure to reset the value of MERLIN_DEBUG even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + raise AssertionError from exc + + # Reset the value of MERLIN_DEBUG + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + + +def test_default_config_info(temp_output_dir: str): + """ + Test the `default_config_info` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + create_configfile_dir(temp_output_dir) + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_app_yaml(configfile_dir) + cwd = os.getcwd() + os.chdir(configfile_dir) + + # Delete the current val of MERLIN_DEBUG and store it (if there is one) + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + del os.environ["MERLIN_DEBUG"] + reset_merlin_debug = True + + # Create the merlin home directory if it doesn't already exist + merlin_home = f"{os.path.expanduser('~')}/.merlin" + remove_merlin_home = False + if not os.path.exists(merlin_home): + os.mkdir(merlin_home) + remove_merlin_home = True + + # Run the test + try: + expected = { + "config_file": f"{configfile_dir}/app.yaml", + "is_debug": False, + "merlin_home": merlin_home, + "merlin_home_exists": True, + } + actual = default_config_info() + assert actual == expected + except AssertionError as exc: + # Make sure to reset values even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + if remove_merlin_home: + os.rmdir(merlin_home) + raise AssertionError from exc + + # Reset values if necessary + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + if remove_merlin_home: + os.rmdir(merlin_home) + + os.chdir(cwd) + + +def test_get_cert_file_all_valid_args(mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_cert_file` function with all valid arguments. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + expected = f"{merlin_server_dir}/{CERT_FILES['ssl_key']}" + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="keyfile", cert_path=merlin_server_dir + ) + assert actual == expected + + +def test_get_cert_file_invalid_cert_name(mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_cert_file` function with an invalid cert_name argument. This should just return None. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="invalid", cert_path=merlin_server_dir + ) + assert actual is None + + +def test_get_cert_file_nonexistent_cert_path( + mysql_results_backend_config: "fixture", temp_output_dir: str, merlin_server_dir: str # noqa: F821 +): + """ + Test the `get_cert_file` function with cert_path argument that doesn't exist. + This should still return the nonexistent path at the root of our temporary directory for testing. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + CONFIG.results_backend.certfile = "new_certfile.pem" + expected = f"{temp_output_dir}/new_certfile.pem" + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="certfile", cert_path=merlin_server_dir + ) + assert actual == expected + + +def test_get_ssl_entries_required_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll make + cert reqs be required. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "required" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_REQUIRED, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_optional_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll make + cert reqs be optional. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "optional" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_OPTIONAL, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_none_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we won't require + any cert reqs. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "none" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_NONE, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_omit_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll completely + omit the cert_reqs option + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + del CONFIG.results_backend.cert_reqs + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_REQUIRED, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_with_ssl_protocol(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll add in a + dummy ssl_protocol value that should get added to the dict that's output. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + protocol = "test_protocol" + CONFIG.results_backend.ssl_protocol = protocol + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_NONE, + "ssl_protocol": protocol, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_process_ssl_map_mysql(): + """Test the `process_ssl_map` function with mysql as the server name.""" + expected = {"keyfile": "ssl_key", "certfile": "ssl_cert", "ca_certs": "ssl_ca"} + actual = process_ssl_map("mysql") + assert actual == expected + + +def test_process_ssl_map_rediss(): + """Test the `process_ssl_map` function with rediss as the server name.""" + expected = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = process_ssl_map("rediss") + assert actual == expected + + +def test_merge_sslmap_all_keys_present(): + """ + Test the `merge_sslmap` function with all keys from server_ssl in ssl_map. + We'll assume we're using a rediss server for this. + """ + expected = { + "ssl_keyfile": "/path/to/keyfile", + "ssl_certfile": "/path/to/certfile", + "ssl_ca_certs": "/path/to/ca_file", + "ssl_cert_reqs": ssl.CERT_NONE, + } + test_server_ssl = { + "keyfile": "/path/to/keyfile", + "certfile": "/path/to/certfile", + "ca_certs": "/path/to/ca_file", + "cert_reqs": ssl.CERT_NONE, + } + test_ssl_map = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = merge_sslmap(test_server_ssl, test_ssl_map) + assert actual == expected + + +def test_merge_sslmap_some_keys_present(): + """ + Test the `merge_sslmap` function with some keys from server_ssl in ssl_map and others not. + We'll assume we're using a rediss server for this. + """ + expected = { + "ssl_keyfile": "/path/to/keyfile", + "ssl_certfile": "/path/to/certfile", + "ssl_ca_certs": "/path/to/ca_file", + "ssl_cert_reqs": ssl.CERT_NONE, + "new_key": "new_val", + "second_new_key": "second_new_val", + } + test_server_ssl = { + "keyfile": "/path/to/keyfile", + "certfile": "/path/to/certfile", + "ca_certs": "/path/to/ca_file", + "cert_reqs": ssl.CERT_NONE, + "new_key": "new_val", + "second_new_key": "second_new_val", + } + test_ssl_map = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = merge_sslmap(test_server_ssl, test_ssl_map) + assert actual == expected From ed3a6605e74f6edc0320a756ba5786e149b214e9 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 10:40:13 -0800 Subject: [PATCH 065/201] add tests for the utils.py file in config/ --- tests/unit/config/test_utils.py | 83 +++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/unit/config/test_utils.py diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py new file mode 100644 index 000000000..a02bc1ff1 --- /dev/null +++ b/tests/unit/config/test_utils.py @@ -0,0 +1,83 @@ +""" +Tests for the merlin/config/utils.py module. +""" + +import pytest + +from merlin.config.configfile import CONFIG +from merlin.config.utils import Priority, get_priority, is_rabbit_broker, is_redis_broker + + +def test_is_rabbit_broker(): + """Test the `is_rabbit_broker` by passing in rabbit as the broker""" + assert is_rabbit_broker("rabbitmq") is True + assert is_rabbit_broker("amqp") is True + assert is_rabbit_broker("amqps") is True + + +def test_is_rabbit_broker_invalid(): + """Test the `is_rabbit_broker` by passing in an invalid broker""" + assert is_rabbit_broker("redis") is False + assert is_rabbit_broker("") is False + + +def test_is_redis_broker(): + """Test the `is_redis_broker` by passing in redis as the broker""" + assert is_redis_broker("redis") is True + assert is_redis_broker("rediss") is True + assert is_redis_broker("redis+socket") is True + + +def test_is_redis_broker_invalid(): + """Test the `is_redis_broker` by passing in an invalid broker""" + assert is_redis_broker("rabbitmq") is False + assert is_redis_broker("") is False + + +def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with rabbit as the broker. + Low priority for rabbit is 1 and high is 10. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_priority(Priority.LOW) == 1 + assert get_priority(Priority.MID) == 5 + assert get_priority(Priority.HIGH) == 10 + + +def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with redis as the broker. + Low priority for redis is 10 and high is 1. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_priority(Priority.LOW) == 10 + assert get_priority(Priority.MID) == 5 + assert get_priority(Priority.HIGH) == 1 + + +def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with an invalid broker. + This should raise a ValueError. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "invalid" + with pytest.raises(ValueError) as excinfo: + get_priority(Priority.LOW) + assert "Function get_priority has reached unknown state! Maybe unsupported broker invalid?" in str(excinfo.value) + + +def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with an invalid priority. + This should raise a TypeError. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + with pytest.raises(TypeError) as excinfo: + get_priority("invalid_priority") + assert "Unrecognized priority 'invalid_priority'!" in str(excinfo.value) From 45dfa519c30f213f7d8d92ecb437ce3102b48579 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 13:10:02 -0800 Subject: [PATCH 066/201] create utilities file and constants file --- tests/conftest.py | 60 +---------------------- tests/constants.py | 10 ++++ tests/unit/config/test_broker.py | 3 +- tests/unit/config/test_results_backend.py | 3 +- tests/unit/config/utils.py | 24 --------- tests/utils.py | 35 +++++++++++++ 6 files changed, 51 insertions(+), 84 deletions(-) create mode 100644 tests/constants.py delete mode 100644 tests/unit/config/utils.py create mode 100644 tests/utils.py diff --git a/tests/conftest.py b/tests/conftest.py index 529b7cce3..90e69b387 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,16 +43,10 @@ from celery.canvas import Signature from merlin.config.configfile import CONFIG +from tests.constants import SERVER_PASS, CERT_FILES from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager - - -SERVER_PASS = "merlin-test-server" -CERT_FILES = { - "ssl_cert": "test-rabbit-client-cert.pem", - "ssl_ca": "test-mysql-ca-cert.pem", - "ssl_key": "test-rabbit-client-key.pem", -} +from tests.utils import create_cert_files, create_pass_file ####################################### @@ -68,18 +62,6 @@ ####################################### -def create_pass_file(pass_filepath: str): - """ - Check if a password file already exists (it will if the redis server has been started) - and if it hasn't then create one and write the password to the file. - - :param pass_filepath: The path to the password file that we need to check for/create - """ - if not os.path.exists(pass_filepath): - with open(pass_filepath, "w") as pass_file: - pass_file.write(SERVER_PASS) - - def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_filepath: str = None): """ Check if an encryption file already exists (it will if the redis server has been started) @@ -106,44 +88,6 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi yaml.dump(app_yaml, app_yaml_file) -def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): - """ - Check if cert files already exist and if they don't then create them. - - :param cert_filepath: The path to the cert files - :param cert_files: A dict of certification files to create - """ - for cert_file in cert_files.values(): - full_cert_filepath = f"{cert_filepath}/{cert_file}" - if not os.path.exists(full_cert_filepath): - with open(full_cert_filepath, "w"): - pass - - -def set_config(broker: Dict[str, str], results_backend: Dict[str, str]): - """ - Given configuration options for the broker and results_backend, update - the CONFIG object. - - :param broker: A dict of the configuration settings for the broker - :param results_backend: A dict of configuration settings for the results_backend - """ - # Set the broker configuration for testing - CONFIG.broker.password = broker["password"] - CONFIG.broker.port = broker["port"] - CONFIG.broker.server = broker["server"] - CONFIG.broker.username = broker["username"] - CONFIG.broker.vhost = broker["vhost"] - CONFIG.broker.name = broker["name"] - - # Set the results_backend configuration for testing - CONFIG.results_backend.password = results_backend["password"] - CONFIG.results_backend.port = results_backend["port"] - CONFIG.results_backend.server = results_backend["server"] - CONFIG.results_backend.username = results_backend["username"] - CONFIG.results_backend.encryption_key = results_backend["encryption_key"] - - ####################################### ######### Fixture Definitions ######### ####################################### diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 000000000..a2b354146 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,10 @@ +""" +This module will store constants that will be used throughout our test suite. +""" + +SERVER_PASS = "merlin-test-server" +CERT_FILES = { + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem", +} \ No newline at end of file diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 490b47649..8af1dda75 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -18,7 +18,8 @@ read_file, ) from merlin.config.configfile import CONFIG -from tests.conftest import SERVER_PASS, create_pass_file +from tests.constants import SERVER_PASS +from tests.utils import create_pass_file def test_read_file(merlin_server_dir: str): diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 59e53a5ae..314df6ce7 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -19,7 +19,8 @@ get_redis, get_ssl_config, ) -from tests.conftest import CERT_FILES, SERVER_PASS, create_cert_files, create_pass_file +from tests.constants import CERT_FILES, SERVER_PASS +from tests.utils import create_cert_files, create_pass_file RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" diff --git a/tests/unit/config/utils.py b/tests/unit/config/utils.py deleted file mode 100644 index 1765e8478..000000000 --- a/tests/unit/config/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Utils module for common test functionality. -""" - -import os - - -def mkfile(tmpdir, filename, content=""): - """ - A simple function for creating a file and returning the path. This is to - abstract out file creation logic in the tests. - - :param tmpdir: (str) The path to the temp directory. - :param filename: (str) The name of the file. - :param contents: (str) Optional contents to write to the file. Defaults to - an empty string. - :returns: (str) The appended path of the given tempdir and filename. - """ - filepath = os.path.join(tmpdir, filename) - - with open(filepath, "w") as f: - f.write(content) - - return filepath diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..51fbd56cc --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,35 @@ +""" +Utility functions for our test suite. +""" +import os +from typing import Dict + +from tests.constants import SERVER_PASS + + +def create_pass_file(pass_filepath: str): + """ + Check if a password file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the password to the file. + + :param pass_filepath: The path to the password file that we need to check for/create + """ + if not os.path.exists(pass_filepath): + with open(pass_filepath, "w") as pass_file: + pass_file.write(SERVER_PASS) + + +def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): + """ + Check if cert files already exist and if they don't then create them. + + :param cert_filepath: The path to the cert files + :param cert_files: A dict of certification files to create + """ + for cert_file in cert_files.values(): + full_cert_filepath = f"{cert_filepath}/{cert_file}" + if not os.path.exists(full_cert_filepath): + with open(full_cert_filepath, "w"): + pass + + From 598ffb7d38a417fc043ad2719d542b7e8065febb Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 16:42:23 -0800 Subject: [PATCH 067/201] move create_dir function to utils.py --- tests/unit/config/test_configfile.py | 24 +++++++----------------- tests/utils.py | 9 +++++++++ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py index 5d635e79b..aeb1da941 100644 --- a/tests/unit/config/test_configfile.py +++ b/tests/unit/config/test_configfile.py @@ -25,7 +25,8 @@ merge_sslmap, process_ssl_map, ) -from tests.conftest import CERT_FILES +from tests.constants import CERT_FILES +from tests.utils import create_dir CONFIGFILE_DIR = "{temp_output_dir}/test_configfile" @@ -33,17 +34,6 @@ DUMMY_APP_FILEPATH = f"{os.path.dirname(__file__)}/dummy_app.yaml" -def create_configfile_dir(temp_output_dir: str): - """ - Create the configfile dir if it doesn't exist yet. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - full_configfile_dirpath = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) - if not os.path.exists(full_configfile_dirpath): - os.mkdir(full_configfile_dirpath) - - def create_app_yaml(app_yaml_filepath: str): """ Create a dummy app.yaml file at `app_yaml_filepath`. @@ -61,8 +51,8 @@ def test_load_config(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: @@ -85,8 +75,8 @@ def test_find_config_file_valid_path(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) assert find_config_file(configfile_dir) == f"{configfile_dir}/app.yaml" @@ -109,8 +99,8 @@ def test_find_config_file_local_path(temp_output_dir: str): """ # Create the configfile directory and put an app.yaml file there - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) # Move into the configfile directory and run the test @@ -330,8 +320,8 @@ def test_get_config(temp_output_dir: str): """ # Create the configfile directory and put an app.yaml file there - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) # Load up the contents of the dummy app.yaml file that we copied @@ -422,8 +412,8 @@ def test_default_config_info(temp_output_dir: str): """ # Create the configfile directory and put an app.yaml file there - create_configfile_dir(temp_output_dir) configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) create_app_yaml(configfile_dir) cwd = os.getcwd() os.chdir(configfile_dir) diff --git a/tests/utils.py b/tests/utils.py index 51fbd56cc..3a75622b8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -33,3 +33,12 @@ def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): pass +def create_dir(dirpath: str): + """ + Check if `dirpath` exists and if it doesn't then create it. + + :param dirpath: The directory to create + """ + if not os.path.exists(dirpath): + os.mkdir(dirpath) + From a8d0d1d89f1015a439bf0ce380201d12a11e98eb Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 16:42:52 -0800 Subject: [PATCH 068/201] add tests for merlin/examples/generator.py --- merlin/examples/generator.py | 8 + setup.cfg | 1 + tests/unit/test_examples_generator.py | 575 ++++++++++++++++++++++++++ 3 files changed, 584 insertions(+) create mode 100644 tests/unit/test_examples_generator.py diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index a553d703b..4de205672 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -48,6 +48,14 @@ EXAMPLES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "workflows") +# TODO modify the example command to eliminate redundancy +# - e.g. running `merlin example flux_local` will produce the same output +# as running `merlin example flux_par` or `merlin example flux_par_restart`. +# This should just be `merlin example flux`. +# - restart and restart delay should be one example +# - feature demo and remote feature demo should be one example +# - all openfoam examples should just be under one openfoam label + def gather_example_dirs(): """Get all the example directories""" diff --git a/setup.cfg b/setup.cfg index 6b4278799..77ac2d84f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,3 +31,4 @@ ignore_missing_imports=true omit = merlin/ascii_art.py merlin/config/celeryconfig.py + merlin/examples/examples.py diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py new file mode 100644 index 000000000..7d7ccc5bf --- /dev/null +++ b/tests/unit/test_examples_generator.py @@ -0,0 +1,575 @@ +""" +Tests for the `merlin/examples/generator.py` module. +""" +import os +import pathlib +from typing import List + +from tabulate import tabulate + +from merlin.examples.generator import ( + EXAMPLES_DIR, + gather_all_examples, + gather_example_dirs, + list_examples, + setup_example, + write_example +) +from tests.utils import create_dir + + +EXAMPLES_GENERATOR_DIR = "{temp_output_dir}/examples_generator" + + +def test_gather_example_dirs(): + """Test the `gather_example_dirs` function.""" + example_workflows = [ + "feature_demo", + "flux", + "hello", + "hpc_demo", + "iterative_demo", + "lsf", + "null_spec", + "openfoam_wf", + "openfoam_wf_no_docker", + "openfoam_wf_singularity", + "optimization", + "remote_feature_demo", + "restart", + "restart_delay", + "simple_chain", + "slurm" + ] + expected = {} + for wf_dir in example_workflows: + expected[wf_dir] = wf_dir + actual = gather_example_dirs() + assert actual == expected + + +def test_gather_all_examples(): + """Test the `gather_all_examples` function.""" + expected = [ + f"{EXAMPLES_DIR}/feature_demo/feature_demo.yaml", + f"{EXAMPLES_DIR}/flux/flux_local.yaml", + f"{EXAMPLES_DIR}/flux/flux_par_restart.yaml", + f"{EXAMPLES_DIR}/flux/flux_par.yaml", + f"{EXAMPLES_DIR}/flux/paper.yaml", + f"{EXAMPLES_DIR}/hello/hello_samples.yaml", + f"{EXAMPLES_DIR}/hello/hello.yaml", + f"{EXAMPLES_DIR}/hello/my_hello.yaml", + f"{EXAMPLES_DIR}/hpc_demo/hpc_demo.yaml", + f"{EXAMPLES_DIR}/iterative_demo/iterative_demo.yaml", + f"{EXAMPLES_DIR}/lsf/lsf_par_srun.yaml", + f"{EXAMPLES_DIR}/lsf/lsf_par.yaml", + f"{EXAMPLES_DIR}/null_spec/null_chain.yaml", + f"{EXAMPLES_DIR}/null_spec/null_spec.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity.yaml", + f"{EXAMPLES_DIR}/optimization/optimization_basic.yaml", + f"{EXAMPLES_DIR}/remote_feature_demo/remote_feature_demo.yaml", + f"{EXAMPLES_DIR}/restart/restart.yaml", + f"{EXAMPLES_DIR}/restart_delay/restart_delay.yaml", + f"{EXAMPLES_DIR}/simple_chain/simple_chain.yaml", + f"{EXAMPLES_DIR}/slurm/slurm_par_restart.yaml", + f"{EXAMPLES_DIR}/slurm/slurm_par.yaml" + ] + actual = gather_all_examples() + assert sorted(actual) == sorted(expected) + + +def test_write_example_dir(temp_output_dir: str): + """ + Test the `write_example` function with the src_path as a directory. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + dir_to_copy = f"{EXAMPLES_DIR}/feature_demo/" + + write_example(dir_to_copy, generator_dir) + assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(generator_dir)) + + +def test_write_example_file(temp_output_dir: str): + """ + Test the `write_example` function with the src_path as a file. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + + dst_path = f"{generator_dir}/flux_par.yaml" + file_to_copy = f"{EXAMPLES_DIR}/flux/flux_par.yaml" + + write_example(file_to_copy, generator_dir) + assert os.path.exists(dst_path) + + +def test_list_examples(): + """Test the `list_examples` function to see if it gives us all of the examples that we want.""" + expected_headers = ["name", "description"] + expected_rows = [ + ["openfoam_wf_no_docker", "A parameter study that includes initializing, running,\n" \ + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" \ + "without using docker."], + ["optimization_basic", "Design Optimization Template\n" \ + "To use,\n" \ + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" \ + "2. Run the template_config file in current directory using `python template_config.py`\n" \ + "3. Merlin run as usual (merlin run optimization.yaml)\n" \ + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" \ + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts"], + ["feature_demo", "Run 10 hello worlds."], + ["flux_local", "Run a scan through Merlin/Maestro"], + ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], + ["flux_par_restart", "A simple ensemble of parallel MPI jobs run by flux."], + ["paper_flux", "Use flux to run single core MPI jobs and record timings."], + ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], + ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], + ["restart", "A simple ensemble of with restarts."], + ["restart_delay", "A simple ensemble of with restart delay times."], + ["simple_chain", "test to see that chains are not run in parallel"], + ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["remote_feature_demo", "Run 10 hello worlds."], + ["hello", "a very simple merlin workflow"], + ["hello_samples", "a very simple merlin workflow, with samples"], + ["hpc_demo", "Demo running a workflow on HPC machines"], + ["openfoam_wf", "A parameter study that includes initializing, running,\n" \ + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ + "using docker."], + ["openfoam_wf_singularity", "A parameter study that includes initializing, running,\n" \ + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ + "using singularity."], + ["null_chain", "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" \ + "May be used to measure overhead in merlin.\n" \ + "Iterates thru a chain of workflows."], + ["null_spec", "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin."], + ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ] + expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" + actual = list_examples() + assert actual == expected + + +def test_setup_example_invalid_name(): + """ + Test the `setup_example` function with an invalid example name. + This should just return None. + """ + assert setup_example("invalid_example_name", None) is None + + +def test_setup_example_no_outdir(temp_output_dir: str): + """ + Test the `setup_example` function with an invalid example name. + This should create a directory with the example name (in this case hello) + and copy all of the example contents to this folder. + We'll create a directory specifically for this test and move into it so that + the `setup_example` function creates the hello/ subdirectory in a directory with + the name of this test (setup_no_outdir). + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + cwd = os.getcwd() + + # Create the temp path to store this setup and move into that directory + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + setup_example_dir = os.path.join(generator_dir, "setup_no_outdir") + create_dir(setup_example_dir) + os.chdir(setup_example_dir) + + # This should still work and return to us the name of the example + try: + assert setup_example("hello", None) == "hello" + except AssertionError as exc: + os.chdir(cwd) + raise AssertionError from exc + + # All files from this example should be written to a directory with the example name + full_output_path = os.path.join(setup_example_dir, "hello") + expected_files = [ + os.path.join(full_output_path, "hello_samples.yaml"), + os.path.join(full_output_path, "hello.yaml"), + os.path.join(full_output_path, "my_hello.yaml"), + os.path.join(full_output_path, "requirements.txt"), + os.path.join(full_output_path, "make_samples.py"), + ] + try: + for file in expected_files: + assert os.path.exists(file) + except AssertionError as exc: + os.chdir(cwd) + raise AssertionError from exc + + +def test_setup_example_outdir_exists(temp_output_dir: str): + """ + Test the `setup_example` function with an output directory that already exists. + This should just return None. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + + assert setup_example("hello", generator_dir) is None + + +##################################### +# Tests for setting up each example # +##################################### + + +def run_setup_example(temp_output_dir: str, example_name: str, example_files: List[str], expected_return: str): + """ + Helper function to run tests for the `setup_example` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param example_name: The name of the example to setup + :param example_files: A list of filenames that should be copied by setup_example + :param expected_return: The expected return value from `setup_example` + """ + # Create the temp path to store this setup + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + setup_example_dir = os.path.join(generator_dir, f"setup_{example_name}") + + # Ensure that the example name is returned + actual = setup_example(example_name, setup_example_dir) + assert actual == expected_return + + # Ensure all of the files that should've been copied were copied + expected_files = [os.path.join(setup_example_dir, expected_file) for expected_file in example_files] + for file in expected_files: + assert os.path.exists(file) + + +def test_setup_example_feature_demo(temp_output_dir: str): + """ + Test the `setup_example` function for the feature_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "feature_demo" + example_files = [ + ".gitignore", + "feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_flux(temp_output_dir: str): + """ + Test the `setup_example` function for the flux example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_files = [ + "flux_local.yaml", + "flux_par_restart.yaml", + "flux_par.yaml", + "paper.yaml", + "requirements.txt", + "scripts/flux_info.py", + "scripts/hello_sleep.c", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/paper_workers.sbatch", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + "scripts/workers.bsub", + ] + + run_setup_example(temp_output_dir, "flux_local", example_files, "flux") + + +def test_setup_example_lsf(temp_output_dir: str): + """ + Test the `setup_example` function for the lsf example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # TODO should there be a workers.bsub for this example? + example_files = [ + "lsf_par_srun.yaml", + "lsf_par.yaml", + "scripts/hello.c", + "scripts/make_samples.py", + ] + + run_setup_example(temp_output_dir, "lsf_par", example_files, "lsf") + + +def test_setup_example_slurm(temp_output_dir: str): + """ + Test the `setup_example` function for the slurm example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_files = [ + "slurm_par.yaml", + "slurm_par_restart.yaml", + "requirements.txt", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + ] + + run_setup_example(temp_output_dir, "slurm_par", example_files, "slurm") + + +def test_setup_example_hello(temp_output_dir: str): + """ + Test the `setup_example` function for the hello example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "hello" + example_files = [ + "hello_samples.yaml", + "hello.yaml", + "my_hello.yaml", + "requirements.txt", + "make_samples.py", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_hpc(temp_output_dir: str): + """ + Test the `setup_example` function for the hpc_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "hpc_demo" + example_files = [ + "hpc_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_iterative(temp_output_dir: str): + """ + Test the `setup_example` function for the iterative_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "iterative_demo" + example_files = [ + "iterative_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_null(temp_output_dir: str): + """ + Test the `setup_example` function for the null_spec example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "null_spec" + example_files = [ + "null_spec.yaml", + "null_chain.yaml", + ".gitignore", + "Makefile", + "requirements.txt", + "scripts/aggregate_chain_output.sh", + "scripts/aggregate_output.sh", + "scripts/check_completion.sh", + "scripts/kill_all.sh", + "scripts/launch_chain_job.py", + "scripts/launch_jobs.py", + "scripts/make_samples.py", + "scripts/read_output_chain.py", + "scripts/read_output.py", + "scripts/search.sh", + "scripts/submit_chain.sbatch", + "scripts/submit.sbatch", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_openfoam(temp_output_dir: str): + """ + Test the `setup_example` function for the openfoam_wf example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "openfoam_wf" + example_files = [ + "openfoam_wf.yaml", + "openfoam_wf_template.yaml", + "README.md", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_openfoam_no_docker(temp_output_dir: str): + """ + Test the `setup_example` function for the openfoam_wf_no_docker example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "openfoam_wf_no_docker" + example_files = [ + "openfoam_wf_no_docker.yaml", + "openfoam_wf_no_docker_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_openfoam_singularity(temp_output_dir: str): + """ + Test the `setup_example` function for the openfoam_wf_singularity example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "openfoam_wf_singularity" + example_files = [ + "openfoam_wf_singularity.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_optimization(temp_output_dir: str): + """ + Test the `setup_example` function for the optimization example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_files = [ + "optimization_basic.yaml", + "requirements.txt", + "template_config.py", + "template_optimization.temp", + "scripts/collector.py", + "scripts/optimizer.py", + "scripts/test_functions.py", + "scripts/visualizer.py", + ] + + run_setup_example(temp_output_dir, "optimization_basic", example_files, "optimization") + + +def test_setup_example_remote_feature_demo(temp_output_dir: str): + """ + Test the `setup_example` function for the remote_feature_demo example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "remote_feature_demo" + example_files = [ + ".gitignore", + "remote_feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_restart(temp_output_dir: str): + """ + Test the `setup_example` function for the restart example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "restart" + example_files = [ + "restart.yaml", + "scripts/make_samples.py" + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_restart_delay(temp_output_dir: str): + """ + Test the `setup_example` function for the restart_delay example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + example_name = "restart_delay" + example_files = [ + "restart_delay.yaml", + "scripts/make_samples.py" + ] + + run_setup_example(temp_output_dir, example_name, example_files, example_name) + + +def test_setup_example_simple_chain(temp_output_dir: str): + """ + Test the `setup_example` function for the simple_chain example. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the temp path to store this setup + generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) + create_dir(generator_dir) + output_file = os.path.join(generator_dir, "simple_chain.yaml") + + # Ensure that the example name is returned + actual = setup_example("simple_chain", output_file) + assert actual == "simple_chain" + assert os.path.exists(output_file) From 77288937bed0d34db971e7cd59b034e1f125207e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 19 Dec 2023 16:47:28 -0800 Subject: [PATCH 069/201] run fix-style and update changelog --- CHANGELOG.md | 5 +- tests/conftest.py | 2 +- tests/constants.py | 3 +- tests/unit/test_examples_generator.py | 79 +++++++++++++++------------ tests/utils.py | 1 - 5 files changed, 51 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c325328c..87259fd39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,10 +98,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Coverage to the test suite. This includes adding tests for: - `merlin/common/` - `merlin/config/` + - `merlin/examples/` - `celeryadapter.py` - Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures - - RedisServerManager: context to help with starting/stopping a redis server for tests - - CeleryWorkersManager: context to help with starting/stopping workers for tests + - `RedisServerManager`: context to help with starting/stopping a redis server for tests + - `CeleryWorkersManager`: context to help with starting/stopping workers for tests - Ability to copy and print the `Config` object from `merlin/config/__init__.py` ### Changed diff --git a/tests/conftest.py b/tests/conftest.py index 90e69b387..a0f77bc9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,7 @@ from celery.canvas import Signature from merlin.config.configfile import CONFIG -from tests.constants import SERVER_PASS, CERT_FILES +from tests.constants import CERT_FILES, SERVER_PASS from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager from tests.utils import create_cert_files, create_pass_file diff --git a/tests/constants.py b/tests/constants.py index a2b354146..26cfe4c0a 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -3,8 +3,9 @@ """ SERVER_PASS = "merlin-test-server" + CERT_FILES = { "ssl_cert": "test-rabbit-client-cert.pem", "ssl_ca": "test-mysql-ca-cert.pem", "ssl_key": "test-rabbit-client-key.pem", -} \ No newline at end of file +} diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 7d7ccc5bf..97948feaf 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -2,7 +2,6 @@ Tests for the `merlin/examples/generator.py` module. """ import os -import pathlib from typing import List from tabulate import tabulate @@ -13,7 +12,7 @@ gather_example_dirs, list_examples, setup_example, - write_example + write_example, ) from tests.utils import create_dir @@ -39,7 +38,7 @@ def test_gather_example_dirs(): "restart", "restart_delay", "simple_chain", - "slurm" + "slurm", ] expected = {} for wf_dir in example_workflows: @@ -76,7 +75,7 @@ def test_gather_all_examples(): f"{EXAMPLES_DIR}/restart_delay/restart_delay.yaml", f"{EXAMPLES_DIR}/simple_chain/simple_chain.yaml", f"{EXAMPLES_DIR}/slurm/slurm_par_restart.yaml", - f"{EXAMPLES_DIR}/slurm/slurm_par.yaml" + f"{EXAMPLES_DIR}/slurm/slurm_par.yaml", ] actual = gather_all_examples() assert sorted(actual) == sorted(expected) @@ -85,7 +84,7 @@ def test_gather_all_examples(): def test_write_example_dir(temp_output_dir: str): """ Test the `write_example` function with the src_path as a directory. - + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) @@ -98,7 +97,7 @@ def test_write_example_dir(temp_output_dir: str): def test_write_example_file(temp_output_dir: str): """ Test the `write_example` function with the src_path as a file. - + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) @@ -115,16 +114,22 @@ def test_list_examples(): """Test the `list_examples` function to see if it gives us all of the examples that we want.""" expected_headers = ["name", "description"] expected_rows = [ - ["openfoam_wf_no_docker", "A parameter study that includes initializing, running,\n" \ - "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" \ - "without using docker."], - ["optimization_basic", "Design Optimization Template\n" \ - "To use,\n" \ - "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" \ - "2. Run the template_config file in current directory using `python template_config.py`\n" \ - "3. Merlin run as usual (merlin run optimization.yaml)\n" \ - "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" \ - "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts"], + [ + "openfoam_wf_no_docker", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" + "without using docker.", + ], + [ + "optimization_basic", + "Design Optimization Template\n" + "To use,\n" + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" + "2. Run the template_config file in current directory using `python template_config.py`\n" + "3. Merlin run as usual (merlin run optimization.yaml)\n" + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", + ], ["feature_demo", "Run 10 hello worlds."], ["flux_local", "Run a scan through Merlin/Maestro"], ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], @@ -141,16 +146,28 @@ def test_list_examples(): ["hello", "a very simple merlin workflow"], ["hello_samples", "a very simple merlin workflow, with samples"], ["hpc_demo", "Demo running a workflow on HPC machines"], - ["openfoam_wf", "A parameter study that includes initializing, running,\n" \ - "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ - "using docker."], - ["openfoam_wf_singularity", "A parameter study that includes initializing, running,\n" \ - "post-processing, collecting, learning and visualizing OpenFOAM runs\n" \ - "using singularity."], - ["null_chain", "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" \ - "May be used to measure overhead in merlin.\n" \ - "Iterates thru a chain of workflows."], - ["null_spec", "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin."], + [ + "openfoam_wf", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" + "using docker.", + ], + [ + "openfoam_wf_singularity", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" + "using singularity.", + ], + [ + "null_chain", + "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" + "May be used to measure overhead in merlin.\n" + "Iterates thru a chain of workflows.", + ], + [ + "null_spec", + "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + ], ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" @@ -534,10 +551,7 @@ def test_setup_example_restart(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ example_name = "restart" - example_files = [ - "restart.yaml", - "scripts/make_samples.py" - ] + example_files = ["restart.yaml", "scripts/make_samples.py"] run_setup_example(temp_output_dir, example_name, example_files, example_name) @@ -549,10 +563,7 @@ def test_setup_example_restart_delay(temp_output_dir: str): :param temp_output_dir: The path to the temporary output directory we'll be using for this test run """ example_name = "restart_delay" - example_files = [ - "restart_delay.yaml", - "scripts/make_samples.py" - ] + example_files = ["restart_delay.yaml", "scripts/make_samples.py"] run_setup_example(temp_output_dir, example_name, example_files, example_name) diff --git a/tests/utils.py b/tests/utils.py index 3a75622b8..d883b83cd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -41,4 +41,3 @@ def create_dir(dirpath: str): """ if not os.path.exists(dirpath): os.mkdir(dirpath) - From 6b97b0ab860adb0e04bfaaa51d2eb79dec99f065 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 25 Apr 2024 14:04:43 -0700 Subject: [PATCH 070/201] fix tests/bugs introduced by merging in develop --- merlin/config/utils.py | 10 ++++-- merlin/examples/generator.py | 1 + tests/unit/config/test_utils.py | 50 ++++++++++++++++++++++----- tests/unit/test_examples_generator.py | 6 ++-- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 46672ba1f..c37936d9e 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -77,8 +77,14 @@ def get_priority(priority: Priority) -> int: :param priority: The priority value that we want :returns: The priority value as an integer """ - if priority not in Priority: - raise ValueError(f"Invalid priority: {priority}") + priority_err_msg = f"Invalid priority: {priority}" + try: + # In python 3.12+ if something is not in the enum it will just return False + if priority not in Priority: + raise ValueError(priority_err_msg) + # In python 3.11 and below, a TypeError is raised when looking for something in an enum that is not there + except TypeError: + raise ValueError(priority_err_msg) priority_map = determine_priority_map(CONFIG.broker.name.lower()) return priority_map.get(priority, priority_map[Priority.MID]) # Default to MID priority for unknown priorities diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 4de205672..b1b89952d 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -146,4 +146,5 @@ def setup_example(name, outdir): LOG.info(f"Copying example '{name}' to {outdir}") write_example(src_path, outdir) + print(f'example: {example}') return example diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py index a02bc1ff1..9d64c10c7 100644 --- a/tests/unit/config/test_utils.py +++ b/tests/unit/config/test_utils.py @@ -5,7 +5,7 @@ import pytest from merlin.config.configfile import CONFIG -from merlin.config.utils import Priority, get_priority, is_rabbit_broker, is_redis_broker +from merlin.config.utils import Priority, determine_priority_map, get_priority, is_rabbit_broker, is_redis_broker def test_is_rabbit_broker(): @@ -37,25 +37,27 @@ def test_is_redis_broker_invalid(): def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F821 """ Test the `get_priority` function with rabbit as the broker. - Low priority for rabbit is 1 and high is 10. + Low priority for rabbit is 1 and high is 9. :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_priority(Priority.LOW) == 1 assert get_priority(Priority.MID) == 5 - assert get_priority(Priority.HIGH) == 10 + assert get_priority(Priority.HIGH) == 9 + assert get_priority(Priority.RETRY) == 10 def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 """ Test the `get_priority` function with redis as the broker. - Low priority for redis is 10 and high is 1. + Low priority for redis is 10 and high is 2. :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_priority(Priority.LOW) == 10 assert get_priority(Priority.MID) == 5 - assert get_priority(Priority.HIGH) == 1 + assert get_priority(Priority.HIGH) == 2 + assert get_priority(Priority.RETRY) == 1 def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 @@ -68,7 +70,7 @@ def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F CONFIG.broker.name = "invalid" with pytest.raises(ValueError) as excinfo: get_priority(Priority.LOW) - assert "Function get_priority has reached unknown state! Maybe unsupported broker invalid?" in str(excinfo.value) + assert "Unsupported broker name: invalid" in str(excinfo.value) def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 @@ -78,6 +80,38 @@ def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here """ - with pytest.raises(TypeError) as excinfo: + with pytest.raises(ValueError) as excinfo: get_priority("invalid_priority") - assert "Unrecognized priority 'invalid_priority'!" in str(excinfo.value) + assert "Invalid priority: invalid_priority" in str(excinfo.value) + + +def test_determine_priority_map_rabbit(): + """ + Test the `determine_priority_map` function with rabbit as the broker. + This should return the following map: + {Priority.LOW: 1, Priority.MID: 5, Priority.HIGH: 9, Priority.RETRY: 10} + """ + expected = {Priority.LOW: 1, Priority.MID: 5, Priority.HIGH: 9, Priority.RETRY: 10} + actual = determine_priority_map("rabbitmq") + assert actual == expected + + +def test_determine_priority_map_redis(): + """ + Test the `determine_priority_map` function with redis as the broker. + This should return the following map: + {Priority.LOW: 10, Priority.MID: 5, Priority.HIGH: 2, Priority.RETRY: 1} + """ + expected = {Priority.LOW: 10, Priority.MID: 5, Priority.HIGH: 2, Priority.RETRY: 1} + actual = determine_priority_map("redis") + assert actual == expected + + +def test_determine_priority_map_invalid(): + """ + Test the `determine_priority_map` function with an invalid broker. + This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + determine_priority_map("invalid_broker") + assert "Unsupported broker name: invalid_broker" in str(excinfo.value) diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 97948feaf..5a05e3599 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -64,11 +64,12 @@ def test_gather_all_examples(): f"{EXAMPLES_DIR}/lsf/lsf_par.yaml", f"{EXAMPLES_DIR}/null_spec/null_chain.yaml", f"{EXAMPLES_DIR}/null_spec/null_spec.yaml", - f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_docker_template.yaml", f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf.yaml", f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml", f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml", f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity_template.yaml", f"{EXAMPLES_DIR}/optimization/optimization_basic.yaml", f"{EXAMPLES_DIR}/remote_feature_demo/remote_feature_demo.yaml", f"{EXAMPLES_DIR}/restart/restart.yaml", @@ -445,7 +446,7 @@ def test_setup_example_openfoam(temp_output_dir: str): example_name = "openfoam_wf" example_files = [ "openfoam_wf.yaml", - "openfoam_wf_template.yaml", + "openfoam_wf_docker_template.yaml", "README.md", "requirements.txt", "scripts/make_samples.py", @@ -492,6 +493,7 @@ def test_setup_example_openfoam_singularity(temp_output_dir: str): example_name = "openfoam_wf_singularity" example_files = [ "openfoam_wf_singularity.yaml", + "openfoam_wf_singularity_template.yaml", "requirements.txt", "scripts/make_samples.py", "scripts/blockMesh_template.txt", From 57f0446a477cdb2489a6dccf463917ed946e069d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 25 Apr 2024 16:36:46 -0700 Subject: [PATCH 071/201] add a unit test file for the dumper module --- tests/unit/common/test_dumper.py | 156 +++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 tests/unit/common/test_dumper.py diff --git a/tests/unit/common/test_dumper.py b/tests/unit/common/test_dumper.py new file mode 100644 index 000000000..7c437fde9 --- /dev/null +++ b/tests/unit/common/test_dumper.py @@ -0,0 +1,156 @@ +""" +Tests for the `dumper.py` file. +""" +import csv +import json +import os +import pytest + +from datetime import datetime +from time import sleep + +from merlin.common.dumper import dump_handler + +NUM_ROWS = 5 +CSV_INFO_TO_DUMP = {"row_num": [i for i in range(1, NUM_ROWS+1)], "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS+1)]} +JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS+1)} +DUMP_HANDLER_DIR = "{temp_output_dir}/dump_handler" + +def test_dump_handler_invalid_dump_file(): + """ + This is really testing the initialization of the Dumper class with an invalid file type. + This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + dump_handler("bad_file.txt", CSV_INFO_TO_DUMP) + assert "Invalid file type for bad_file.txt. Supported file types are: ['csv', 'json']" in str(excinfo.value) + +def get_output_file(temp_dir: str, file_name: str): + """ + Helper function to get a full path to the temporary output file. + + :param temp_dir: The path to the temporary output directory that pytest gives us + :param file_name: The name of the file + """ + dump_dir = DUMP_HANDLER_DIR.format(temp_output_dir=temp_dir) + if not os.path.exists(dump_dir): + os.mkdir(dump_dir) + dump_file = f"{dump_dir}/{file_name}" + return dump_file + +def run_csv_dump_test(dump_file: str, fmode: str): + """ + Run the test for csv dump. + + :param dump_file: The file that the dump was written to + :param fmode: The type of write that we're testing ("w" for write, "a" for append) + """ + + # Check that the file exists and that read in the contents of the file + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + reader = csv.reader(df) + written_data = list(reader) + + expected_rows = NUM_ROWS*2 if fmode == "a" else NUM_ROWS + assert len(written_data) == expected_rows+1 # Adding one because of the header row + for i, row in enumerate(written_data): + assert len(row) == 2 # Check number of columns + if i == 0: # Checking the header row + assert row[0] == "row_num" + assert row[1] == "other_info" + else: # Checking the data rows + assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i%NUM_ROWS)-1]) + assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i%NUM_ROWS)-1]) + +def test_dump_handler_csv_write(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class. + This should create a csv file and write to it. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "csv_write.csv") + + # Run the actual call to dump to the file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Assert that everything ran properly + run_csv_dump_test(dump_file, "w") + +def test_dump_handler_csv_append(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class with the file write mode set to append. + We'll write to a csv file first and then run again to make sure we can append to it properly. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "csv_append.csv") + + # Run the first call to create the csv file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Run the second call to append to the csv file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Assert that everything ran properly + run_csv_dump_test(dump_file, "a") + +def test_dump_handler_json_write(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class. + This should create a json file and write to it. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "json_write.json") + + # Run the actual call to dump to the file + dump_handler(dump_file, JSON_INFO_TO_DUMP) + + # Check that the file exists and that the contents are correct + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + contents = json.load(df) + assert contents == JSON_INFO_TO_DUMP + +def test_dump_handler_json_append(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class with the file write mode set to append. + We'll write to a json file first and then run again to make sure we can append to it properly. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "json_append.json") + + # Run the first call to create the file + timestamp_1 = str(datetime.now()) + first_dump = {timestamp_1: JSON_INFO_TO_DUMP} + dump_handler(dump_file, first_dump) + + # Sleep so we don't accidentally get the same timestamp + sleep(.5) + + # Run the second call to append to the file + timestamp_2 = str(datetime.now()) + second_dump = {timestamp_2: JSON_INFO_TO_DUMP} + dump_handler(dump_file, second_dump) + + # Check that the file exists and that the contents are correct + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + contents = json.load(df) + keys = contents.keys() + assert len(keys) == 2 + assert timestamp_1 in keys + assert timestamp_2 in keys + assert contents[timestamp_1] == JSON_INFO_TO_DUMP + assert contents[timestamp_2] == JSON_INFO_TO_DUMP \ No newline at end of file From 963fed1e06ee14a0a75765d25a7b364eb5d4a8e9 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 7 May 2024 14:58:41 -0700 Subject: [PATCH 072/201] begin work on server tests and modular fixtures --- merlin/server/server_util.py | 39 +++- tests/fixtures/server.py | 84 ++++++++ tests/unit/server/__init__.py | 0 tests/unit/server/test_server_util.py | 295 ++++++++++++++++++++++++++ 4 files changed, 411 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/server.py create mode 100644 tests/unit/server/__init__.py create mode 100644 tests/unit/server/test_server_util.py diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index aa7c2765b..99cce51fe 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -60,7 +60,7 @@ def valid_ipv4(ip: str) -> bool: # pylint: disable=C0103 return False for i in arr: - if int(i) < 0 and int(i) > 255: + if int(i) < 0 or int(i) > 255: return False return True @@ -121,6 +121,15 @@ def __init__(self, data: dict) -> None: self.pass_file = data["pass_file"] if "pass_file" in data else self.PASSWORD_FILE self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE + def __eq__(self, other: "ContainerFormatConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ContainerFormatConfig object to check if they're the same + """ + variables = ("format", "image_type", "image", "url", "config", "config_dir", "pfile", "pass_file", "user_file") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_format(self) -> str: """Getter method to get the container format""" return self.format @@ -208,6 +217,15 @@ def __init__(self, data: dict) -> None: self.stop_command = data["stop_command"] if "stop_command" in data else self.STOP_COMMAND self.pull_command = data["pull_command"] if "pull_command" in data else self.PULL_COMMAND + def __eq__(self, other: "ContainerFormatConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ContainerFormatConfig object to check if they're the same + """ + variables = ("command", "run_command", "stop_command", "pull_command") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_command(self) -> str: """Getter method to get the container command""" return self.command @@ -242,6 +260,15 @@ def __init__(self, data: dict) -> None: self.status = data["status"] if "status" in data else self.STATUS_COMMAND self.kill = data["kill"] if "kill" in data else self.KILL_COMMAND + def __eq__(self, other: "ProcessConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ProcessConfig object to check if they're the same + """ + variables = ("status", "kill") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_status_command(self) -> str: """Getter method to get the status command""" return self.status @@ -264,12 +291,10 @@ class ServerConfig: # pylint: disable=R0903 container_format: ContainerFormatConfig = None def __init__(self, data: dict) -> None: - if "container" in data: - self.container = ContainerConfig(data["container"]) - if "process" in data: - self.process = ProcessConfig(data["process"]) - if self.container.get_format() in data: - self.container_format = ContainerFormatConfig(data[self.container.get_format()]) + self.container = ContainerConfig(data["container"]) if "container" in data else None + self.process = ProcessConfig(data["process"]) if "process" in data else None + container_format_data = data.get(self.container.get_format() if self.container else None) + self.container_format = ContainerFormatConfig(container_format_data) if container_format_data else None class RedisConfig: diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py new file mode 100644 index 000000000..35efdcd65 --- /dev/null +++ b/tests/fixtures/server.py @@ -0,0 +1,84 @@ +""" +Fixtures specifically for help testing the modules in the server/ directory. +""" +import pytest +import shutil +from typing import Dict + +@pytest.fixture(scope="class") +def server_container_config_data(temp_output_dir: str): + """ + Fixture to provide sample data for ContainerConfig tests + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + return { + "format": "docker", + "image_type": "postgres", + "image": "postgres:latest", + "url": "postgres://localhost", + "config": "postgres.conf", + "config_dir": "/path/to/config", + "pfile": "merlin_server_postgres.pf", + "pass_file": f"{temp_output_dir}/postgres.pass", + "user_file": "postgres.users", + } + +@pytest.fixture(scope="class") +def server_container_format_config_data(): + """ + Fixture to provide sample data for ContainerFormatConfig tests + """ + return { + "command": "docker", + "run_command": "{command} run --name {name} -d {image}", + "stop_command": "{command} stop {name}", + "pull_command": "{command} pull {url}", + } + +@pytest.fixture(scope="class") +def server_process_config_data(): + """ + Fixture to provide sample data for ProcessConfig tests + """ + return { + "status": "status {pid}", + "kill": "terminate {pid}", + } + +@pytest.fixture(scope="class") +def server_server_config( + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], + server_container_format_config_data: Dict[str, str], +): + """ + Fixture to provide sample data for ServerConfig tests + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + """ + return { + "container": server_container_config_data, + "process": server_process_config_data, + "docker": server_container_format_config_data, + } + + +@pytest.fixture(scope="class") +def server_redis_conf_file(temp_output_dir: str): + """ + Fixture to copy the redis.conf file from the merlin/server/ directory to the + temporary output directory and provide the path to the copied file + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + # TODO + # - will probably have to do more than just copying over the conf file + # - likely want to create our own test conf file with the settings that + # can be modified by RedisConf instead + path_to_redis_conf = f"{os.path.dirname(os.path.abspath(__file__))}/../../merlin/server/redis.conf" + path_to_copied_redis = f"{temp_output_dir}/redis.conf" + shutil.copy(path_to_redis_conf, path_to_copied_redis) + return path_to_copied_redis \ No newline at end of file diff --git a/tests/unit/server/__init__.py b/tests/unit/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py new file mode 100644 index 000000000..384e1ea37 --- /dev/null +++ b/tests/unit/server/test_server_util.py @@ -0,0 +1,295 @@ +""" +Tests for the `server_util.py` module. +""" +import os +import pytest +from typing import Callable, Dict, Union + +from merlin.server.server_util import ( + AppYaml, + ContainerConfig, + ContainerFormatConfig, + ProcessConfig, + RedisConfig, + RedisUsers, + ServerConfig, + valid_ipv4, + valid_port +) + +@pytest.mark.parametrize("valid_ip", [ + "0.0.0.0", + "127.0.0.1", + "14.105.200.58", + "255.255.255.255", +]) +def test_valid_ipv4_valid_ip(valid_ip: str): + """ + Test the `valid_ipv4` function with valid IPs. + This should return True. + + :param valid_ip: A valid port to test. + These are pulled from the parametrized list defined above this test. + """ + assert valid_ipv4(valid_ip) + +@pytest.mark.parametrize("invalid_ip", [ + "256.0.0.1", + "-1.0.0.1", + None, + "127.0.01", +]) +def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): + """ + Test the `valid_ipv4` function with invalid IPs. + An IP is valid if every integer separated by the '.' delimiter are between 0 and 255. + This should return False for both IPs tested here. + + :param invalid_ip: An invalid port to test. + These are pulled from the parametrized list defined above this test. + """ + assert not valid_ipv4(invalid_ip) + +@pytest.mark.parametrize("valid_input", [ + 1, + 433, + 65535, +]) +def test_valid_port_valid_input(valid_input: int): + """ + Test the `valid_port` function with valid port numbers. + Valid ports are ports between 1 and 65535. + This should return True. + + :param valid_input: A valid input value to test. + These are pulled from the parametrized list defined above this test. + """ + assert valid_port(valid_input) + +@pytest.mark.parametrize("invalid_input", [ + -1, + 0, + 65536, +]) +def test_valid_port_invalid_input(invalid_input: int): + """ + Test the `valid_port` function with invalid inputs. + Valid ports are ports between 1 and 65535. + This should return False for each invalid input tested. + + :param invalid_input: An invalid input value to test. + These are pulled from the parametrized list defined above this test. + """ + assert not valid_port(invalid_input) + + +class TestContainerConfig: + """Tests for the ContainerConfig class.""" + + def test_init_with_complete_data(self, server_container_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + config = ContainerConfig(server_container_config_data) + assert config.format == server_container_config_data["format"] + assert config.image_type == server_container_config_data["image_type"] + assert config.image == server_container_config_data["image"] + assert config.url == server_container_config_data["url"] + assert config.config == server_container_config_data["config"] + assert config.config_dir == server_container_config_data["config_dir"] + assert config.pfile == server_container_config_data["pfile"] + assert config.pass_file == server_container_config_data["pass_file"] + assert config.user_file == server_container_config_data["user_file"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data + """ + incomplete_data = {"format": "docker"} + config = ContainerConfig(incomplete_data) + assert config.format == incomplete_data["format"] + assert config.image_type == ContainerConfig.IMAGE_TYPE + assert config.image == ContainerConfig.IMAGE_NAME + assert config.url == ContainerConfig.REDIS_URL + assert config.config == ContainerConfig.CONFIG_FILE + assert config.config_dir == ContainerConfig.CONFIG_DIR + assert config.pfile == ContainerConfig.PROCESS_FILE + assert config.pass_file == ContainerConfig.PASSWORD_FILE + assert config.user_file == ContainerConfig.USERS_FILE + + @pytest.mark.parametrize("attr_name", [ + "image", + "config", + "pfile", + "pass_file", + "user_file", + ]) + def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): + """ + Tests that get_*_path methods construct the correct path + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param attr_name: Name of the attribute to be tested. These are pulled from the parametrized list defined above this test. + """ + config = ContainerConfig(server_container_config_data) + get_path_method = getattr(config, f"get_{attr_name}_path") # Dynamically get the method based on attr_name + expected_path = os.path.join(server_container_config_data["config_dir"], server_container_config_data[attr_name]) + assert get_path_method() == expected_path + + @pytest.mark.parametrize("getter_name, expected_attr", [ + ("get_format", "format"), + ("get_image_type", "image_type"), + ("get_image_name", "image"), + ("get_image_url", "url"), + ("get_config_name", "config"), + ("get_config_dir", "config_dir"), + ("get_pfile_name", "pfile"), + ("get_pass_file_name", "pass_file"), + ("get_user_file_name", "user_file"), + ]) + def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ContainerConfig(server_container_config_data) + getter = getattr(config, getter_name) + assert getter() == server_container_config_data[expected_attr] + + def test_get_container_password(self, server_container_config_data: Dict[str, str]): + """ + Test that the get_container_password is reading the password file properly + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + # Write a fake password to the password file + test_password = "super-secret-password" + with open(server_container_config_data["pass_file"], "w") as pass_file: + pass_file.write(test_password) + + # Run the test + config = ContainerConfig(server_container_config_data) + assert config.get_container_password() == test_password + + +class TestContainerFormatConfig: + """Tests for the ContainerFormatConfig class.""" + + def test_init_with_complete_data(self, server_container_format_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + """ + config = ContainerFormatConfig(server_container_format_config_data) + assert config.command == server_container_format_config_data["command"] + assert config.run_command == server_container_format_config_data["run_command"] + assert config.stop_command == server_container_format_config_data["stop_command"] + assert config.pull_command == server_container_format_config_data["pull_command"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data + """ + incomplete_data = {"command": "docker"} + config = ContainerFormatConfig(incomplete_data) + assert config.command == incomplete_data["command"] + assert config.run_command == config.RUN_COMMAND + assert config.stop_command == config.STOP_COMMAND + assert config.pull_command == config.PULL_COMMAND + + @pytest.mark.parametrize("getter_name, expected_attr", [ + ("get_command", "command"), + ("get_run_command", "run_command"), + ("get_stop_command", "stop_command"), + ("get_pull_command", "pull_command"), + ]) + def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values + + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ContainerFormatConfig(server_container_format_config_data) + getter = getattr(config, getter_name) + assert getter() == server_container_format_config_data[expected_attr] + + +class TestProcessConfig: + """Tests for the ProcessConfig class.""" + + def test_init_with_complete_data(self, server_process_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + """ + config = ProcessConfig(server_process_config_data) + assert config.status == server_process_config_data["status"] + assert config.kill == server_process_config_data["kill"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data + """ + incomplete_data = {"status": "status {pid}"} + config = ProcessConfig(incomplete_data) + assert config.status == incomplete_data["status"] + assert config.kill == config.KILL_COMMAND + + @pytest.mark.parametrize("getter_name, expected_attr", [ + ("get_status_command", "status"), + ("get_kill_command", "kill"), + ]) + def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values + + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ProcessConfig(server_process_config_data) + getter = getattr(config, getter_name) + assert getter() == server_process_config_data[expected_attr] + + +class TestServerConfig: + """Tests for the ServerConfig class.""" + + def test_init_with_complete_data(self, server_server_config: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data + + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + config = ServerConfig(server_server_config) + assert config.container == ContainerConfig(server_server_config["container"]) + assert config.process == ProcessConfig(server_server_config["process"]) + assert config.container_format == ContainerFormatConfig(server_server_config["docker"]) + + def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): + """ + Tests that __init__ uses None for missing data + + :param server_process_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + incomplete_data = {"process": server_process_config_data} + config = ServerConfig(incomplete_data) + assert config.process == ProcessConfig(server_process_config_data) + assert config.container is None + assert config.container_format is None + + +# class TestRedisConfig: +# """Tests for the RedisConfig class.""" + +# def test_parse(self, server_redis_conf_file): +# raise ValueError From 8649ca175e1fce5174419d9952d54bf972fd330e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 23 May 2024 08:00:48 -0700 Subject: [PATCH 073/201] start work on tests for RedisConfig --- tests/fixtures/server.py | 84 +++++++++++++++++++++------ tests/unit/server/test_server_util.py | 62 ++++++++++++++++++-- 2 files changed, 125 insertions(+), 21 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 35efdcd65..04c858f46 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -1,16 +1,17 @@ """ Fixtures specifically for help testing the modules in the server/ directory. """ +import os import pytest -import shutil from typing import Dict @pytest.fixture(scope="class") -def server_container_config_data(temp_output_dir: str): +def server_container_config_data(temp_output_dir: str) -> Dict[str, str]: """ Fixture to provide sample data for ContainerConfig tests :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: A dict containing the necessary key/values for the ContainerConfig object """ return { "format": "docker", @@ -25,9 +26,11 @@ def server_container_config_data(temp_output_dir: str): } @pytest.fixture(scope="class") -def server_container_format_config_data(): +def server_container_format_config_data() -> Dict[str, str]: """ Fixture to provide sample data for ContainerFormatConfig tests + + :returns: A dict containing the necessary key/values for the ContainerFormatConfig object """ return { "command": "docker", @@ -37,9 +40,11 @@ def server_container_format_config_data(): } @pytest.fixture(scope="class") -def server_process_config_data(): +def server_process_config_data() -> Dict[str, str]: """ Fixture to provide sample data for ProcessConfig tests + + :returns: A dict containing the necessary key/values for the ProcessConfig object """ return { "status": "status {pid}", @@ -51,13 +56,14 @@ def server_server_config( server_container_config_data: Dict[str, str], server_process_config_data: Dict[str, str], server_container_format_config_data: Dict[str, str], -): +) -> Dict[str, Dict[str, str]]: """ Fixture to provide sample data for ServerConfig tests :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :returns: A dictionary containing each of the configuration dicts we'll need """ return { "container": server_container_config_data, @@ -66,19 +72,63 @@ def server_server_config( } -@pytest.fixture(scope="class") -def server_redis_conf_file(temp_output_dir: str): +@pytest.fixture(scope="session") +def server_testing_dir(temp_output_dir: str) -> str: """ - Fixture to copy the redis.conf file from the merlin/server/ directory to the - temporary output directory and provide the path to the copied file + Fixture to create a temporary output directory for tests related to the server functionality. :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for server tests + """ + testing_dir = f"{temp_output_dir}/server_testing/" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir + + +@pytest.fixture(scope="session") +def server_redis_conf_file(server_testing_dir: str) -> str: + """ + Fixture to copy the redis.conf file from the merlin/server/ directory to the + temporary output directory and provide the path to the copied file. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :returns: The path to the redis configuration file we'll use for testing """ - # TODO - # - will probably have to do more than just copying over the conf file - # - likely want to create our own test conf file with the settings that - # can be modified by RedisConf instead - path_to_redis_conf = f"{os.path.dirname(os.path.abspath(__file__))}/../../merlin/server/redis.conf" - path_to_copied_redis = f"{temp_output_dir}/redis.conf" - shutil.copy(path_to_redis_conf, path_to_copied_redis) - return path_to_copied_redis \ No newline at end of file + redis_conf_file = f"{server_testing_dir}/redis.conf" + file_contents = """ + # ip address + bind 127.0.0.1 + + # port + port 6379 + + # password + requirepass merlin_password + + # directory + dir ./ + + # snapshot + save 300 100 + + # db file + dbfilename dump.rdb + + # append mode + appendfsync everysec + + # append file + appendfilename appendonly.aof + + # dummy trailing comment + """.strip().replace(" ", "") + + with open(redis_conf_file, "w") as rcf: + rcf.write(file_contents) + + return redis_conf_file diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 384e1ea37..c71e854eb 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,8 +1,10 @@ """ Tests for the `server_util.py` module. """ +import filecmp import os import pytest +import shutil from typing import Callable, Dict, Union from merlin.server.server_util import ( @@ -288,8 +290,60 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.container_format is None -# class TestRedisConfig: -# """Tests for the RedisConfig class.""" +class TestRedisConfig: + """Tests for the RedisConfig class.""" + + def test_initialization(self, server_redis_conf_file: str): + """ + Using a dummy redis configuration file, test that the initialization + of the RedisConfig class behaves as expected. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + expected_entries = { + "bind": "127.0.0.1", + "port": "6379", + "requirepass": "merlin_password", + "dir": "./", + "save": "300 100", + "dbfilename": "dump.rdb", + "appendfsync": "everysec", + "appendfilename": "appendonly.aof", + } + expected_comments = { + "bind": "# ip address\n", + "port": "\n# port\n", + "requirepass": "\n# password\n", + "dir": "\n# directory\n", + "save": "\n# snapshot\n", + "dbfilename": "\n# db file\n", + "appendfsync": "\n# append mode\n", + "appendfilename": "\n# append file\n", + } + expected_trailing_comment = "\n# dummy trailing comment" + expected_entry_order = list(expected_entries.keys()) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.filename == server_redis_conf_file + assert not redis_config.changed + assert redis_config.entries == expected_entries + assert redis_config.entry_order == expected_entry_order + assert redis_config.comments == expected_comments + assert redis_config.trailing_comments == expected_trailing_comment + + def test_write(self, server_redis_conf_file: str, server_testing_dir: str): + """ + """ + copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" + + # Create a RedisConf object with the basic redis conf file + redis_config = RedisConfig(server_redis_conf_file) + + # Change the filepath of the redis config file to be the copy that we'll write to + redis_config.filename = copy_redis_conf_file + + # Run the test + redis_config.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) -# def test_parse(self, server_redis_conf_file): -# raise ValueError From 91ad49b9b8da9c3d46e7d1f1eb17d08202a69a49 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 4 Jun 2024 15:07:05 -0700 Subject: [PATCH 074/201] add tests for RedisConfig object --- merlin/server/server_commands.py | 4 +- merlin/server/server_util.py | 79 ++-- tests/unit/server/test_RedisConfig.py | 538 ++++++++++++++++++++++++++ tests/unit/server/test_server_util.py | 64 +-- 4 files changed, 577 insertions(+), 108 deletions(-) create mode 100644 tests/unit/server/test_RedisConfig.py diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index be2b944a0..ef156f1ab 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -98,9 +98,7 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 redis_config.set_directory(args.directory) - redis_config.set_snapshot_seconds(args.snapshot_seconds) - - redis_config.set_snapshot_changes(args.snapshot_changes) + redis_config.set_snapshot(seconds=args.snapshot_seconds, changes=args.snapshot_changes) redis_config.set_snapshot_file(args.snapshot_file) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 99cce51fe..3a4cfc2f8 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -304,16 +304,14 @@ class RedisConfig: to write those changes into a redis readable config file. """ - filename = "" - entry_order = [] - entries = {} - comments = {} - trailing_comments = "" - changed = False - def __init__(self, filename) -> None: self.filename = filename self.changed = False + self.entry_order = [] + self.entries = {} + self.comments = {} + self.trailing_comments = "" + self.changed = False self.parse() def parse(self) -> None: @@ -393,7 +391,7 @@ def get_port(self) -> str: """Getter method to get the port from the redis config""" return self.get_config_value("port") - def set_port(self, port: str) -> bool: + def set_port(self, port: int) -> bool: """Validates and sets a given port""" if port is None: return False @@ -428,59 +426,56 @@ def set_directory(self, directory: str) -> bool: """ if directory is None: return False + # Create the directory if it doesn't exist if not os.path.exists(directory): os.mkdir(directory) LOG.info(f"Created directory {directory}") - # Validate the directory input - if os.path.exists(directory): - # Set the save directory to the redis config - if not self.set_config_value("dir", directory): - LOG.error("Unable to set directory for redis config") - return False - else: - LOG.error(f"Directory {directory} given does not exist and could not be created.") + # Set the save directory to the redis config + if not self.set_config_value("dir", directory): + LOG.error("Unable to set directory for redis config") return False LOG.info(f"Directory is set to {directory}") return True - def set_snapshot_seconds(self, seconds: int) -> bool: - """Sets the snapshot wait time""" - if seconds is None: - return False - # Set the snapshot second in the redis config - value = self.get_config_value("save") - if value is None: - LOG.error("Unable to get exisiting parameter values for snapshot") - return False + def set_snapshot(self, seconds: int = None, changes: int = None) -> bool: + """ + Sets the 'seconds' and/or 'changes' values of the snapshot setting, + depending on what the user requests. + + :param seconds: The first value of snapshot to change. If we're leaving it the + same this will be None. + :param changes: The second value of snapshot to change. If we're leaving it the + same this will be None. + :returns: True if successful, False otherwise. + """ - value = value.split() - value[0] = str(seconds) - value = " ".join(value) - if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") + # If both values are None, this method is doing nothing + if seconds is None and changes is None: return False - LOG.info(f"Snapshot wait time is set to {seconds} seconds") - return True - - def set_snapshot_changes(self, changes: int) -> bool: - """Sets the snapshot threshold""" - if changes is None: - return False - # Set the snapshot changes into the redis config + # Grab the snapshot value from the redis config value = self.get_config_value("save") if value is None: LOG.error("Unable to get exisiting parameter values for snapshot") return False + # Update the snapshot value value = value.split() - value[1] = str(changes) + log_msg = "" + if seconds is not None: + value[0] = str(seconds) + log_msg += f"Snapshot wait time is set to {seconds} seconds. " + if changes is not None: + value[1] = str(changes) + log_msg += f"Snapshot threshold is set to {changes} changes." value = " ".join(value) + + # Set the new snapshot value if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") + LOG.error("Unable to set snapshot value") return False - LOG.info(f"Snapshot threshold is set to {changes} changes") + LOG.info(log_msg) return True def set_snapshot_file(self, file: str) -> bool: @@ -508,7 +503,7 @@ def set_append_mode(self, mode: str) -> bool: LOG.error("Unable to set append_mode in redis config") return False else: - LOG.error("Not a valid append_mode(Only valid modes are always, everysec, no)") + LOG.error("Not a valid append_mode (Only valid modes are always, everysec, no)") return False LOG.info(f"Append mode is set to {mode}") diff --git a/tests/unit/server/test_RedisConfig.py b/tests/unit/server/test_RedisConfig.py new file mode 100644 index 000000000..12880d4d6 --- /dev/null +++ b/tests/unit/server/test_RedisConfig.py @@ -0,0 +1,538 @@ +""" +Tests for the RedisConfig class of the `server_util.py` module. + +This class is especially large so that's why these tests have been +moved to their own file. +""" +import filecmp +import logging +import pytest +from typing import Any + +from merlin.server.server_util import RedisConfig + +class TestRedisConfig: + """Tests for the RedisConfig class.""" + + def test_initialization(self, server_redis_conf_file: str): + """ + Using a dummy redis configuration file, test that the initialization + of the RedisConfig class behaves as expected. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + expected_entries = { + "bind": "127.0.0.1", + "port": "6379", + "requirepass": "merlin_password", + "dir": "./", + "save": "300 100", + "dbfilename": "dump.rdb", + "appendfsync": "everysec", + "appendfilename": "appendonly.aof", + } + expected_comments = { + "bind": "# ip address\n", + "port": "\n# port\n", + "requirepass": "\n# password\n", + "dir": "\n# directory\n", + "save": "\n# snapshot\n", + "dbfilename": "\n# db file\n", + "appendfsync": "\n# append mode\n", + "appendfilename": "\n# append file\n", + } + expected_trailing_comment = "\n# dummy trailing comment" + expected_entry_order = list(expected_entries.keys()) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.filename == server_redis_conf_file + assert not redis_config.changed + assert redis_config.entries == expected_entries + assert redis_config.entry_order == expected_entry_order + assert redis_config.comments == expected_comments + assert redis_config.trailing_comments == expected_trailing_comment + + def test_write(self, server_redis_conf_file: str, server_testing_dir: str): + """ + Test that the write functionality works by writing the contents of a dummy + configuration file to a blank configuration file. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" + + # Create a RedisConf object with the basic redis conf file + redis_config = RedisConfig(server_redis_conf_file) + + # Change the filepath of the redis config file to be the copy that we'll write to + redis_config.set_filename(copy_redis_conf_file) + + # Run the test + redis_config.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) + + @pytest.mark.parametrize("key, val, expected_return", [ + ("port", 1234, True), + ("invalid_key", "dummy_val", False) + ]) + def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, expected_return: bool): + """ + Test the `set_config_value` method with valid and invalid keys. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param key: The key value to modify with `set_config_value` + :param val: The value to set `key` to + :param expected_return: The expected return from `set_config_value` + """ + redis_config = RedisConfig(server_redis_conf_file) + actual_return = redis_config.set_config_value(key, val) + assert actual_return == expected_return + if expected_return: + assert redis_config.entries[key] == val + assert redis_config.changes_made() + else: + assert not redis_config.changes_made() + + @pytest.mark.parametrize("key, expected_val", [ + ("bind", "127.0.0.1"), + ("port", "6379"), + ("requirepass", "merlin_password"), + ("dir", "./"), + ("save", "300 100"), + ("dbfilename", "dump.rdb"), + ("appendfsync", "everysec"), + ("appendfilename", "appendonly.aof"), + ("invalid_key", None) + ]) + def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_val: str): + """ + Test the `get_config_value` method with valid and invalid keys. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param key: The key value to modify with `set_config_value` + :param expected_val: The value we're expecting to get by querying `key` + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.get_config_value(key) == expected_val + + @pytest.mark.parametrize("ip_to_set", [ + "127.0.0.1", # Most common IP + "0.0.0.0", # Edge case (low) + "255.255.255.255", # Edge case (high) + "123.222.199.20", # Random valid IP + ]) + def test_set_ip_address_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + ip_to_set: str + ): + """ + Test the `set_ip_address` method with valid ips. These should all return True + and set the 'bind' value to whatever `ip_to_set` is. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param ip_to_set: The ip address to set + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.set_ip_address(ip_to_set) + assert f"Ipaddress is set to {ip_to_set}" in caplog.text, "Missing expected log message" + assert redis_config.get_ip_address() == ip_to_set + + @pytest.mark.parametrize("ip_to_set, expected_log", [ + (None, None), # No IP + ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 + ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist + ]) + def test_set_ip_address_invalid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + ip_to_set: str, + expected_log: str, + ): + """ + Test the `set_ip_address` method with invalid ips. These should all return False. + and not modify the 'bind' setting. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param ip_to_set: The ip address to set + :param expected_log: The string we're expecting the logger to log + """ + redis_config = RedisConfig(server_redis_conf_file) + # For the test where bind is unset, delete bind from dict and set new ip val to a valid value + if ip_to_set == "bind-unset": + del redis_config.entries["bind"] + ip_to_set = "127.0.0.1" + assert not redis_config.set_ip_address(ip_to_set) + assert redis_config.get_ip_address() != ip_to_set + if expected_log is not None: + assert expected_log in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("port_to_set", [ + 6379, # Most common port + 1, # Edge case (low) + 65535, # Edge case (high) + 12345, # Random valid port + ]) + def test_set_port_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + port_to_set: str, + ): + """ + Test the `set_port` method with valid ports. These should all return True + and set the 'port' value to whatever `port_to_set` is. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param port_to_set: The port to set + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.set_port(port_to_set) + assert redis_config.get_port() == port_to_set + assert f"Port is set to {port_to_set}" in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("port_to_set, expected_log", [ + (None, None), # No port + (0, "Invalid port given."), # Edge case (low) + (65536, "Invalid port given."), # Edge case (high) + ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist + ]) + def test_set_port_invalid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + port_to_set: str, + expected_log: str, + ): + """ + Test the `set_port` method with invalid inputs. These should all return False + and not modify the 'port' setting. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param port_to_set: The port to set + :param expected_log: The string we're expecting the logger to log + """ + redis_config = RedisConfig(server_redis_conf_file) + # For the test where port is unset, delete port from dict and set port val to a valid value + if port_to_set == "port-unset": + del redis_config.entries["port"] + port_to_set = 5 + assert not redis_config.set_port(port_to_set) + assert redis_config.get_port() != port_to_set + if expected_log is not None: + assert expected_log in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("pass_to_set, expected_return", [ + ("valid_password", True), # Valid password + (None, False), # Invalid password + ]) + def test_set_password( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + pass_to_set: str, + expected_return: bool, + ): + """ + Test the `set_password` method with both valid and invalid input. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param pass_to_set: The password to set + :param expected_return: The expected return value + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.set_password(pass_to_set) == expected_return + if expected_return: + assert redis_conf.get_password() == pass_to_set + assert "New password set" in caplog.text, "Missing expected log message" + + def test_set_directory_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + server_testing_dir: str, + ): + """ + Test the `set_directory` method with valid input. This should return True, modify the + 'dir' value, and log some messages about creating/setting the directory. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + dir_to_set = f"{server_testing_dir}/dummy_dir" + assert redis_config.set_directory(dir_to_set) + assert redis_config.get_config_value("dir") == dir_to_set + assert f"Created directory {dir_to_set}" in caplog.text, "Missing created log message" + assert f"Directory is set to {dir_to_set}" in caplog.text, "Missing set log message" + + def test_set_directory_none(self, server_redis_conf_file: str): + """ + Test the `set_directory` method with None as the input. This should return False + and not modify the 'dir' setting. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_config = RedisConfig(server_redis_conf_file) + assert not redis_config.set_directory(None) + assert redis_config.get_config_value("dir") != None + + def test_set_directory_dir_unset( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + server_testing_dir: str, + ): + """ + Test the `set_directory` method with the 'dir' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + redis_config = RedisConfig(server_redis_conf_file) + del redis_config.entries["dir"] + dir_to_set = f"{server_testing_dir}/dummy_dir" + assert not redis_config.set_directory(dir_to_set) + assert "Unable to set directory for redis config" in caplog.text, "Missing expected log message" + + def test_set_snapshot_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'seconds' and 'changes'. + This should return True and modify both values of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + snap_sec_to_set = 20 + snap_changes_to_set = 30 + assert redis_conf.set_snapshot(seconds=snap_sec_to_set, changes=snap_changes_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == str(snap_sec_to_set) + assert save_val[1] == str(snap_changes_to_set) + expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " \ + f"Snapshot threshold is set to {snap_changes_to_set} changes" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_just_seconds(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'seconds'. This should + return True and modify the first value of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + orig_save = redis_conf.get_config_value("save").split() + snap_sec_to_set = 20 + assert redis_conf.set_snapshot(seconds=snap_sec_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == str(snap_sec_to_set) + assert save_val[1] == orig_save[1] + expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_just_changes(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'changes'. This should + return True and modify the second value of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + orig_save = redis_conf.get_config_value("save").split() + snap_changes_to_set = 30 + assert redis_conf.set_snapshot(changes=snap_changes_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == orig_save[0] + assert save_val[1] == str(snap_changes_to_set) + expected_log = f"Snapshot threshold is set to {snap_changes_to_set} changes" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_none(self, server_redis_conf_file: str): + """ + Test the `set_snapshot` method with None as the input for both seconds + and changes. This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_snapshot(seconds=None, changes=None) + + def test_set_snapshot_save_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with the 'save' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["save"] + assert not redis_conf.set_snapshot(seconds=20) + assert "Unable to get exisiting parameter values for snapshot" in caplog.text, "Missing expected log message" + + def test_set_snapshot_file_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot_file` method with a valid input. This should + return True and modify the value of 'dbfilename'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + filename = "dummy_file.rdb" + assert redis_conf.set_snapshot_file(filename) + assert redis_conf.get_config_value("dbfilename") == filename + assert f"Snapshot file is set to {filename}" in caplog.text, "Missing expected log message" + + def test_set_snapshot_file_none(self, server_redis_conf_file: str): + """ + Test the `set_snapshot_file` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_snapshot_file(None) + + def test_set_snapshot_file_dbfilename_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with the 'dbfilename' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["dbfilename"] + filename = "dummy_file.rdb" + assert not redis_conf.set_snapshot_file(filename) + assert redis_conf.get_config_value("dbfilename") != filename + assert "Unable to set snapshot_file name" in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize("mode_to_set", [ + "always", + "everysec", + "no", + ]) + def test_set_append_mode_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + mode_to_set: str, + ): + """ + Test the `set_append_mode` method with valid modes. These should all return True + and modify the value of 'appendfsync'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param mode_to_set: The mode to set + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.set_append_mode(mode_to_set) + assert redis_conf.get_config_value("appendfsync") == mode_to_set + assert f"Append mode is set to {mode_to_set}" in caplog.text, "Missing expected log message" + + def test_set_append_mode_invalid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_mode` method with an invalid mode. This should return False + and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + invalid_mode = "invalid" + assert not redis_conf.set_append_mode(invalid_mode) + assert redis_conf.get_config_value("appendfsync") != invalid_mode + expected_log = "Not a valid append_mode (Only valid modes are always, everysec, no)" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_append_mode_none(self, server_redis_conf_file: str): + """ + Test the `set_append_mode` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_append_mode(None) + + def test_set_append_mode_appendfsync_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_mode` method with the 'appendfsync' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["appendfsync"] + mode = "no" + assert not redis_conf.set_append_mode(mode) + assert redis_conf.get_config_value("appendfsync") != mode + assert "Unable to set append_mode in redis config" in caplog.text, "Missing expected log message" + + def test_set_append_file_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_file` method with a valid file. This should return True + and modify the value of 'appendfilename'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + valid_file = "valid" + assert redis_conf.set_append_file(valid_file) + assert redis_conf.get_config_value("appendfilename") == f'"{valid_file}"' + assert f"Append file is set to {valid_file}" in caplog.text, "Missing expected log message" + + def test_set_append_file_none(self, server_redis_conf_file: str): + """ + Test the `set_append_file` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_append_file(None) + + def test_set_append_file_appendfilename_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_file` method with the 'appendfilename' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["appendfilename"] + filename = "valid_filename" + assert not redis_conf.set_append_file(filename) + assert redis_conf.get_config_value("appendfilename") != filename + assert "Unable to set append filename." in caplog.text, "Missing expected log message" diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index c71e854eb..0332be944 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,18 +1,15 @@ """ Tests for the `server_util.py` module. """ -import filecmp import os import pytest -import shutil -from typing import Callable, Dict, Union +from typing import Dict, Union from merlin.server.server_util import ( AppYaml, ContainerConfig, ContainerFormatConfig, ProcessConfig, - RedisConfig, RedisUsers, ServerConfig, valid_ipv4, @@ -288,62 +285,3 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.process == ProcessConfig(server_process_config_data) assert config.container is None assert config.container_format is None - - -class TestRedisConfig: - """Tests for the RedisConfig class.""" - - def test_initialization(self, server_redis_conf_file: str): - """ - Using a dummy redis configuration file, test that the initialization - of the RedisConfig class behaves as expected. - - :param server_redis_conf_file: The path to a dummy redis configuration file - """ - expected_entries = { - "bind": "127.0.0.1", - "port": "6379", - "requirepass": "merlin_password", - "dir": "./", - "save": "300 100", - "dbfilename": "dump.rdb", - "appendfsync": "everysec", - "appendfilename": "appendonly.aof", - } - expected_comments = { - "bind": "# ip address\n", - "port": "\n# port\n", - "requirepass": "\n# password\n", - "dir": "\n# directory\n", - "save": "\n# snapshot\n", - "dbfilename": "\n# db file\n", - "appendfsync": "\n# append mode\n", - "appendfilename": "\n# append file\n", - } - expected_trailing_comment = "\n# dummy trailing comment" - expected_entry_order = list(expected_entries.keys()) - redis_config = RedisConfig(server_redis_conf_file) - assert redis_config.filename == server_redis_conf_file - assert not redis_config.changed - assert redis_config.entries == expected_entries - assert redis_config.entry_order == expected_entry_order - assert redis_config.comments == expected_comments - assert redis_config.trailing_comments == expected_trailing_comment - - def test_write(self, server_redis_conf_file: str, server_testing_dir: str): - """ - """ - copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" - - # Create a RedisConf object with the basic redis conf file - redis_config = RedisConfig(server_redis_conf_file) - - # Change the filepath of the redis config file to be the copy that we'll write to - redis_config.filename = copy_redis_conf_file - - # Run the test - redis_config.write() - - # Check that the contents of the copied file match the contents of the basic file - assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) - From 1df0e442e7216a238b00472167e4fb4449e42d16 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 4 Jun 2024 16:41:12 -0700 Subject: [PATCH 075/201] add tests for RedisUsers class --- merlin/server/server_util.py | 2 +- tests/fixtures/server.py | 48 +++++++- tests/unit/server/test_server_util.py | 167 ++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 3 deletions(-) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 3a4cfc2f8..0af8509e0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -623,7 +623,7 @@ def set_password(self, user: str, password: str): self.users[user].set_password(password) return True - def remove_user(self, user) -> bool: + def remove_user(self, user: str) -> bool: """Remove a user from the dict of users""" if user in self.users: del self.users[user] diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 04c858f46..5c5dea102 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -3,6 +3,7 @@ """ import os import pytest +import yaml from typing import Dict @pytest.fixture(scope="class") @@ -90,8 +91,7 @@ def server_testing_dir(temp_output_dir: str) -> str: @pytest.fixture(scope="session") def server_redis_conf_file(server_testing_dir: str) -> str: """ - Fixture to copy the redis.conf file from the merlin/server/ directory to the - temporary output directory and provide the path to the copied file. + Fixture to write a redis.conf file to the temporary output directory. If a test will modify this file with a file write, you should make a copy of this file to modify instead. @@ -132,3 +132,47 @@ def server_redis_conf_file(server_testing_dir: str) -> str: rcf.write(file_contents) return redis_conf_file + +@pytest.fixture(scope="session") +def server_users() -> dict: + """ + Create a dictionary of two test users with identical configuration settings. + + :returns: A dict containing the two test users and their settings + """ + users = { + "default": { + "channels": '*', + "commands": '@all', + "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', + "keys": '*', + "status": 'on', + }, + "test_user": { + "channels": '*', + "commands": '@all', + "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', + "keys": '*', + "status": 'on', + } + } + return users + +@pytest.fixture(scope="session") +def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: + """ + Fixture to write a redis.users file to the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :param server_users: A dict of test user configurations + :returns: The path to the redis user configuration file we'll use for testing + """ + redis_users_file = f"{server_testing_dir}/redis.users" + + with open(redis_users_file, "w") as ruf: + yaml.dump(server_users, ruf) + + return redis_users_file \ No newline at end of file diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 0332be944..61f29293c 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,6 +1,8 @@ """ Tests for the `server_util.py` module. """ +import filecmp +import hashlib import os import pytest from typing import Dict, Union @@ -285,3 +287,168 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.process == ProcessConfig(server_process_config_data) assert config.container is None assert config.container_format is None + +class TestRedisUsers: + """ + Tests for the RedisUsers class. + + TODO add integration test(s) for `apply_to_redis` method of this class. + """ + + class TestUser: + """Tests for the RedisUsers.User class""" + + def test_initializaiton(self): + """Test the initialization process of the User class.""" + user = RedisUsers.User() + assert user.status == "on" + assert user.hash_password == hashlib.sha256(b"password").hexdigest() + assert user.keys == "*" + assert user.channels == "*" + assert user.commands == "@all" + + def test_parse_dict(self): + """Test the `parse_dict` method of the User class.""" + test_dict = { + "status": "test_status", + "hash_password": "test_password", + "keys": "test_keys", + "channels": "test_channels", + "commands": "test_commands", + } + user = RedisUsers.User() + user.parse_dict(test_dict) + assert user.status == test_dict["status"] + assert user.hash_password == test_dict["hash_password"] + assert user.keys == test_dict["keys"] + assert user.channels == test_dict["channels"] + assert user.commands == test_dict["commands"] + + def test_get_user_dict(self): + """Test the `get_user_dict` method of the User class.""" + test_dict = { + "status": "test_status", + "hash_password": "test_password", + "keys": "test_keys", + "channels": "test_channels", + "commands": "test_commands", + "invalid_key": "invalid_val", + } + user = RedisUsers.User() + user.parse_dict(test_dict) # Set the test values + actual_dict = user.get_user_dict() + assert "invalid_key" not in actual_dict # Check that the invalid key isn't parsed + + # Check that the values are as expected + for key, val in actual_dict.items(): + if key == "status": + assert val == "on" + else: + assert val == test_dict[key] + + def test_set_password(self): + """Test the `set_password` method of the User class.""" + user = RedisUsers.User() + pass_to_set = "dummy_password" + user.set_password(pass_to_set) + assert user.hash_password == hashlib.sha256(bytes(pass_to_set, "utf-8")).hexdigest() + + def test_initialization(self, server_redis_users_file: str, server_users: dict): + """ + Test the initialization process of the RedisUsers class. + + :param server_redis_users_file: The path to a dummy redis users file + :param server_users: A dict of test user configurations + """ + redis_users = RedisUsers(server_redis_users_file) + assert redis_users.filename == server_redis_users_file + assert len(redis_users.users) == len(server_users) + + def test_write(self, server_redis_users_file: str, server_testing_dir: str): + """ + Test that the write functionality works by writing the contents of a dummy + users file to a blank users file. + + :param server_redis_users_file: The path to a dummy redis users file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_redis_users_file = f"{server_testing_dir}/redis_copy.users" + + # Create a RedisUsers object with the basic redis users file + redis_users = RedisUsers(server_redis_users_file) + + # Change the filepath of the redis users file to be the copy that we'll write to + redis_users.filename = copy_redis_users_file + + # Run the test + redis_users.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_users_file, copy_redis_users_file) + + def test_add_user_nonexistent(self, server_redis_users_file: str): + """ + Test the `add_user` method with a user that doesn't exists. + This should return True and add the user to the list of users. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + num_users_before = len(redis_users.users) + assert redis_users.add_user("new_user") + assert len(redis_users.users) == num_users_before + 1 + + def test_add_user_exists(self, server_redis_users_file: str): + """ + Test the `add_user` method with a user that already exists. + This should return False. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.add_user("test_user") + + def test_set_password_valid(self, server_redis_users_file: str): + """ + Test the `set_password` method with a user that exists. + This should return True and change the password for the user. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + pass_to_set = "new_password" + assert redis_users.set_password("test_user", pass_to_set) + expected_hash_pass = hashlib.sha256(bytes(pass_to_set, "utf-8")).hexdigest() + assert redis_users.users["test_user"].hash_password == expected_hash_pass + + def test_set_password_invalid(self, server_redis_users_file: str): + """ + Test the `set_password` method with a user that doesn't exist. + This should return False. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.set_password("nonexistent_user", "new_password") + + def test_remove_user_valid(self, server_redis_users_file: str): + """ + Test the `remove_user` method with a user that exists. + This should return True and remove the user from the list of users. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + num_users_before = len(redis_users.users) + assert redis_users.remove_user("test_user") + assert len(redis_users.users) == num_users_before - 1 + + def test_remove_user_invalid(self, server_redis_users_file: str): + """ + Test the `remove_user` method with a user that doesn't exist. + This should return False and not modify the user list. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.remove_user("nonexistent_user") \ No newline at end of file From 921c38b8607932ab412850651ba230ed0b21848b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 10:01:03 -0700 Subject: [PATCH 076/201] change server fixtures to use redis config files --- tests/fixtures/server.py | 174 +++++++++++++++----------- tests/unit/server/test_server_util.py | 6 +- 2 files changed, 107 insertions(+), 73 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 5c5dea102..ae3a966c8 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -6,72 +6,6 @@ import yaml from typing import Dict -@pytest.fixture(scope="class") -def server_container_config_data(temp_output_dir: str) -> Dict[str, str]: - """ - Fixture to provide sample data for ContainerConfig tests - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: A dict containing the necessary key/values for the ContainerConfig object - """ - return { - "format": "docker", - "image_type": "postgres", - "image": "postgres:latest", - "url": "postgres://localhost", - "config": "postgres.conf", - "config_dir": "/path/to/config", - "pfile": "merlin_server_postgres.pf", - "pass_file": f"{temp_output_dir}/postgres.pass", - "user_file": "postgres.users", - } - -@pytest.fixture(scope="class") -def server_container_format_config_data() -> Dict[str, str]: - """ - Fixture to provide sample data for ContainerFormatConfig tests - - :returns: A dict containing the necessary key/values for the ContainerFormatConfig object - """ - return { - "command": "docker", - "run_command": "{command} run --name {name} -d {image}", - "stop_command": "{command} stop {name}", - "pull_command": "{command} pull {url}", - } - -@pytest.fixture(scope="class") -def server_process_config_data() -> Dict[str, str]: - """ - Fixture to provide sample data for ProcessConfig tests - - :returns: A dict containing the necessary key/values for the ProcessConfig object - """ - return { - "status": "status {pid}", - "kill": "terminate {pid}", - } - -@pytest.fixture(scope="class") -def server_server_config( - server_container_config_data: Dict[str, str], - server_process_config_data: Dict[str, str], - server_container_format_config_data: Dict[str, str], -) -> Dict[str, Dict[str, str]]: - """ - Fixture to provide sample data for ServerConfig tests - - :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class - :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class - :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class - :returns: A dictionary containing each of the configuration dicts we'll need - """ - return { - "container": server_container_config_data, - "process": server_process_config_data, - "docker": server_container_format_config_data, - } - @pytest.fixture(scope="session") def server_testing_dir(temp_output_dir: str) -> str: @@ -81,7 +15,7 @@ def server_testing_dir(temp_output_dir: str) -> str: :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :returns: The path to the temporary testing directory for server tests """ - testing_dir = f"{temp_output_dir}/server_testing/" + testing_dir = f"{temp_output_dir}/server_testing" if not os.path.exists(testing_dir): os.mkdir(testing_dir) @@ -96,7 +30,7 @@ def server_redis_conf_file(server_testing_dir: str) -> str: If a test will modify this file with a file write, you should make a copy of this file to modify instead. - :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to :returns: The path to the redis configuration file we'll use for testing """ redis_conf_file = f"{server_testing_dir}/redis.conf" @@ -133,6 +67,26 @@ def server_redis_conf_file(server_testing_dir: str) -> str: return redis_conf_file + +@pytest.fixture(scope="session") +def server_redis_pass_file(server_testing_dir: str) -> str: + """ + Fixture to create a redis password file in the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :returns: The path to the redis password file + """ + redis_pass_file = f"{server_testing_dir}/redis.pass" + + with open(redis_pass_file, "w") as rpf: + rpf.write("server-tests-password") + + return redis_pass_file + + @pytest.fixture(scope="session") def server_users() -> dict: """ @@ -158,6 +112,7 @@ def server_users() -> dict: } return users + @pytest.fixture(scope="session") def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: """ @@ -166,7 +121,7 @@ def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: If a test will modify this file with a file write, you should make a copy of this file to modify instead. - :param server_testing_dir: A pytest fixture that defines a path to the the output directory we'll write to + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to :param server_users: A dict of test user configurations :returns: The path to the redis user configuration file we'll use for testing """ @@ -175,4 +130,83 @@ def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: with open(redis_users_file, "w") as ruf: yaml.dump(server_users, ruf) - return redis_users_file \ No newline at end of file + return redis_users_file + + +@pytest.fixture(scope="class") +def server_container_config_data( + server_testing_dir: str, + server_redis_conf_file: str, + server_redis_pass_file: str, + server_redis_users_file: str, +) -> Dict[str, str]: + """ + Fixture to provide sample data for ContainerConfig tests. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_redis_conf_file: A pytest fixture that defines a path to a redis configuration file + :param server_redis_pass_file: A pytest fixture that defines a path to a redis password file + :param server_redis_users_file: A pytest fixture that defines a path to a redis users file + :returns: A dict containing the necessary key/values for the ContainerConfig object + """ + + return { + "format": "singularity", + "image_type": "redis", + "image": "redis_latest.sif", + "url": "docker://redis", + "config": server_redis_conf_file.split("/")[-1], + "config_dir": server_testing_dir, + "pfile": "merlin_server.pf", + "pass_file": server_redis_pass_file.split("/")[-1], + "user_file": server_redis_users_file.split("/")[-1], + } + + +@pytest.fixture(scope="class") +def server_container_format_config_data() -> Dict[str, str]: + """ + Fixture to provide sample data for ContainerFormatConfig tests + + :returns: A dict containing the necessary key/values for the ContainerFormatConfig object + """ + return { + "command": "singularity", + "run_command": "{command} run -H {home_dir} {image} {config}", + "stop_command": "kill", + "pull_command": "{command} pull {image} {url}", + } + + +@pytest.fixture(scope="class") +def server_process_config_data() -> Dict[str, str]: + """ + Fixture to provide sample data for ProcessConfig tests + + :returns: A dict containing the necessary key/values for the ProcessConfig object + """ + return { + "status": "pgrep -P {pid}", + "kill": "kill {pid}", + } + + +@pytest.fixture(scope="class") +def server_server_config( + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], + server_container_format_config_data: Dict[str, str], +) -> Dict[str, Dict[str, str]]: + """ + Fixture to provide sample data for ServerConfig tests + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :returns: A dictionary containing each of the configuration dicts we'll need + """ + return { + "container": server_container_config_data, + "process": server_process_config_data, + "singularity": server_container_format_config_data, + } diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 61f29293c..2986e22de 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -169,7 +169,7 @@ def test_get_container_password(self, server_container_config_data: Dict[str, st :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ # Write a fake password to the password file - test_password = "super-secret-password" + test_password = "server-tests-password" with open(server_container_config_data["pass_file"], "w") as pass_file: pass_file.write(test_password) @@ -274,7 +274,7 @@ def test_init_with_complete_data(self, server_server_config: Dict[str, str]): config = ServerConfig(server_server_config) assert config.container == ContainerConfig(server_server_config["container"]) assert config.process == ProcessConfig(server_server_config["process"]) - assert config.container_format == ContainerFormatConfig(server_server_config["docker"]) + assert config.container_format == ContainerFormatConfig(server_server_config["singularity"]) def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): """ @@ -451,4 +451,4 @@ def test_remove_user_invalid(self, server_redis_users_file: str): :param server_redis_users_file: The path to a dummy redis users file """ redis_users = RedisUsers(server_redis_users_file) - assert not redis_users.remove_user("nonexistent_user") \ No newline at end of file + assert not redis_users.remove_user("nonexistent_user") From 2cad8cbebde0b0fd26aba119ca400ebee70c9b9f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 10:55:35 -0700 Subject: [PATCH 077/201] add tests for AppYaml class --- tests/fixtures/server.py | 62 ++++++++++++++++++++++++ tests/unit/server/test_server_util.py | 70 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index ae3a966c8..284084e2c 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -210,3 +210,65 @@ def server_server_config( "process": server_process_config_data, "singularity": server_container_format_config_data, } + + +@pytest.fixture(scope="function") +def server_app_yaml_contents( + server_redis_pass_file: str, + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], +) -> Dict[str, str]: + """ + Fixture to create the contents of an app.yaml file. + + :param server_redis_pass_file: A pytest fixture that defines a path to a redis password file + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :returns: A dict with typical app.yaml contents + """ + contents = { + "broker": { + "cert_reqs": "none", + "name": "redis", + "password": server_redis_pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "testhost", + }, + "container": server_container_config_data, + "process": server_process_config_data, + "results_backend": { + "cert_reqs": "none", + "db_num": 0, + "name": "redis", + "password": server_redis_pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + } + } + return contents + + +@pytest.fixture(scope="function") +def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> str: + """ + Fixture to create an app.yaml file in the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + NOTE this must be function scoped since server_app_yaml_contents is function scoped. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_app_yaml_contents: A pytest fixture that creates a dict of contents for an app.yaml file + :returns: The path to the app.yaml file + """ + app_yaml_file = f"{server_testing_dir}/app.yaml" + + if not os.path.exists(app_yaml_file): + with open(app_yaml_file, "w") as ayf: + yaml.dump(server_app_yaml_contents, ayf) + + return app_yaml_file \ No newline at end of file diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 2986e22de..20ff922bb 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -12,6 +12,7 @@ ContainerConfig, ContainerFormatConfig, ProcessConfig, + RedisConfig, RedisUsers, ServerConfig, valid_ipv4, @@ -288,6 +289,7 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] assert config.container is None assert config.container_format is None + class TestRedisUsers: """ Tests for the RedisUsers class. @@ -452,3 +454,71 @@ def test_remove_user_invalid(self, server_redis_users_file: str): """ redis_users = RedisUsers(server_redis_users_file) assert not redis_users.remove_user("nonexistent_user") + + +class TestAppYaml: + """Tests for the AppYaml class.""" + + def test_initialization(self, server_app_yaml: str, server_app_yaml_contents: dict): + """ + Test the initialization process of the AppYaml class. + + :param server_app_yaml: The path to an app.yaml file + :param server_app_yaml_contents: A dict of app.yaml configurations + """ + app_yaml = AppYaml(server_app_yaml) + assert app_yaml.get_data() == server_app_yaml_contents + + def test_apply_server_config(self, server_app_yaml: str, server_server_config: Dict[str, str]): + """ + Test the `apply_server_config` method. This should update the data attribute. + + :param server_app_yaml: The path to an app.yaml file + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + app_yaml = AppYaml(server_app_yaml) + server_config = ServerConfig(server_server_config) + redis_config = RedisConfig(server_config.container.get_config_path()) + app_yaml.apply_server_config(server_config) + + assert app_yaml.data[app_yaml.broker_name]["name"] == server_config.container.get_image_type() + assert app_yaml.data[app_yaml.broker_name]["username"] == "default" + assert app_yaml.data[app_yaml.broker_name]["password"] == server_config.container.get_pass_file_path() + assert app_yaml.data[app_yaml.broker_name]["server"] == redis_config.get_ip_address() + assert app_yaml.data[app_yaml.broker_name]["port"] == redis_config.get_port() + + assert app_yaml.data[app_yaml.results_name]["name"] == server_config.container.get_image_type() + assert app_yaml.data[app_yaml.results_name]["username"] == "default" + assert app_yaml.data[app_yaml.results_name]["password"] == server_config.container.get_pass_file_path() + assert app_yaml.data[app_yaml.results_name]["server"] == redis_config.get_ip_address() + assert app_yaml.data[app_yaml.results_name]["port"] == redis_config.get_port() + + def test_update_data(self, server_app_yaml: str): + """ + Test the `update_data` method. This should update the data attribute. + + :param server_app_yaml: The path to an app.yaml file + """ + app_yaml = AppYaml(server_app_yaml) + new_data = {app_yaml.broker_name: {"username": "new_user"}} + app_yaml.update_data(new_data) + + assert app_yaml.data[app_yaml.broker_name]["username"] == "new_user" + + def test_write(self, server_app_yaml: str, server_testing_dir: str): + """ + Test the `write` method. This should write data to a file. + + :param server_app_yaml: The path to an app.yaml file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_app_yaml = f"{server_testing_dir}/app_copy.yaml" + + # Create a AppYaml object with the basic app.yaml file + app_yaml = AppYaml(server_app_yaml) + + # Run the test + app_yaml.write(copy_app_yaml) + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_app_yaml, copy_app_yaml) From f28cad3de18964413c46afba592f93056796d45a Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 12:06:09 -0700 Subject: [PATCH 078/201] final cleanup of server_utils --- tests/fixtures/server.py | 9 ++-- tests/unit/server/test_server_util.py | 67 ++++++++++++++++----------- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 284084e2c..01db7bd56 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -4,7 +4,7 @@ import os import pytest import yaml -from typing import Dict +from typing import Dict, Union @pytest.fixture(scope="session") @@ -88,7 +88,7 @@ def server_redis_pass_file(server_testing_dir: str) -> str: @pytest.fixture(scope="session") -def server_users() -> dict: +def server_users() -> Dict[str, Dict[str, str]]: """ Create a dictionary of two test users with identical configuration settings. @@ -217,7 +217,7 @@ def server_app_yaml_contents( server_redis_pass_file: str, server_container_config_data: Dict[str, str], server_process_config_data: Dict[str, str], -) -> Dict[str, str]: +) -> Dict[str, Union[str, int]]: """ Fixture to create the contents of an app.yaml file. @@ -256,9 +256,6 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> """ Fixture to create an app.yaml file in the temporary output directory. - If a test will modify this file with a file write, you should make a copy of - this file to modify instead. - NOTE this must be function scoped since server_app_yaml_contents is function scoped. :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index 20ff922bb..c9b59e83e 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -30,8 +30,8 @@ def test_valid_ipv4_valid_ip(valid_ip: str): Test the `valid_ipv4` function with valid IPs. This should return True. - :param valid_ip: A valid port to test. - These are pulled from the parametrized list defined above this test. + :param valid_ip: A valid port to test. These are pulled from the parametrized + list defined above this test. """ assert valid_ipv4(valid_ip) @@ -47,8 +47,8 @@ def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): An IP is valid if every integer separated by the '.' delimiter are between 0 and 255. This should return False for both IPs tested here. - :param invalid_ip: An invalid port to test. - These are pulled from the parametrized list defined above this test. + :param invalid_ip: An invalid port to test. These are pulled from the parametrized + list defined above this test. """ assert not valid_ipv4(invalid_ip) @@ -63,8 +63,8 @@ def test_valid_port_valid_input(valid_input: int): Valid ports are ports between 1 and 65535. This should return True. - :param valid_input: A valid input value to test. - These are pulled from the parametrized list defined above this test. + :param valid_input: A valid input value to test. These are pulled from the parametrized + list defined above this test. """ assert valid_port(valid_input) @@ -79,8 +79,8 @@ def test_valid_port_invalid_input(invalid_input: int): Valid ports are ports between 1 and 65535. This should return False for each invalid input tested. - :param invalid_input: An invalid input value to test. - These are pulled from the parametrized list defined above this test. + :param invalid_input: An invalid input value to test. These are pulled from the parametrized + list defined above this test. """ assert not valid_port(invalid_input) @@ -90,7 +90,7 @@ class TestContainerConfig: def test_init_with_complete_data(self, server_container_config_data: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ @@ -107,7 +107,7 @@ def test_init_with_complete_data(self, server_container_config_data: Dict[str, s def test_init_with_missing_data(self): """ - Tests that __init__ uses defaults for missing data + Tests that __init__ uses defaults for missing data. """ incomplete_data = {"format": "docker"} config = ContainerConfig(incomplete_data) @@ -130,7 +130,7 @@ def test_init_with_missing_data(self): ]) def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): """ - Tests that get_*_path methods construct the correct path + Tests that get_*_path methods construct the correct path. :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class :param attr_name: Name of the attribute to be tested. These are pulled from the parametrized list defined above this test. @@ -153,7 +153,7 @@ def test_get_path_methods(self, server_container_config_data: Dict[str, str], at ]) def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ - Tests that all getter methods return the correct attribute values + Tests that all getter methods return the correct attribute values. :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. @@ -163,20 +163,31 @@ def test_getter_methods(self, server_container_config_data: Dict[str, str], gett getter = getattr(config, getter_name) assert getter() == server_container_config_data[expected_attr] - def test_get_container_password(self, server_container_config_data: Dict[str, str]): + def test_get_container_password(self, server_testing_dir: str, server_container_config_data: Dict[str, str]): """ - Test that the get_container_password is reading the password file properly + Test that the `get_container_password` method is reading the password file properly. + :param server_testing_dir: The path to the the temp output directory for server tests :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ # Write a fake password to the password file - test_password = "server-tests-password" - with open(server_container_config_data["pass_file"], "w") as pass_file: + test_password = "super-secret-password" + temp_pass_file = f"{server_testing_dir}/temp.pass" + with open(temp_pass_file, "w") as pass_file: pass_file.write(test_password) - # Run the test - config = ContainerConfig(server_container_config_data) - assert config.get_container_password() == test_password + # Use temp pass file + orig_pass_file = server_container_config_data["pass_file"] + server_container_config_data["pass_file"] = temp_pass_file + + try: + # Run the test + config = ContainerConfig(server_container_config_data) + assert config.get_container_password() == test_password + except Exception as exc: + # If there was a problem, reset to the original password file + server_container_config_data["pass_file"] = orig_pass_file + raise exc class TestContainerFormatConfig: @@ -184,7 +195,7 @@ class TestContainerFormatConfig: def test_init_with_complete_data(self, server_container_format_config_data: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class """ @@ -196,7 +207,7 @@ def test_init_with_complete_data(self, server_container_format_config_data: Dict def test_init_with_missing_data(self): """ - Tests that __init__ uses defaults for missing data + Tests that __init__ uses defaults for missing data. """ incomplete_data = {"command": "docker"} config = ContainerFormatConfig(incomplete_data) @@ -213,7 +224,7 @@ def test_init_with_missing_data(self): ]) def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ - Tests that all getter methods return the correct attribute values + Tests that all getter methods return the correct attribute values. :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. @@ -229,7 +240,7 @@ class TestProcessConfig: def test_init_with_complete_data(self, server_process_config_data: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class """ @@ -239,7 +250,7 @@ def test_init_with_complete_data(self, server_process_config_data: Dict[str, str def test_init_with_missing_data(self): """ - Tests that __init__ uses defaults for missing data + Tests that __init__ uses defaults for missing data. """ incomplete_data = {"status": "status {pid}"} config = ProcessConfig(incomplete_data) @@ -252,7 +263,7 @@ def test_init_with_missing_data(self): ]) def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ - Tests that all getter methods return the correct attribute values + Tests that all getter methods return the correct attribute values. :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. @@ -268,7 +279,7 @@ class TestServerConfig: def test_init_with_complete_data(self, server_server_config: Dict[str, str]): """ - Tests that __init__ populates attributes correctly with complete data + Tests that __init__ populates attributes correctly with complete data. :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ @@ -279,7 +290,7 @@ def test_init_with_complete_data(self, server_server_config: Dict[str, str]): def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): """ - Tests that __init__ uses None for missing data + Tests that __init__ uses None for missing data. :param server_process_config_data: A pytest fixture of test data to pass to the ContainerConfig class """ @@ -298,7 +309,7 @@ class TestRedisUsers: """ class TestUser: - """Tests for the RedisUsers.User class""" + """Tests for the RedisUsers.User class.""" def test_initializaiton(self): """Test the initialization process of the User class.""" From 716dc322a15675fc1c5051a54b674bfa431e7a80 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 12:31:00 -0700 Subject: [PATCH 079/201] fix lint issues --- merlin/examples/generator.py | 2 +- merlin/server/server_util.py | 8 +- tests/conftest.py | 15 ++- tests/context_managers/server_manager.py | 1 + tests/fixtures/server.py | 37 +++--- tests/fixtures/status.py | 6 +- tests/unit/common/test_dumper.py | 34 ++++-- tests/unit/common/test_encryption.py | 1 + tests/unit/common/test_sample_index.py | 1 + tests/unit/common/test_util_sampling.py | 1 + tests/unit/config/test_broker.py | 1 + tests/unit/config/test_config_object.py | 1 + tests/unit/config/test_configfile.py | 1 + tests/unit/config/test_results_backend.py | 1 + tests/unit/server/test_RedisConfig.py | 132 ++++++++++++--------- tests/unit/server/test_server_util.py | 138 +++++++++++++--------- tests/unit/test_examples_generator.py | 1 + tests/utils.py | 1 + 18 files changed, 232 insertions(+), 150 deletions(-) diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index b1b89952d..bcdf87b8d 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -146,5 +146,5 @@ def setup_example(name, outdir): LOG.info(f"Copying example '{name}' to {outdir}") write_example(src_path, outdir) - print(f'example: {example}') + print(f"example: {example}") return example diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 0af8509e0..30de856af 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -124,7 +124,7 @@ def __init__(self, data: dict) -> None: def __eq__(self, other: "ContainerFormatConfig"): """ Equality magic method used for testing this class - + :param other: Another ContainerFormatConfig object to check if they're the same """ variables = ("format", "image_type", "image", "url", "config", "config_dir", "pfile", "pass_file", "user_file") @@ -220,7 +220,7 @@ def __init__(self, data: dict) -> None: def __eq__(self, other: "ContainerFormatConfig"): """ Equality magic method used for testing this class - + :param other: Another ContainerFormatConfig object to check if they're the same """ variables = ("command", "run_command", "stop_command", "pull_command") @@ -263,7 +263,7 @@ def __init__(self, data: dict) -> None: def __eq__(self, other: "ProcessConfig"): """ Equality magic method used for testing this class - + :param other: Another ProcessConfig object to check if they're the same """ variables = ("status", "kill") @@ -441,7 +441,7 @@ def set_snapshot(self, seconds: int = None, changes: int = None) -> bool: """ Sets the 'seconds' and/or 'changes' values of the snapshot setting, depending on what the user requests. - + :param seconds: The first value of snapshot to change. If we're leaving it the same this will be None. :param changes: The second value of snapshot to change. If we're leaving it the diff --git a/tests/conftest.py b/tests/conftest.py index a0f77bc9d..fb06586b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,9 @@ from tests.utils import create_cert_files, create_pass_file +# pylint: disable=redefined-outer-name + + ####################################### # Loading in Module Specific Fixtures # ####################################### @@ -114,7 +117,7 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str) -> str: # pylint: disable=redefined-outer-name +def merlin_server_dir(temp_output_dir: str) -> str: """ The path to the merlin_server directory that will be created by the `redis_server` fixture. @@ -128,7 +131,7 @@ def merlin_server_dir(temp_output_dir: str) -> str: # pylint: disable=redefined @pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # pylint: disable=redefined-outer-name +def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. @@ -149,7 +152,7 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # @pytest.fixture(scope="session") -def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer-name +def celery_app(redis_server: str) -> Celery: """ Create the celery app to be used throughout our integration tests. @@ -160,7 +163,7 @@ def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer- @pytest.fixture(scope="session") -def sleep_sig(celery_app: Celery) -> Signature: # pylint: disable=redefined-outer-name +def sleep_sig(celery_app: Celery) -> Signature: """ Create a task registered to our celery app and return a signature for it. Once requested by a test, you can set the queue you'd like to send this to @@ -192,7 +195,7 @@ def worker_queue_map() -> Dict[str, str]: @pytest.fixture(scope="class") -def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pylint: disable=redefined-outer-name +def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): """ Launch the workers on the celery app fixture using the worker and queue names defined in the worker_queue_map fixture. @@ -235,7 +238,7 @@ def test_encryption_key() -> bytes: @pytest.fixture(scope="function") -def config(merlin_server_dir: str, test_encryption_key: bytes): # pylint: disable=redefined-outer-name +def config(merlin_server_dir: str, test_encryption_key: bytes): """ DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index ea6a731ff..c88948772 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -2,6 +2,7 @@ Module to define functionality for managing the containerized server used for testing. """ + import os import signal import subprocess diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 01db7bd56..156a374d7 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -1,10 +1,15 @@ """ Fixtures specifically for help testing the modules in the server/ directory. """ + import os +from typing import Dict, Union + import pytest import yaml -from typing import Dict, Union + + +# pylint: disable=redefined-outer-name @pytest.fixture(scope="session") @@ -60,7 +65,9 @@ def server_redis_conf_file(server_testing_dir: str) -> str: appendfilename appendonly.aof # dummy trailing comment - """.strip().replace(" ", "") + """.strip().replace( + " ", "" + ) with open(redis_conf_file, "w") as rcf: rcf.write(file_contents) @@ -96,19 +103,19 @@ def server_users() -> Dict[str, Dict[str, str]]: """ users = { "default": { - "channels": '*', - "commands": '@all', - "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', - "keys": '*', - "status": 'on', + "channels": "*", + "commands": "@all", + "hash_password": "1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a", + "keys": "*", + "status": "on", }, "test_user": { - "channels": '*', - "commands": '@all', - "hash_password": '1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a', - "keys": '*', - "status": 'on', - } + "channels": "*", + "commands": "@all", + "hash_password": "1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a", + "keys": "*", + "status": "on", + }, } return users @@ -246,7 +253,7 @@ def server_app_yaml_contents( "port": 6379, "server": "127.0.0.1", "username": "default", - } + }, } return contents @@ -268,4 +275,4 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> with open(app_yaml_file, "w") as ayf: yaml.dump(server_app_yaml_contents, ayf) - return app_yaml_file \ No newline at end of file + return app_yaml_file diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index f26cea37c..85a01ed99 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -13,8 +13,10 @@ from tests.unit.study.status_test_files import status_test_variables +# pylint: disable=redefined-outer-name -@pytest.fixture(scope="session") + +@pytest.fixture(scope="class") def status_testing_dir(temp_output_dir: str) -> str: """ A pytest fixture to set up a temporary directory to write files to for testing status. @@ -29,7 +31,7 @@ def status_testing_dir(temp_output_dir: str) -> str: return testing_dir -@pytest.fixture(scope="session") +@pytest.fixture(scope="class") def status_empty_file(status_testing_dir: str) -> str: # pylint: disable=W0621 """ A pytest fixture to create an empty status file. diff --git a/tests/unit/common/test_dumper.py b/tests/unit/common/test_dumper.py index 7c437fde9..c52e9fe90 100644 --- a/tests/unit/common/test_dumper.py +++ b/tests/unit/common/test_dumper.py @@ -1,21 +1,27 @@ """ Tests for the `dumper.py` file. """ + import csv import json import os -import pytest - from datetime import datetime from time import sleep +import pytest + from merlin.common.dumper import dump_handler + NUM_ROWS = 5 -CSV_INFO_TO_DUMP = {"row_num": [i for i in range(1, NUM_ROWS+1)], "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS+1)]} -JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS+1)} +CSV_INFO_TO_DUMP = { + "row_num": [i for i in range(1, NUM_ROWS + 1)], + "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS + 1)], +} +JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS + 1)} DUMP_HANDLER_DIR = "{temp_output_dir}/dump_handler" + def test_dump_handler_invalid_dump_file(): """ This is really testing the initialization of the Dumper class with an invalid file type. @@ -25,6 +31,7 @@ def test_dump_handler_invalid_dump_file(): dump_handler("bad_file.txt", CSV_INFO_TO_DUMP) assert "Invalid file type for bad_file.txt. Supported file types are: ['csv', 'json']" in str(excinfo.value) + def get_output_file(temp_dir: str, file_name: str): """ Helper function to get a full path to the temporary output file. @@ -38,6 +45,7 @@ def get_output_file(temp_dir: str, file_name: str): dump_file = f"{dump_dir}/{file_name}" return dump_file + def run_csv_dump_test(dump_file: str, fmode: str): """ Run the test for csv dump. @@ -52,16 +60,17 @@ def run_csv_dump_test(dump_file: str, fmode: str): reader = csv.reader(df) written_data = list(reader) - expected_rows = NUM_ROWS*2 if fmode == "a" else NUM_ROWS - assert len(written_data) == expected_rows+1 # Adding one because of the header row + expected_rows = NUM_ROWS * 2 if fmode == "a" else NUM_ROWS + assert len(written_data) == expected_rows + 1 # Adding one because of the header row for i, row in enumerate(written_data): assert len(row) == 2 # Check number of columns if i == 0: # Checking the header row assert row[0] == "row_num" assert row[1] == "other_info" else: # Checking the data rows - assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i%NUM_ROWS)-1]) - assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i%NUM_ROWS)-1]) + assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i % NUM_ROWS) - 1]) + assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i % NUM_ROWS) - 1]) + def test_dump_handler_csv_write(temp_output_dir: str): """ @@ -80,6 +89,7 @@ def test_dump_handler_csv_write(temp_output_dir: str): # Assert that everything ran properly run_csv_dump_test(dump_file, "w") + def test_dump_handler_csv_append(temp_output_dir: str): """ This is really testing the write method of the Dumper class with the file write mode set to append. @@ -93,13 +103,14 @@ def test_dump_handler_csv_append(temp_output_dir: str): # Run the first call to create the csv file dump_handler(dump_file, CSV_INFO_TO_DUMP) - + # Run the second call to append to the csv file dump_handler(dump_file, CSV_INFO_TO_DUMP) # Assert that everything ran properly run_csv_dump_test(dump_file, "a") + def test_dump_handler_json_write(temp_output_dir: str): """ This is really testing the write method of the Dumper class. @@ -120,6 +131,7 @@ def test_dump_handler_json_write(temp_output_dir: str): contents = json.load(df) assert contents == JSON_INFO_TO_DUMP + def test_dump_handler_json_append(temp_output_dir: str): """ This is really testing the write method of the Dumper class with the file write mode set to append. @@ -137,7 +149,7 @@ def test_dump_handler_json_append(temp_output_dir: str): dump_handler(dump_file, first_dump) # Sleep so we don't accidentally get the same timestamp - sleep(.5) + sleep(0.5) # Run the second call to append to the file timestamp_2 = str(datetime.now()) @@ -153,4 +165,4 @@ def test_dump_handler_json_append(temp_output_dir: str): assert timestamp_1 in keys assert timestamp_2 in keys assert contents[timestamp_1] == JSON_INFO_TO_DUMP - assert contents[timestamp_2] == JSON_INFO_TO_DUMP \ No newline at end of file + assert contents[timestamp_2] == JSON_INFO_TO_DUMP diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index d797f68c0..3e37cef84 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -1,6 +1,7 @@ """ Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. """ + import os import celery diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index cdb5b2f4f..d857b7ce5 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,6 +1,7 @@ """ Tests for the `sample_index.py` and `sample_index_factory.py` files. """ + import os import pytest diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py index c957ac105..b4cc252d5 100644 --- a/tests/unit/common/test_util_sampling.py +++ b/tests/unit/common/test_util_sampling.py @@ -1,6 +1,7 @@ """ Tests for the `util_sampling.py` file. """ + import numpy as np import pytest diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 8af1dda75..581b19488 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -1,6 +1,7 @@ """ Tests for the `broker.py` file. """ + import os from ssl import CERT_NONE from typing import Any, Dict diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index bd658bc66..64e56b7d9 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -1,6 +1,7 @@ """ Test the functionality of the Config object. """ + from copy import copy, deepcopy from types import SimpleNamespace diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py index aeb1da941..975e19ee4 100644 --- a/tests/unit/config/test_configfile.py +++ b/tests/unit/config/test_configfile.py @@ -1,6 +1,7 @@ """ Tests for the configfile.py module. """ + import getpass import os import shutil diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index 314df6ce7..f49e3e897 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -1,6 +1,7 @@ """ Tests for the `results_backend.py` file. """ + import os from ssl import CERT_NONE from typing import Any, Dict diff --git a/tests/unit/server/test_RedisConfig.py b/tests/unit/server/test_RedisConfig.py index 12880d4d6..321d2f38a 100644 --- a/tests/unit/server/test_RedisConfig.py +++ b/tests/unit/server/test_RedisConfig.py @@ -4,13 +4,16 @@ This class is especially large so that's why these tests have been moved to their own file. """ + import filecmp import logging -import pytest from typing import Any +import pytest + from merlin.server.server_util import RedisConfig + class TestRedisConfig: """Tests for the RedisConfig class.""" @@ -73,10 +76,7 @@ def test_write(self, server_redis_conf_file: str, server_testing_dir: str): # Check that the contents of the copied file match the contents of the basic file assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) - @pytest.mark.parametrize("key, val, expected_return", [ - ("port", 1234, True), - ("invalid_key", "dummy_val", False) - ]) + @pytest.mark.parametrize("key, val, expected_return", [("port", 1234, True), ("invalid_key", "dummy_val", False)]) def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, expected_return: bool): """ Test the `set_config_value` method with valid and invalid keys. @@ -95,17 +95,20 @@ def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, else: assert not redis_config.changes_made() - @pytest.mark.parametrize("key, expected_val", [ - ("bind", "127.0.0.1"), - ("port", "6379"), - ("requirepass", "merlin_password"), - ("dir", "./"), - ("save", "300 100"), - ("dbfilename", "dump.rdb"), - ("appendfsync", "everysec"), - ("appendfilename", "appendonly.aof"), - ("invalid_key", None) - ]) + @pytest.mark.parametrize( + "key, expected_val", + [ + ("bind", "127.0.0.1"), + ("port", "6379"), + ("requirepass", "merlin_password"), + ("dir", "./"), + ("save", "300 100"), + ("dbfilename", "dump.rdb"), + ("appendfsync", "everysec"), + ("appendfilename", "appendonly.aof"), + ("invalid_key", None), + ], + ) def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_val: str): """ Test the `get_config_value` method with valid and invalid keys. @@ -117,18 +120,16 @@ def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_ redis_conf = RedisConfig(server_redis_conf_file) assert redis_conf.get_config_value(key) == expected_val - @pytest.mark.parametrize("ip_to_set", [ - "127.0.0.1", # Most common IP - "0.0.0.0", # Edge case (low) - "255.255.255.255", # Edge case (high) - "123.222.199.20", # Random valid IP - ]) - def test_set_ip_address_valid( - self, - caplog: "Fixture", # noqa: F821 - server_redis_conf_file: str, - ip_to_set: str - ): + @pytest.mark.parametrize( + "ip_to_set", + [ + "127.0.0.1", # Most common IP + "0.0.0.0", # Edge case (low) + "255.255.255.255", # Edge case (high) + "123.222.199.20", # Random valid IP + ], + ) + def test_set_ip_address_valid(self, caplog: "Fixture", server_redis_conf_file: str, ip_to_set: str): # noqa: F821 """ Test the `set_ip_address` method with valid ips. These should all return True and set the 'bind' value to whatever `ip_to_set` is. @@ -143,11 +144,14 @@ def test_set_ip_address_valid( assert f"Ipaddress is set to {ip_to_set}" in caplog.text, "Missing expected log message" assert redis_config.get_ip_address() == ip_to_set - @pytest.mark.parametrize("ip_to_set, expected_log", [ - (None, None), # No IP - ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 - ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist - ]) + @pytest.mark.parametrize( + "ip_to_set, expected_log", + [ + (None, None), # No IP + ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 + ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist + ], + ) def test_set_ip_address_invalid( self, caplog: "Fixture", # noqa: F821 @@ -174,12 +178,15 @@ def test_set_ip_address_invalid( if expected_log is not None: assert expected_log in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("port_to_set", [ - 6379, # Most common port - 1, # Edge case (low) - 65535, # Edge case (high) - 12345, # Random valid port - ]) + @pytest.mark.parametrize( + "port_to_set", + [ + 6379, # Most common port + 1, # Edge case (low) + 65535, # Edge case (high) + 12345, # Random valid port + ], + ) def test_set_port_valid( self, caplog: "Fixture", # noqa: F821 @@ -200,12 +207,15 @@ def test_set_port_valid( assert redis_config.get_port() == port_to_set assert f"Port is set to {port_to_set}" in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("port_to_set, expected_log", [ - (None, None), # No port - (0, "Invalid port given."), # Edge case (low) - (65536, "Invalid port given."), # Edge case (high) - ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist - ]) + @pytest.mark.parametrize( + "port_to_set, expected_log", + [ + (None, None), # No port + (0, "Invalid port given."), # Edge case (low) + (65536, "Invalid port given."), # Edge case (high) + ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist + ], + ) def test_set_port_invalid( self, caplog: "Fixture", # noqa: F821 @@ -232,10 +242,13 @@ def test_set_port_invalid( if expected_log is not None: assert expected_log in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("pass_to_set, expected_return", [ - ("valid_password", True), # Valid password - (None, False), # Invalid password - ]) + @pytest.mark.parametrize( + "pass_to_set, expected_return", + [ + ("valid_password", True), # Valid password + (None, False), # Invalid password + ], + ) def test_set_password( self, caplog: "Fixture", # noqa: F821 @@ -289,7 +302,7 @@ def test_set_directory_none(self, server_redis_conf_file: str): """ redis_config = RedisConfig(server_redis_conf_file) assert not redis_config.set_directory(None) - assert redis_config.get_config_value("dir") != None + assert redis_config.get_config_value("dir") is not None def test_set_directory_dir_unset( self, @@ -327,8 +340,10 @@ def test_set_snapshot_valid(self, caplog: "Fixture", server_redis_conf_file: str save_val = redis_conf.get_config_value("save").split() assert save_val[0] == str(snap_sec_to_set) assert save_val[1] == str(snap_changes_to_set) - expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " \ - f"Snapshot threshold is set to {snap_changes_to_set} changes" + expected_log = ( + f"Snapshot wait time is set to {snap_sec_to_set} seconds. " + f"Snapshot threshold is set to {snap_changes_to_set} changes" + ) assert expected_log in caplog.text, "Missing expected log message" def test_set_snapshot_just_seconds(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 @@ -432,11 +447,14 @@ def test_set_snapshot_file_dbfilename_unset(self, caplog: "Fixture", server_redi assert redis_conf.get_config_value("dbfilename") != filename assert "Unable to set snapshot_file name" in caplog.text, "Missing expected log message" - @pytest.mark.parametrize("mode_to_set", [ - "always", - "everysec", - "no", - ]) + @pytest.mark.parametrize( + "mode_to_set", + [ + "always", + "everysec", + "no", + ], + ) def test_set_append_mode_valid( self, caplog: "Fixture", # noqa: F821 diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py index c9b59e83e..909cb7cdf 100644 --- a/tests/unit/server/test_server_util.py +++ b/tests/unit/server/test_server_util.py @@ -1,12 +1,14 @@ """ Tests for the `server_util.py` module. """ + import filecmp import hashlib import os -import pytest from typing import Dict, Union +import pytest + from merlin.server.server_util import ( AppYaml, ContainerConfig, @@ -16,15 +18,19 @@ RedisUsers, ServerConfig, valid_ipv4, - valid_port + valid_port, ) -@pytest.mark.parametrize("valid_ip", [ - "0.0.0.0", - "127.0.0.1", - "14.105.200.58", - "255.255.255.255", -]) + +@pytest.mark.parametrize( + "valid_ip", + [ + "0.0.0.0", + "127.0.0.1", + "14.105.200.58", + "255.255.255.255", + ], +) def test_valid_ipv4_valid_ip(valid_ip: str): """ Test the `valid_ipv4` function with valid IPs. @@ -35,12 +41,16 @@ def test_valid_ipv4_valid_ip(valid_ip: str): """ assert valid_ipv4(valid_ip) -@pytest.mark.parametrize("invalid_ip", [ - "256.0.0.1", - "-1.0.0.1", - None, - "127.0.01", -]) + +@pytest.mark.parametrize( + "invalid_ip", + [ + "256.0.0.1", + "-1.0.0.1", + None, + "127.0.01", + ], +) def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): """ Test the `valid_ipv4` function with invalid IPs. @@ -52,11 +62,15 @@ def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): """ assert not valid_ipv4(invalid_ip) -@pytest.mark.parametrize("valid_input", [ - 1, - 433, - 65535, -]) + +@pytest.mark.parametrize( + "valid_input", + [ + 1, + 433, + 65535, + ], +) def test_valid_port_valid_input(valid_input: int): """ Test the `valid_port` function with valid port numbers. @@ -68,11 +82,15 @@ def test_valid_port_valid_input(valid_input: int): """ assert valid_port(valid_input) -@pytest.mark.parametrize("invalid_input", [ - -1, - 0, - 65536, -]) + +@pytest.mark.parametrize( + "invalid_input", + [ + -1, + 0, + 65536, + ], +) def test_valid_port_invalid_input(invalid_input: int): """ Test the `valid_port` function with invalid inputs. @@ -121,13 +139,16 @@ def test_init_with_missing_data(self): assert config.pass_file == ContainerConfig.PASSWORD_FILE assert config.user_file == ContainerConfig.USERS_FILE - @pytest.mark.parametrize("attr_name", [ - "image", - "config", - "pfile", - "pass_file", - "user_file", - ]) + @pytest.mark.parametrize( + "attr_name", + [ + "image", + "config", + "pfile", + "pass_file", + "user_file", + ], + ) def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): """ Tests that get_*_path methods construct the correct path. @@ -140,17 +161,20 @@ def test_get_path_methods(self, server_container_config_data: Dict[str, str], at expected_path = os.path.join(server_container_config_data["config_dir"], server_container_config_data[attr_name]) assert get_path_method() == expected_path - @pytest.mark.parametrize("getter_name, expected_attr", [ - ("get_format", "format"), - ("get_image_type", "image_type"), - ("get_image_name", "image"), - ("get_image_url", "url"), - ("get_config_name", "config"), - ("get_config_dir", "config_dir"), - ("get_pfile_name", "pfile"), - ("get_pass_file_name", "pass_file"), - ("get_user_file_name", "user_file"), - ]) + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_format", "format"), + ("get_image_type", "image_type"), + ("get_image_name", "image"), + ("get_image_url", "url"), + ("get_config_name", "config"), + ("get_config_dir", "config_dir"), + ("get_pfile_name", "pfile"), + ("get_pass_file_name", "pass_file"), + ("get_user_file_name", "user_file"), + ], + ) def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ Tests that all getter methods return the correct attribute values. @@ -216,12 +240,15 @@ def test_init_with_missing_data(self): assert config.stop_command == config.STOP_COMMAND assert config.pull_command == config.PULL_COMMAND - @pytest.mark.parametrize("getter_name, expected_attr", [ - ("get_command", "command"), - ("get_run_command", "run_command"), - ("get_stop_command", "stop_command"), - ("get_pull_command", "pull_command"), - ]) + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_command", "command"), + ("get_run_command", "run_command"), + ("get_stop_command", "stop_command"), + ("get_pull_command", "pull_command"), + ], + ) def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ Tests that all getter methods return the correct attribute values. @@ -257,10 +284,13 @@ def test_init_with_missing_data(self): assert config.status == incomplete_data["status"] assert config.kill == config.KILL_COMMAND - @pytest.mark.parametrize("getter_name, expected_attr", [ - ("get_status_command", "status"), - ("get_kill_command", "kill"), - ]) + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_status_command", "status"), + ("get_kill_command", "kill"), + ], + ) def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): """ Tests that all getter methods return the correct attribute values. @@ -304,7 +334,7 @@ def test_init_with_missing_data(self, server_process_config_data: Dict[str, str] class TestRedisUsers: """ Tests for the RedisUsers class. - + TODO add integration test(s) for `apply_to_redis` method of this class. """ @@ -358,7 +388,7 @@ def test_get_user_dict(self): assert val == "on" else: assert val == test_dict[key] - + def test_set_password(self): """Test the `set_password` method of the User class.""" user = RedisUsers.User() diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 5a05e3599..fe7378540 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -1,6 +1,7 @@ """ Tests for the `merlin/examples/generator.py` module. """ + import os from typing import List diff --git a/tests/utils.py b/tests/utils.py index d883b83cd..0b408db54 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,7 @@ """ Utility functions for our test suite. """ + import os from typing import Dict From c1b71f0d5c11ff6aeb8dd7e6181dc2da3e85dd66 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 14:09:18 -0700 Subject: [PATCH 080/201] parametrize setup examples tests --- tests/fixtures/examples.py | 20 + tests/unit/test_examples_generator.py | 572 ++++++++++---------------- 2 files changed, 248 insertions(+), 344 deletions(-) create mode 100644 tests/fixtures/examples.py diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py new file mode 100644 index 000000000..16a2f576d --- /dev/null +++ b/tests/fixtures/examples.py @@ -0,0 +1,20 @@ +""" +Fixtures specifically for help testing the modules in the examples/ directory. +""" + +import os +import pytest + +@pytest.fixture(scope="session") +def examples_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to the examples functionality. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for examples tests + """ + testing_dir = f"{temp_output_dir}/examples_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir \ No newline at end of file diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index fe7378540..3f0f2df9d 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -3,6 +3,7 @@ """ import os +import pytest from typing import List from tabulate import tabulate @@ -83,32 +84,27 @@ def test_gather_all_examples(): assert sorted(actual) == sorted(expected) -def test_write_example_dir(temp_output_dir: str): +def test_write_example_dir(examples_testing_dir: str): """ Test the `write_example` function with the src_path as a directory. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) dir_to_copy = f"{EXAMPLES_DIR}/feature_demo/" + dst_dir = f"{examples_testing_dir}/write_example_dir" + write_example(dir_to_copy, dst_dir) + assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(dst_dir)) - write_example(dir_to_copy, generator_dir) - assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(generator_dir)) - -def test_write_example_file(temp_output_dir: str): +def test_write_example_file(examples_testing_dir: str): """ Test the `write_example` function with the src_path as a file. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - - dst_path = f"{generator_dir}/flux_par.yaml" file_to_copy = f"{EXAMPLES_DIR}/flux/flux_par.yaml" - - write_example(file_to_copy, generator_dir) + dst_path = f"{examples_testing_dir}/flux_par.yaml" + write_example(file_to_copy, dst_path) assert os.path.exists(dst_path) @@ -174,6 +170,8 @@ def test_list_examples(): ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" actual = list_examples() + print(f"expected:\n{expected}") + print(f"actual:\n{actual}") assert actual == expected @@ -185,7 +183,7 @@ def test_setup_example_invalid_name(): assert setup_example("invalid_example_name", None) is None -def test_setup_example_no_outdir(temp_output_dir: str): +def test_setup_example_no_outdir(examples_testing_dir: str): """ Test the `setup_example` function with an invalid example name. This should create a directory with the example name (in this case hello) @@ -194,14 +192,12 @@ def test_setup_example_no_outdir(temp_output_dir: str): the `setup_example` function creates the hello/ subdirectory in a directory with the name of this test (setup_no_outdir). - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ cwd = os.getcwd() # Create the temp path to store this setup and move into that directory - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - setup_example_dir = os.path.join(generator_dir, "setup_no_outdir") + setup_example_dir = os.path.join(examples_testing_dir, "setup_no_outdir") create_dir(setup_example_dir) os.chdir(setup_example_dir) @@ -229,37 +225,226 @@ def test_setup_example_no_outdir(temp_output_dir: str): raise AssertionError from exc -def test_setup_example_outdir_exists(temp_output_dir: str): +def test_setup_example_outdir_exists(examples_testing_dir: str): """ Test the `setup_example` function with an output directory that already exists. This should just return None. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - - assert setup_example("hello", generator_dir) is None - - -##################################### -# Tests for setting up each example # -##################################### - - -def run_setup_example(temp_output_dir: str, example_name: str, example_files: List[str], expected_return: str): + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + assert setup_example("hello", examples_testing_dir) is None + + +@pytest.mark.parametrize( + "example_name, example_files, expected_return", + [ + ( + "feature_demo", + [ + ".gitignore", + "feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ], + "feature_demo", + ), + ( + "flux_local", + [ + "flux_local.yaml", + "flux_par_restart.yaml", + "flux_par.yaml", + "paper.yaml", + "requirements.txt", + "scripts/flux_info.py", + "scripts/hello_sleep.c", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/paper_workers.sbatch", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + "scripts/workers.bsub", + ], + "flux", + ), + ( + "lsf_par", + [ + "lsf_par_srun.yaml", + "lsf_par.yaml", + "scripts/hello.c", + "scripts/make_samples.py", + ], + "lsf", + ), + ( + "slurm_par", + [ + "slurm_par.yaml", + "slurm_par_restart.yaml", + "requirements.txt", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + ], + "slurm", + ), + ( + "hello", + [ + "hello_samples.yaml", + "hello.yaml", + "my_hello.yaml", + "requirements.txt", + "make_samples.py", + ], + "hello", + ), + ( + "hpc_demo", + [ + "hpc_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ], + "hpc_demo", + ), + ( + "iterative_demo", + [ + "iterative_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ], + "iterative_demo", + ), + ( + "null_spec", + [ + "null_spec.yaml", + "null_chain.yaml", + ".gitignore", + "Makefile", + "requirements.txt", + "scripts/aggregate_chain_output.sh", + "scripts/aggregate_output.sh", + "scripts/check_completion.sh", + "scripts/kill_all.sh", + "scripts/launch_chain_job.py", + "scripts/launch_jobs.py", + "scripts/make_samples.py", + "scripts/read_output_chain.py", + "scripts/read_output.py", + "scripts/search.sh", + "scripts/submit_chain.sbatch", + "scripts/submit.sbatch", + ], + "null_spec", + ), + ( + "openfoam_wf", + [ + "openfoam_wf.yaml", + "openfoam_wf_docker_template.yaml", + "README.md", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf", + ), + ( + "openfoam_wf_no_docker", + [ + "openfoam_wf_no_docker.yaml", + "openfoam_wf_no_docker_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf_no_docker", + ), + ( + "openfoam_wf_singularity", + [ + "openfoam_wf_singularity.yaml", + "openfoam_wf_singularity_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf_singularity", + ), + ( + "optimization_basic", + [ + "optimization_basic.yaml", + "requirements.txt", + "template_config.py", + "template_optimization.temp", + "scripts/collector.py", + "scripts/optimizer.py", + "scripts/test_functions.py", + "scripts/visualizer.py", + ], + "optimization", + ), + ( + "remote_feature_demo", + [ + ".gitignore", + "remote_feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ], + "remote_feature_demo", + ), + ("restart", ["restart.yaml", "scripts/make_samples.py"], "restart"), + ("restart_delay", ["restart_delay.yaml", "scripts/make_samples.py"], "restart_delay"), + ], +) +def test_setup_example(examples_testing_dir: str, example_name: str, example_files: List[str], expected_return: str): """ - Helper function to run tests for the `setup_example` function. + Run tests for the `setup_example` function. + Each test will consist of: + 1. The name of the example to setup + 2. A list of files that we're expecting to be setup + 3. The expected return value + Each test is a tuple in the parametrize decorator above this test function. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests :param example_name: The name of the example to setup :param example_files: A list of filenames that should be copied by setup_example :param expected_return: The expected return value from `setup_example` """ # Create the temp path to store this setup - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - setup_example_dir = os.path.join(generator_dir, f"setup_{example_name}") + setup_example_dir = os.path.join(examples_testing_dir, f"setup_{example_name}") # Ensure that the example name is returned actual = setup_example(example_name, setup_example_dir) @@ -271,317 +456,16 @@ def run_setup_example(temp_output_dir: str, example_name: str, example_files: Li assert os.path.exists(file) -def test_setup_example_feature_demo(temp_output_dir: str): - """ - Test the `setup_example` function for the feature_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "feature_demo" - example_files = [ - ".gitignore", - "feature_demo.yaml", - "requirements.txt", - "scripts/features.json", - "scripts/hello_world.py", - "scripts/pgen.py", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_flux(temp_output_dir: str): - """ - Test the `setup_example` function for the flux example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_files = [ - "flux_local.yaml", - "flux_par_restart.yaml", - "flux_par.yaml", - "paper.yaml", - "requirements.txt", - "scripts/flux_info.py", - "scripts/hello_sleep.c", - "scripts/hello.c", - "scripts/make_samples.py", - "scripts/paper_workers.sbatch", - "scripts/test_workers.sbatch", - "scripts/workers.sbatch", - "scripts/workers.bsub", - ] - - run_setup_example(temp_output_dir, "flux_local", example_files, "flux") - - -def test_setup_example_lsf(temp_output_dir: str): - """ - Test the `setup_example` function for the lsf example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - - # TODO should there be a workers.bsub for this example? - example_files = [ - "lsf_par_srun.yaml", - "lsf_par.yaml", - "scripts/hello.c", - "scripts/make_samples.py", - ] - - run_setup_example(temp_output_dir, "lsf_par", example_files, "lsf") - - -def test_setup_example_slurm(temp_output_dir: str): - """ - Test the `setup_example` function for the slurm example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_files = [ - "slurm_par.yaml", - "slurm_par_restart.yaml", - "requirements.txt", - "scripts/hello.c", - "scripts/make_samples.py", - "scripts/test_workers.sbatch", - "scripts/workers.sbatch", - ] - - run_setup_example(temp_output_dir, "slurm_par", example_files, "slurm") - - -def test_setup_example_hello(temp_output_dir: str): - """ - Test the `setup_example` function for the hello example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "hello" - example_files = [ - "hello_samples.yaml", - "hello.yaml", - "my_hello.yaml", - "requirements.txt", - "make_samples.py", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_hpc(temp_output_dir: str): - """ - Test the `setup_example` function for the hpc_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "hpc_demo" - example_files = [ - "hpc_demo.yaml", - "cumulative_sample_processor.py", - "faker_sample.py", - "sample_collector.py", - "sample_processor.py", - "requirements.txt", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_iterative(temp_output_dir: str): - """ - Test the `setup_example` function for the iterative_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "iterative_demo" - example_files = [ - "iterative_demo.yaml", - "cumulative_sample_processor.py", - "faker_sample.py", - "sample_collector.py", - "sample_processor.py", - "requirements.txt", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_null(temp_output_dir: str): - """ - Test the `setup_example` function for the null_spec example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "null_spec" - example_files = [ - "null_spec.yaml", - "null_chain.yaml", - ".gitignore", - "Makefile", - "requirements.txt", - "scripts/aggregate_chain_output.sh", - "scripts/aggregate_output.sh", - "scripts/check_completion.sh", - "scripts/kill_all.sh", - "scripts/launch_chain_job.py", - "scripts/launch_jobs.py", - "scripts/make_samples.py", - "scripts/read_output_chain.py", - "scripts/read_output.py", - "scripts/search.sh", - "scripts/submit_chain.sbatch", - "scripts/submit.sbatch", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_openfoam(temp_output_dir: str): - """ - Test the `setup_example` function for the openfoam_wf example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "openfoam_wf" - example_files = [ - "openfoam_wf.yaml", - "openfoam_wf_docker_template.yaml", - "README.md", - "requirements.txt", - "scripts/make_samples.py", - "scripts/blockMesh_template.txt", - "scripts/cavity_setup.sh", - "scripts/combine_outputs.py", - "scripts/learn.py", - "scripts/mesh_param_script.py", - "scripts/run_openfoam", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_openfoam_no_docker(temp_output_dir: str): - """ - Test the `setup_example` function for the openfoam_wf_no_docker example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "openfoam_wf_no_docker" - example_files = [ - "openfoam_wf_no_docker.yaml", - "openfoam_wf_no_docker_template.yaml", - "requirements.txt", - "scripts/make_samples.py", - "scripts/blockMesh_template.txt", - "scripts/cavity_setup.sh", - "scripts/combine_outputs.py", - "scripts/learn.py", - "scripts/mesh_param_script.py", - "scripts/run_openfoam", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_openfoam_singularity(temp_output_dir: str): - """ - Test the `setup_example` function for the openfoam_wf_singularity example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "openfoam_wf_singularity" - example_files = [ - "openfoam_wf_singularity.yaml", - "openfoam_wf_singularity_template.yaml", - "requirements.txt", - "scripts/make_samples.py", - "scripts/blockMesh_template.txt", - "scripts/cavity_setup.sh", - "scripts/combine_outputs.py", - "scripts/learn.py", - "scripts/mesh_param_script.py", - "scripts/run_openfoam", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_optimization(temp_output_dir: str): - """ - Test the `setup_example` function for the optimization example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_files = [ - "optimization_basic.yaml", - "requirements.txt", - "template_config.py", - "template_optimization.temp", - "scripts/collector.py", - "scripts/optimizer.py", - "scripts/test_functions.py", - "scripts/visualizer.py", - ] - - run_setup_example(temp_output_dir, "optimization_basic", example_files, "optimization") - - -def test_setup_example_remote_feature_demo(temp_output_dir: str): - """ - Test the `setup_example` function for the remote_feature_demo example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "remote_feature_demo" - example_files = [ - ".gitignore", - "remote_feature_demo.yaml", - "requirements.txt", - "scripts/features.json", - "scripts/hello_world.py", - "scripts/pgen.py", - ] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_restart(temp_output_dir: str): - """ - Test the `setup_example` function for the restart example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "restart" - example_files = ["restart.yaml", "scripts/make_samples.py"] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_restart_delay(temp_output_dir: str): - """ - Test the `setup_example` function for the restart_delay example. - - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - """ - example_name = "restart_delay" - example_files = ["restart_delay.yaml", "scripts/make_samples.py"] - - run_setup_example(temp_output_dir, example_name, example_files, example_name) - - -def test_setup_example_simple_chain(temp_output_dir: str): +def test_setup_example_simple_chain(examples_testing_dir: str): """ Test the `setup_example` function for the simple_chain example. + This example just writes a single file so we can't run it in the `test_setup_example` test. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param examples_testing_dir: The path to the the temp output directory for examples tests """ # Create the temp path to store this setup - generator_dir = EXAMPLES_GENERATOR_DIR.format(temp_output_dir=temp_output_dir) - create_dir(generator_dir) - output_file = os.path.join(generator_dir, "simple_chain.yaml") + output_file = os.path.join(examples_testing_dir, "simple_chain.yaml") # Ensure that the example name is returned actual = setup_example("simple_chain", output_file) From bd598f4f696f3b9adb757e61f438bc8e3783b3a5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 14:24:51 -0700 Subject: [PATCH 081/201] sort example output --- merlin/examples/generator.py | 2 +- tests/unit/test_examples_generator.py | 68 +++++++++++++-------------- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index bcdf87b8d..eab07a6a7 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -60,7 +60,7 @@ def gather_example_dirs(): """Get all the example directories""" result = {} - for directory in os.listdir(EXAMPLES_DIR): + for directory in sorted(os.listdir(EXAMPLES_DIR)): result[directory] = directory return result diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 3f0f2df9d..7548c8a49 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -112,44 +112,39 @@ def test_list_examples(): """Test the `list_examples` function to see if it gives us all of the examples that we want.""" expected_headers = ["name", "description"] expected_rows = [ - [ - "openfoam_wf_no_docker", - "A parameter study that includes initializing, running,\n" - "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" - "without using docker.", - ], - [ - "optimization_basic", - "Design Optimization Template\n" - "To use,\n" - "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" - "2. Run the template_config file in current directory using `python template_config.py`\n" - "3. Merlin run as usual (merlin run optimization.yaml)\n" - "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" - "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", - ], ["feature_demo", "Run 10 hello worlds."], ["flux_local", "Run a scan through Merlin/Maestro"], ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], ["flux_par_restart", "A simple ensemble of parallel MPI jobs run by flux."], ["paper_flux", "Use flux to run single core MPI jobs and record timings."], - ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], - ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], - ["restart", "A simple ensemble of with restarts."], - ["restart_delay", "A simple ensemble of with restart delay times."], - ["simple_chain", "test to see that chains are not run in parallel"], - ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], - ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], - ["remote_feature_demo", "Run 10 hello worlds."], ["hello", "a very simple merlin workflow"], ["hello_samples", "a very simple merlin workflow, with samples"], ["hpc_demo", "Demo running a workflow on HPC machines"], + ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], + ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], + [ + "null_chain", + "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" + "May be used to measure overhead in merlin.\n" + "Iterates thru a chain of workflows.", + ], + [ + "null_spec", + "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + ], [ "openfoam_wf", "A parameter study that includes initializing, running,\n" "post-processing, collecting, learning and visualizing OpenFOAM runs\n" "using docker.", ], + [ + "openfoam_wf_no_docker", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" + "without using docker.", + ], [ "openfoam_wf_singularity", "A parameter study that includes initializing, running,\n" @@ -157,21 +152,24 @@ def test_list_examples(): "using singularity.", ], [ - "null_chain", - "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" - "May be used to measure overhead in merlin.\n" - "Iterates thru a chain of workflows.", - ], - [ - "null_spec", - "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + "optimization_basic", + "Design Optimization Template\n" + "To use,\n" + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" + "2. Run the template_config file in current directory using `python template_config.py`\n" + "3. Merlin run as usual (merlin run optimization.yaml)\n" + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", ], - ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ["remote_feature_demo", "Run 10 hello worlds."], + ["restart", "A simple ensemble of with restarts."], + ["restart_delay", "A simple ensemble of with restart delay times."], + ["simple_chain", "test to see that chains are not run in parallel"], + ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" actual = list_examples() - print(f"expected:\n{expected}") - print(f"actual:\n{actual}") assert actual == expected From 0487a9e218e719c7a41c5e3f424af04b61813c55 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 15:27:09 -0700 Subject: [PATCH 082/201] ensure directory is changed back on no outdir test --- tests/unit/test_examples_generator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 7548c8a49..25432ffca 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -170,6 +170,8 @@ def test_list_examples(): ] expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" actual = list_examples() + print(f"actual:\n{actual}") + print(f"expected:\n{expected}") assert actual == expected @@ -221,6 +223,8 @@ def test_setup_example_no_outdir(examples_testing_dir: str): except AssertionError as exc: os.chdir(cwd) raise AssertionError from exc + finally: + os.chdir(cwd) def test_setup_example_outdir_exists(examples_testing_dir: str): From c11e82fe5e8e68c913d23f08168becaf8def1b20 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 16:54:24 -0700 Subject: [PATCH 083/201] sort the specs in examples output --- merlin/examples/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index eab07a6a7..aa7679331 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -90,7 +90,7 @@ def list_examples(): for example_dir in gather_example_dirs(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") - for spec in specs: + for spec in sorted(specs): if "template" in spec: continue with open(spec) as f: # pylint: disable=C0103 From a39e65cfb82009ddbe1504a71bdd21ffdb764027 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 6 Jun 2024 16:56:45 -0700 Subject: [PATCH 084/201] fix lint issues --- tests/fixtures/examples.py | 4 +++- tests/unit/test_examples_generator.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py index 16a2f576d..7c4626e3e 100644 --- a/tests/fixtures/examples.py +++ b/tests/fixtures/examples.py @@ -3,8 +3,10 @@ """ import os + import pytest + @pytest.fixture(scope="session") def examples_testing_dir(temp_output_dir: str) -> str: """ @@ -17,4 +19,4 @@ def examples_testing_dir(temp_output_dir: str) -> str: if not os.path.exists(testing_dir): os.mkdir(testing_dir) - return testing_dir \ No newline at end of file + return testing_dir diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py index 25432ffca..7d4d879fb 100644 --- a/tests/unit/test_examples_generator.py +++ b/tests/unit/test_examples_generator.py @@ -3,9 +3,9 @@ """ import os -import pytest from typing import List +import pytest from tabulate import tabulate from merlin.examples.generator import ( From 031fb0e3f449f5427640858ba30850a02743b0de Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 10 Jun 2024 09:47:35 -0700 Subject: [PATCH 085/201] start writing tests for server config --- merlin/examples/generator.py | 1 - merlin/server/server_config.py | 4 +-- tests/unit/server/test_server_config.py | 43 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 tests/unit/server/test_server_config.py diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index aa7679331..7dea1ba82 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -146,5 +146,4 @@ def setup_example(name, outdir): LOG.info(f"Copying example '{name}' to {outdir}") write_example(src_path, outdir) - print(f"example: {example}") return example diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index f58c7567a..142342d86 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -92,8 +92,8 @@ def generate_password(length, pass_command: str = None) -> str: :return:: string value with given length """ if pass_command: - process = subprocess.run(pass_command.split(), shell=True, stdout=subprocess.PIPE) - return process.stdout + process = subprocess.run(pass_command, shell=True, capture_output=True, text=True) + return process.stdout.strip() characters = list(string.ascii_letters + string.digits + "!@#$%^&*()") diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py new file mode 100644 index 000000000..058e77fcf --- /dev/null +++ b/tests/unit/server/test_server_config.py @@ -0,0 +1,43 @@ +""" +Tests for the `server_config.py` module. +""" + +import string + +from merlin.server.server_config import ( + PASSWORD_LENGTH, + check_process_file_format, + config_merlin_server, + create_server_config, + dump_process_file, + generate_password, + get_server_status, + parse_redis_output, + pull_process_file, + pull_server_config, + pull_server_image, +) + + +def test_generate_password_no_pass_command(): + """ + Test the `generate_password` function with no password command. + This should generate a password of 256 (PASSWORD_LENGTH) random ASCII characters. + """ + generated_password = generate_password(PASSWORD_LENGTH) + assert len(generated_password) == PASSWORD_LENGTH + valid_ascii_chars = string.ascii_letters + string.digits + "!@#$%^&*()" + for ch in generated_password: + assert ch in valid_ascii_chars + + +def test_generate_password_with_pass_command(): + """ + Test the `generate_password` function with no password command. + This should generate a password of 256 (PASSWORD_LENGTH) random ASCII characters. + """ + test_pass = "test-password" + generated_password = generate_password(0, pass_command=f"echo {test_pass}") + assert generated_password == test_pass + + From ed3b4eadca308f75563bc9753649f0a51180467f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 10 Jun 2024 18:23:35 -0700 Subject: [PATCH 086/201] bake in LC_ALL env variable setting for server cmds --- merlin/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/merlin/main.py b/merlin/main.py index 4bb005985..318232131 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -425,6 +425,7 @@ def process_server(args: Namespace): Route to the correct function based on the command given via the CLI """ + os.environ["LC_ALL"] = "C" # Necessary for Redis to configure LOCALE if args.commands == "init": init_server() elif args.commands == "start": From 9dc2bd82ca929c8533e3ef5d99b6b3f70990ca5a Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 10 Jun 2024 18:23:56 -0700 Subject: [PATCH 087/201] add tests for parse_redis_output --- merlin/server/server_config.py | 2 +- tests/unit/server/test_server_config.py | 69 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 142342d86..5b89f2f6a 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -119,7 +119,7 @@ def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: server_init = False redis_config = {} line = redis_stdout.readline() - while line != "" or line is not None: + while line != b"" and line is not None: if not server_init: values = [ln for ln in line.split() if b"=" in ln] for val in values: diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 058e77fcf..08415d011 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -2,7 +2,11 @@ Tests for the `server_config.py` module. """ +import io import string +from typing import Dict, Tuple, Union + +import pytest from merlin.server.server_config import ( PASSWORD_LENGTH, @@ -41,3 +45,68 @@ def test_generate_password_with_pass_command(): assert generated_password == test_pass +@pytest.mark.parametrize( + "line, expected_return", + [ + (None, (False, "None passed as redis output")), + (b"", (False, "Reached end of redis output without seeing 'Ready to accept connections'")), + (b"Ready to accept connections", (True, {})), + (b"aborting", (False, "aborting")), + (b"Fatal error", (False, "Fatal error")), + ], +) +def test_parse_redis_output_with_basic_input(line: Union[None, bytes], expected_return: Tuple[bool, Union[str, Dict]]): + """ + Test the `parse_redis_output` function with basic input. + Here "basic input" means single line input or None as input. + + :param line: The value to pass in as input to `parse_redis_output` + :param expected_return: The expected return value based on what was passed in for `line` + """ + if line is None: + reader_input = None + else: + buffer = io.BytesIO(line) + reader_input = io.BufferedReader(buffer) + actual_return = parse_redis_output(reader_input) + assert expected_return == actual_return + + +@pytest.mark.parametrize( + "lines, expected_config", + [ + ( # Testing setting vars before initialized message + b"port=6379 blah blah server=127.0.0.1\n" + b"Server initialized\n" + b"Ready to accept connections", + {"port": "6379", "server": "127.0.0.1"}, + ), + ( # Testing setting vars after initialized message + b"Server initialized\n" + b"port=6379 blah blah server=127.0.0.1\n" + b"Ready to accept connections", + {}, + ), + ( # Testing setting vars before + after initialized message + b"blah blah max_connections=100 blah" + b"Server initialized\n" + b"port=6379 blah blah server=127.0.0.1\n" + b"Ready to accept connections", + {"max_connections": "100"}, + ), + ], +) +def test_parse_redis_output_with_vars(lines: bytes, expected_config: Tuple[bool, Union[str, Dict]]): + """ + Test the `parse_redis_output` function with input that has variables in lines. + This should set any variable given before the "Server initialized" message is provided. + + We'll test setting vars before the initialized message, after, and both before and after. + + :param lines: The lines to pass in as input to `parse_redis_output` + :param expected_config: The expected config dict based on what was passed in for `lines` + """ + buffer = io.BytesIO(lines) + reader_input = io.BufferedReader(buffer) + _, actual_vars = parse_redis_output(reader_input) + assert expected_config == actual_vars From 459aa212c65a86ba55385ee86a0307436c41ee4f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 26 Jun 2024 11:26:58 -0700 Subject: [PATCH 088/201] fix issue with scope of fixture after rebase --- tests/fixtures/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index 85a01ed99..145e81f51 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -16,7 +16,7 @@ # pylint: disable=redefined-outer-name -@pytest.fixture(scope="class") +@pytest.fixture(scope="session") def status_testing_dir(temp_output_dir: str) -> str: """ A pytest fixture to set up a temporary directory to write files to for testing status. From 5945c36037f1640a931018dce59f2afdffa913b6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 26 Jun 2024 12:15:45 -0700 Subject: [PATCH 089/201] run fix-style --- tests/conftest.py | 1 + tests/fixtures/status.py | 1 + tests/unit/server/test_server_config.py | 16 ++++++---------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index aa0b92c63..2a4b5f169 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,6 +97,7 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi ######### Fixture Definitions ######### ####################################### + @pytest.fixture(scope="session") def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: """ diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index 7e581bb87..39a36f9bf 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -13,6 +13,7 @@ from tests.unit.study.status_test_files import status_test_variables + # pylint: disable=redefined-outer-name diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 08415d011..035e70d60 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -75,20 +75,16 @@ def test_parse_redis_output_with_basic_input(line: Union[None, bytes], expected_ @pytest.mark.parametrize( "lines, expected_config", [ - ( # Testing setting vars before initialized message - b"port=6379 blah blah server=127.0.0.1\n" - b"Server initialized\n" - b"Ready to accept connections", + ( # Testing setting vars before initialized message + b"port=6379 blah blah server=127.0.0.1\nServer initialized\nReady to accept connections", {"port": "6379", "server": "127.0.0.1"}, ), - ( # Testing setting vars after initialized message - b"Server initialized\n" - b"port=6379 blah blah server=127.0.0.1\n" - b"Ready to accept connections", + ( # Testing setting vars after initialized message + b"Server initialized\nport=6379 blah blah server=127.0.0.1\nReady to accept connections", {}, ), - ( # Testing setting vars before + after initialized message - b"blah blah max_connections=100 blah" + ( # Testing setting vars before + after initialized message + b"blah blah max_connections=100 blah\n" b"Server initialized\n" b"port=6379 blah blah server=127.0.0.1\n" b"Ready to accept connections", From 78ef6199542e52dbfeabb62b2c3464501b52d9cf Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 23 Jul 2024 12:29:35 -0700 Subject: [PATCH 090/201] Include celerymanager and update celeryadapter to check the status of celery workers. --- merlin/study/celeryadapter.py | 9 ++ merlin/study/celerymanager.py | 144 +++++++++++++++++++++++++++ merlin/study/celerymanageradapter.py | 60 +++++++++++ 3 files changed, 213 insertions(+) create mode 100644 merlin/study/celerymanager.py create mode 100644 merlin/study/celerymanageradapter.py diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 5b5bdd419..11f1244a2 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -47,6 +47,7 @@ from merlin.common.dumper import dump_handler from merlin.config import Config from merlin.study.batch import batch_check_parallel, batch_worker_launch +from merlin.study.celerymanageradapter import add_monitor_workers, remove_monitor_workers from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running @@ -762,6 +763,13 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): """ try: _ = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 + # Get the worker name from worker_cmd and add to be monitored by celery manager + worker_cmd_list = worker_cmd.split() + worker_name = worker_cmd_list[worker_cmd_list.index("-n")+1].replace("%h", kwargs["env"]["HOSTNAME"]) + worker_name = "celery@" + worker_name + add_monitor_workers(workers=(worker_name, )) + LOG.info(f"Added {worker_name} to be monitored") + worker_list.append(worker_cmd) except Exception as e: # pylint: disable=C0103 LOG.error(f"Cannot start celery workers, {e}") @@ -866,6 +874,7 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): if workers_to_stop: LOG.info(f"Sending stop to these workers: {workers_to_stop}") app.control.broadcast("shutdown", destination=workers_to_stop) + remove_monitor_workers(workers=workers_to_stop) else: LOG.warning("No workers found to stop") diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py new file mode 100644 index 000000000..a5310ba94 --- /dev/null +++ b/merlin/study/celerymanager.py @@ -0,0 +1,144 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.12.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + +from merlin.config.configfile import CONFIG +from merlin.config.results_backend import get_backend_password +import os +import redis +import time + + +class WorkerStatus: + running = "Running" + stalled = "Stalled" + stopped = "Stopped" + rebooting = "Rebooting" + +WORKER_INFO = { + "status" : WorkerStatus.running, + "monitored": 1, + "num_unresponsive": 0, +} + +class CeleryManager(): + + def __init__(self, query_frequency=60, query_timeout=0.5, worker_timeout=180): + self.redis_connection = self.get_worker_status_redis_connection() + self.query_frequency = query_frequency + self.query_timeout = query_timeout + self.worker_timeout = worker_timeout + + @staticmethod + def get_worker_status_redis_connection(): + return CeleryManager.get_redis_connection(1) + + @staticmethod + def get_worker_args_redis_connection(): + return CeleryManager.get_redis_connection(2) + + @staticmethod + def get_redis_connection(db_num): + password_file = CONFIG.results_backend.password + try: + password = get_backend_password(password_file) + except IOError: + password = CONFIG.results_backend.password + return redis.Redis(host=CONFIG.results_backend.server, + port=CONFIG.results_backend.port, + db=db_num, + username=CONFIG.results_backend.username, + password=password) + + def get_celery_worker_status(worker): + pass + + def restart_celery_worker(worker): + pass + + def check_pid(pid): + """ Check For the existence of a unix pid. """ + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + + def run(self): + manager_info = { + "status": "Running", + "process id": os.getpid(), + } + self.redis_connection.hmset(name="manager", mapping=manager_info) + + + + + #while True: + # Get the list of running workers + workers = [i.decode("ascii") for i in self.redis_connection.keys()] + workers.remove("manager") + workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] + print("Current Monitored Workers", workers) + + # Check/ Ping each worker to see if they are still running + if workers: + from merlin.celery import app + + celery_app = app.control + ping_result = celery_app.ping(workers, timeout=self.query_timeout) + worker_results = {worker: status for d in ping_result for worker, status in d.items()} + print("Worker result from ping", worker_results) + + # If running set the status on redis that it is running + for worker in list(worker_results.keys()): + self.redis_connection.hset(worker, "status", WorkerStatus.running) + + # If not running attempt to restart it + for worker in workers: + if worker not in worker_results: + # If time where the worker is unresponsive is less than the worker time out then just increment + num_unresponsive = int(self.redis_connection.hget(worker, "num_unresponsive"))+1 + if num_unresponsive*self.query_frequency < self.worker_timeout: + # Attempt to restart worker + + # If successful set the status to running + + # If failed set the status to stopped + #TODO Try to restart the worker + continue + else: + self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) + + #time.sleep(self.query_frequency) + +if __name__ == "__main__": + cm = CeleryManager() + cm.run() \ No newline at end of file diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py new file mode 100644 index 000000000..3f410ee55 --- /dev/null +++ b/merlin/study/celerymanageradapter.py @@ -0,0 +1,60 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.12.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +from merlin.study.celerymanager import CeleryManager, WORKER_INFO + +def add_monitor_workers(workers: list): + if workers is None or len(workers) <= 0: + return + + redis_connection = CeleryManager.get_worker_status_redis_connection() + for worker in workers: + if redis_connection.exists(worker): + redis_connection.hset(worker, "monitored", 1) + redis_connection.hmset(name=worker, mapping=WORKER_INFO) + redis_connection.quit() + +def remove_monitor_workers(workers: list): + if workers is None or len(workers) <= 0: + return + redis_connection = CeleryManager.get_worker_status_redis_connection() + for worker in workers: + if redis_connection.exists(worker): + redis_connection.hset(worker, "monitored", 0) + redis_connection.quit() + +def is_manager_runnning() -> bool: + pass + +def start_manager() -> bool: + pass + +def stop_manager() -> bool: + pass + From 8561a18b4dad9884d928645b35ee2d85bd98ceb0 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 23 Jul 2024 12:37:17 -0700 Subject: [PATCH 091/201] Fixed issue where the update status was outside of if statement for checking workers --- merlin/study/celerymanager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index a5310ba94..abaa15d71 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -117,9 +117,9 @@ def run(self): worker_results = {worker: status for d in ping_result for worker, status in d.items()} print("Worker result from ping", worker_results) - # If running set the status on redis that it is running - for worker in list(worker_results.keys()): - self.redis_connection.hset(worker, "status", WorkerStatus.running) + # If running set the status on redis that it is running + for worker in list(worker_results.keys()): + self.redis_connection.hset(worker, "status", WorkerStatus.running) # If not running attempt to restart it for worker in workers: From 1120dd7cb64bc3e1704d9a0d7763cf47b9bbd5b5 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Thu, 1 Aug 2024 10:31:41 -0700 Subject: [PATCH 092/201] Include worker status stop and add template for merlin restart --- merlin/study/celeryadapter.py | 17 +++++++-- merlin/study/celerymanager.py | 57 ++++++++++++++++++++++------ merlin/study/celerymanageradapter.py | 17 +++++++-- 3 files changed, 71 insertions(+), 20 deletions(-) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 11f1244a2..4dad1efd3 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -47,6 +47,7 @@ from merlin.common.dumper import dump_handler from merlin.config import Config from merlin.study.batch import batch_check_parallel, batch_worker_launch +from merlin.study.celerymanager import CeleryManager from merlin.study.celerymanageradapter import add_monitor_workers, remove_monitor_workers from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running @@ -762,15 +763,23 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): :side effect: Launches a celery worker via a subprocess """ try: - _ = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 + process = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 # Get the worker name from worker_cmd and add to be monitored by celery manager worker_cmd_list = worker_cmd.split() worker_name = worker_cmd_list[worker_cmd_list.index("-n")+1].replace("%h", kwargs["env"]["HOSTNAME"]) worker_name = "celery@" + worker_name - add_monitor_workers(workers=(worker_name, )) - LOG.info(f"Added {worker_name} to be monitored") - worker_list.append(worker_cmd) + + # Adding the worker args to redis db + redis_connection = CeleryManager.get_worker_args_redis_connection() + args = kwargs['env'] + args["worker_cmd"] = worker_cmd + redis_connection.hmset(name=worker_name, mapping=args) + redis_connection.quit() + + # Adding the worker to redis db to be monitored + add_monitor_workers(workers=((worker_name, process.pid), )) + LOG.info(f"Added {worker_name} to be monitored") except Exception as e: # pylint: disable=C0103 LOG.error(f"Cannot start celery workers, {e}") raise diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index abaa15d71..7061b2df7 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -32,6 +32,7 @@ from merlin.config.results_backend import get_backend_password import os import redis +import subprocess import time @@ -43,6 +44,7 @@ class WorkerStatus: WORKER_INFO = { "status" : WorkerStatus.running, + "pid": -1, "monitored": 1, "num_unresponsive": 0, } @@ -74,13 +76,48 @@ def get_redis_connection(db_num): port=CONFIG.results_backend.port, db=db_num, username=CONFIG.results_backend.username, - password=password) + password=password, + decode_responses=True) - def get_celery_worker_status(worker): - pass + def get_celery_workers_status(self, workers): + from merlin.celery import app - def restart_celery_worker(worker): - pass + celery_app = app.control + ping_result = celery_app.ping(workers, timeout=self.query_timeout) + worker_results = {worker: status for d in ping_result for worker, status in d.items()} + print("Worker result from ping", worker_results) + return worker_results + + def stop_celery_worker(self, worker): + """ + Stop a celery worker by first broadcasting shutdown. If unsuccessful kill the worker with pid + :param CeleryManager self: CeleryManager attempting the stop. + :param str worker: Worker that is being stopped. + """ + from merlin.celery import app + + app.control.broadcast("shutdown", destination=(worker, )) + + + + def restart_celery_worker(self, worker): + # Stop the worker that is currently running + + + # Start the worker again with the args saved in redis db + worker_args_connect = self.get_worker_args_redis_connection() + worker_status_connect = self.get_worker_status_redis_connection() + # Get the args and remove the worker_cmd from the hash set + args = worker_args_connect.hgetall(worker) + worker_cmd = args["worker_cmd"] + del args["worker_cmd"] + # Run the subprocess for the worker and save the PID + process = subprocess.Popen(worker_cmd, *args) + worker_status_connect.hset(worker, "pid", process.pid) + + worker_args_connect.quit() + worker_status_connect.quit() + def check_pid(pid): """ Check For the existence of a unix pid. """ @@ -103,19 +140,15 @@ def run(self): #while True: # Get the list of running workers - workers = [i.decode("ascii") for i in self.redis_connection.keys()] + workers = self.redis_connection.keys() workers.remove("manager") workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] print("Current Monitored Workers", workers) + self.restart_celery_worker(workers[0]) # Check/ Ping each worker to see if they are still running if workers: - from merlin.celery import app - - celery_app = app.control - ping_result = celery_app.ping(workers, timeout=self.query_timeout) - worker_results = {worker: status for d in ping_result for worker, status in d.items()} - print("Worker result from ping", worker_results) + worker_results = self.get_celery_workers_status(workers) # If running set the status on redis that it is running for worker in list(worker_results.keys()): diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index 3f410ee55..f990e740a 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -27,17 +27,24 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### -from merlin.study.celerymanager import CeleryManager, WORKER_INFO +from merlin.study.celerymanager import CeleryManager, WORKER_INFO, WorkerStatus def add_monitor_workers(workers: list): + """ + Adds a worker to be monitored by the celery manager. + :param list workers: A list of tuples which includes (worker_name, pid) + """ if workers is None or len(workers) <= 0: return redis_connection = CeleryManager.get_worker_status_redis_connection() for worker in workers: - if redis_connection.exists(worker): - redis_connection.hset(worker, "monitored", 1) - redis_connection.hmset(name=worker, mapping=WORKER_INFO) + if redis_connection.exists(worker[0]): + redis_connection.hset(worker[0], "monitored", 1) + redis_connection.hset(worker[0], "pid", worker[1]) + worker_info = WORKER_INFO + worker_info["pid"] = worker[1] + redis_connection.hmset(name=worker[0], mapping=worker_info) redis_connection.quit() def remove_monitor_workers(workers: list): @@ -47,6 +54,8 @@ def remove_monitor_workers(workers: list): for worker in workers: if redis_connection.exists(worker): redis_connection.hset(worker, "monitored", 0) + redis_connection.hset(worker, "status", WorkerStatus.stopped) + redis_connection.quit() def is_manager_runnning() -> bool: From f41938faf3e2356f0ee0f261c79effe8e58352de Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Thu, 1 Aug 2024 17:17:43 -0700 Subject: [PATCH 093/201] Added comment to the CeleryManager init --- merlin/study/celerymanager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 7061b2df7..1800e76e2 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -51,7 +51,13 @@ class WorkerStatus: class CeleryManager(): - def __init__(self, query_frequency=60, query_timeout=0.5, worker_timeout=180): + def __init__(self, query_frequency:int=60, query_timeout:float=0.5, worker_timeout:int=180): + """ + Initializer for Celery Manager + @param int query_frequency: The frequency at which workers will be queried with ping commands + @param float query_timeout: The timeout for the query pings that are sent to workers + @param int worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. + """ self.redis_connection = self.get_worker_status_redis_connection() self.query_frequency = query_frequency self.query_timeout = query_timeout @@ -98,8 +104,6 @@ def stop_celery_worker(self, worker): app.control.broadcast("shutdown", destination=(worker, )) - - def restart_celery_worker(self, worker): # Stop the worker that is currently running From 690115e7fb6ea2a51f39730fbc7899e8bfbb74ea Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Thu, 1 Aug 2024 17:20:26 -0700 Subject: [PATCH 094/201] Increment db_num instead of being fixed --- merlin/study/celerymanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 1800e76e2..53accf3a6 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -80,7 +80,7 @@ def get_redis_connection(db_num): password = CONFIG.results_backend.password return redis.Redis(host=CONFIG.results_backend.server, port=CONFIG.results_backend.port, - db=db_num, + db=CONFIG.results_backend.db_num+db_num, #Increment db_num to avoid conflicts username=CONFIG.results_backend.username, password=password, decode_responses=True) From de4ffd02ccbd0a428dbae97d67ad12ffd45945eb Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Thu, 1 Aug 2024 18:09:08 -0700 Subject: [PATCH 095/201] Added other subprocess parameters and created a linking system for redis to store env dict --- merlin/study/celeryadapter.py | 16 +++++++++++++++- merlin/study/celerymanager.py | 12 ++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 4dad1efd3..860b30b41 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -772,8 +772,22 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): # Adding the worker args to redis db redis_connection = CeleryManager.get_worker_args_redis_connection() - args = kwargs['env'] + args = kwargs + # Save worker command with the arguements args["worker_cmd"] = worker_cmd + # Store the nested dictionaries into a separate key with a link. + # Note: This only support single nested dicts(for simplicity) and + # further nesting can be accomplished by making this recursive. + for key in kwargs: + if type(kwargs[key]) is dict: + key_name = worker_name+"_"+key + redis_connection.hmset(name=key_name, mapping=kwargs[key]) + args[key] = "link:"+key_name + if type(kwargs[key]) is bool: + if kwargs[key]: + args[key] = "True" + else: + args[key] = "False" redis_connection.hmset(name=worker_name, mapping=args) redis_connection.quit() diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 53accf3a6..3ed3437a2 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -115,8 +115,17 @@ def restart_celery_worker(self, worker): args = worker_args_connect.hgetall(worker) worker_cmd = args["worker_cmd"] del args["worker_cmd"] + kwargs = args + for key in args: + if args[key].startswith("link:"): + kwargs[key] = worker_args_connect.hgetall(args[key].split(":", 1)[1]) + elif args[key] == "True": + kwargs[key] = True + elif args[key] == "False": + kwargs[key] = False + # Run the subprocess for the worker and save the PID - process = subprocess.Popen(worker_cmd, *args) + process = subprocess.Popen(worker_cmd, **kwargs) worker_status_connect.hset(worker, "pid", process.pid) worker_args_connect.quit() @@ -148,7 +157,6 @@ def run(self): workers.remove("manager") workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] print("Current Monitored Workers", workers) - self.restart_celery_worker(workers[0]) # Check/ Ping each worker to see if they are still running if workers: From 67e9268afd3098afe4cf9d4eb0a487f52400e4ea Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 16:12:46 -0700 Subject: [PATCH 096/201] Implemented stopping of celery workers and restarting workers properly --- merlin/study/celerymanager.py | 112 +++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 3ed3437a2..1b30f2027 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -31,6 +31,7 @@ from merlin.config.configfile import CONFIG from merlin.config.results_backend import get_backend_password import os +import psutil import redis import subprocess import time @@ -91,23 +92,42 @@ def get_celery_workers_status(self, workers): celery_app = app.control ping_result = celery_app.ping(workers, timeout=self.query_timeout) worker_results = {worker: status for d in ping_result for worker, status in d.items()} - print("Worker result from ping", worker_results) return worker_results def stop_celery_worker(self, worker): """ - Stop a celery worker by first broadcasting shutdown. If unsuccessful kill the worker with pid + Stop a celery worker by kill the worker with pid :param CeleryManager self: CeleryManager attempting the stop. :param str worker: Worker that is being stopped. + + :return bool: The result of whether a worker was stopped. """ - from merlin.celery import app - app.control.broadcast("shutdown", destination=(worker, )) + # Get the PID associated with the pid + worker_status_connect = self.get_worker_status_redis_connection() + worker_pid = int(worker_status_connect.hget(worker, "pid")) + # Check to see if the pid exists + if psutil.pid_exists(worker_pid): + # Check to see if the pid is associated with celery + worker_process = psutil.Process(worker_pid) + if "celery" in worker_process.name(): + # Kill the pid if both conditions are right + worker_process.kill() + return True + return False def restart_celery_worker(self, worker): - # Stop the worker that is currently running + """ + Restart a celery worker with the same arguements and parameters during its creation + :param CeleryManager self: CeleryManager attempting the stop. + :param str worker: Worker that is being restarted. + :return bool: The result of whether a worker was restarted. + """ + # Stop the worker that is currently running + if not self.stop_celery_worker(worker): + return False # Start the worker again with the args saved in redis db worker_args_connect = self.get_worker_args_redis_connection() worker_status_connect = self.get_worker_status_redis_connection() @@ -130,59 +150,53 @@ def restart_celery_worker(self, worker): worker_args_connect.quit() worker_status_connect.quit() - - def check_pid(pid): - """ Check For the existence of a unix pid. """ - try: - os.kill(pid, 0) - except OSError: - return False - else: - return True + return True + def run(self): + """ + Main manager loop + """ + manager_info = { "status": "Running", "process id": os.getpid(), } self.redis_connection.hmset(name="manager", mapping=manager_info) - - - - #while True: - # Get the list of running workers - workers = self.redis_connection.keys() - workers.remove("manager") - workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] - print("Current Monitored Workers", workers) - - # Check/ Ping each worker to see if they are still running - if workers: - worker_results = self.get_celery_workers_status(workers) - - # If running set the status on redis that it is running - for worker in list(worker_results.keys()): - self.redis_connection.hset(worker, "status", WorkerStatus.running) - - # If not running attempt to restart it - for worker in workers: - if worker not in worker_results: - # If time where the worker is unresponsive is less than the worker time out then just increment - num_unresponsive = int(self.redis_connection.hget(worker, "num_unresponsive"))+1 - if num_unresponsive*self.query_frequency < self.worker_timeout: - # Attempt to restart worker - - # If successful set the status to running - - # If failed set the status to stopped - #TODO Try to restart the worker - continue - else: - self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) - - #time.sleep(self.query_frequency) + while True: #TODO Make it so that it will stop after a list of workers is stopped + # Get the list of running workers + workers = self.redis_connection.keys() + workers.remove("manager") + workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] + + # Check/ Ping each worker to see if they are still running + if workers: + worker_results = self.get_celery_workers_status(workers) + + # If running set the status on redis that it is running + for worker in list(worker_results.keys()): + self.redis_connection.hset(worker, "status", WorkerStatus.running) + + # If not running attempt to restart it + for worker in workers: + if worker not in worker_results: + # If time where the worker is unresponsive is less than the worker time out then just increment + num_unresponsive = int(self.redis_connection.hget(worker, "num_unresponsive"))+1 + if num_unresponsive*self.query_frequency < self.worker_timeout: + # Attempt to restart worker + if self.restart_celery_worker(worker): + # If successful set the status to running and reset num_unresponsive + self.redis_connection.hset(worker, "status", WorkerStatus.running) + self.redis_connection.hset(worker, "num_unresponsive", 0) + # If failed set the status to stopped + self.redis_connection.hset(worker, "status", WorkerStatus.stopped) + else: + self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) + # Sleep for the query_frequency for the next iteration + print("Finished checking") + time.sleep(self.query_frequency) if __name__ == "__main__": cm = CeleryManager() From 406e4c2b280d86aa3fa1eca5630ed9342bf21489 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 16:21:15 -0700 Subject: [PATCH 097/201] Update stopped to stalled for when the worker doesn't respond to restart --- merlin/study/celerymanager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 1b30f2027..3f4c37459 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -106,8 +106,10 @@ def stop_celery_worker(self, worker): # Get the PID associated with the pid worker_status_connect = self.get_worker_status_redis_connection() worker_pid = int(worker_status_connect.hget(worker, "pid")) - # Check to see if the pid exists - if psutil.pid_exists(worker_pid): + worker_status = worker_status_connect.hget(worker, "status") + worker_status_connect.quit() + # Check to see if the pid exists and worker is set as running + if worker_status == WorkerStatus.running and psutil.pid_exists(worker_pid): # Check to see if the pid is associated with celery worker_process = psutil.Process(worker_pid) if "celery" in worker_process.name(): @@ -190,8 +192,8 @@ def run(self): # If successful set the status to running and reset num_unresponsive self.redis_connection.hset(worker, "status", WorkerStatus.running) self.redis_connection.hset(worker, "num_unresponsive", 0) - # If failed set the status to stopped - self.redis_connection.hset(worker, "status", WorkerStatus.stopped) + # If failed set the status to stalled + self.redis_connection.hset(worker, "status", WorkerStatus.stalled) else: self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) # Sleep for the query_frequency for the next iteration From 78e45254ea79b42f8323b601c9a7ea438961f486 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 18:25:19 -0700 Subject: [PATCH 098/201] Working merlin manager run but start and stop not working properly --- merlin/main.py | 101 +++++++++++++++++++++++++++ merlin/study/celerymanager.py | 4 +- merlin/study/celerymanageradapter.py | 48 +++++++++++-- 3 files changed, 146 insertions(+), 7 deletions(-) diff --git a/merlin/main.py b/merlin/main.py index 4bb005985..aba27251c 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -57,6 +57,7 @@ from merlin.server.server_commands import config_server, init_server, restart_server, start_server, status_server, stop_server from merlin.spec.expansion import RESERVED, get_spec_with_expansion from merlin.spec.specification import MerlinSpec +from merlin.study.celerymanageradapter import run_manager, start_manager, stop_manager from merlin.study.status import DetailedStatus, Status from merlin.study.status_constants import VALID_RETURN_CODES, VALID_STATUS_FILTERS from merlin.study.status_renderers import status_renderer_factory @@ -400,6 +401,26 @@ def process_example(args: Namespace) -> None: setup_example(args.workflow, args.path) +def process_manager(args : Namespace): + if args.command == "run": + run_manager(query_frequency=args.query_frequency, + query_timeout=args.query_timeout, + worker_timeout=args.worker_timeout) + elif args.command == "start": + if start_manager(query_frequency=args.query_frequency, + query_timeout=args.query_timeout, + worker_timeout=args.worker_timeout): + LOG.info("Manager started successfully.") + elif args.command == "stop": + if stop_manager(): + LOG.info("Manager stopped successfully.") + else: + LOG.error("Unable to stop manager.") + else: + print("Run manager with a command. Try 'merlin manager -h' for more details") + + + def process_monitor(args): """ CLI command to monitor merlin workers and queues to keep @@ -897,6 +918,86 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: help="regex match for specific workers to stop", ) + # merlin manager + manager : ArgumentParser = subparsers.add_parser( + "manager", + help="Watchdog application to manage workers", + description="A daemon process that helps to restart and communicate with workers while running.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + manager.set_defaults(func=process_manager) + + manager_commands: ArgumentParser = manager.add_subparsers(dest="command") + manager_run = manager_commands.add_parser( + "run", + help="Run the daemon process", + description="Run manager", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + manager_run.add_argument( + "-qf", + "--query_frequency", + action="store", + type=int, + default=60, + help="The frequency at which workers will be queried for response.", + ) + manager_run.add_argument( + "-qt", + "--query_timeout", + action="store", + type=float, + default=0.5, + help="The timeout for the query response that are sent to workers.", + ) + manager_run.add_argument( + "-wt", + "--worker_timeout", + action="store", + type=int, + default=180, + help="The sum total(query_frequency*tries) time before an attempt is made to restart worker.", + ) + manager_run.set_defaults(func=process_manager) + manager_start = manager_commands.add_parser( + "start", + help="Start the daemon process", + description="Start manager", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + manager_start.add_argument( + "-qf", + "--query_frequency", + action="store", + type=int, + default=60, + help="The frequency at which workers will be queried for response.", + ) + manager_start.add_argument( + "-qt", + "--query_timeout", + action="store", + type=float, + default=0.5, + help="The timeout for the query response that are sent to workers.", + ) + manager_start.add_argument( + "-wt", + "--worker_timeout", + action="store", + type=int, + default=180, + help="The sum total(query_frequency*tries) time before an attempt is made to restart worker.", + ) + manager_start.set_defaults(func=process_manager) + manager_stop = manager_commands.add_parser( + "stop", + help="Stop the daemon process", + description="Stop manager", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + manager_stop.set_defaults(func=process_manager) + # merlin monitor monitor: ArgumentParser = subparsers.add_parser( "monitor", diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 3f4c37459..0a2ffaccf 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -155,7 +155,7 @@ def restart_celery_worker(self, worker): return True - + #TODO add some logs def run(self): """ Main manager loop @@ -172,6 +172,7 @@ def run(self): workers = self.redis_connection.keys() workers.remove("manager") workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] + print(f"Monitoring {workers} workers") # Check/ Ping each worker to see if they are still running if workers: @@ -197,7 +198,6 @@ def run(self): else: self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) # Sleep for the query_frequency for the next iteration - print("Finished checking") time.sleep(self.query_frequency) if __name__ == "__main__": diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index f990e740a..f703b099f 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -28,10 +28,12 @@ # SOFTWARE. ############################################################################### from merlin.study.celerymanager import CeleryManager, WORKER_INFO, WorkerStatus +import psutil +import subprocess def add_monitor_workers(workers: list): """ - Adds a worker to be monitored by the celery manager. + Adds workers to be monitored by the celery manager. :param list workers: A list of tuples which includes (worker_name, pid) """ if workers is None or len(workers) <= 0: @@ -48,6 +50,10 @@ def add_monitor_workers(workers: list): redis_connection.quit() def remove_monitor_workers(workers: list): + """ + Remove workers from being monitored by the celery manager. + :param list workers: A worker names + """ if workers is None or len(workers) <= 0: return redis_connection = CeleryManager.get_worker_status_redis_connection() @@ -59,11 +65,43 @@ def remove_monitor_workers(workers: list): redis_connection.quit() def is_manager_runnning() -> bool: - pass + """ + Check to see if the manager is running -def start_manager() -> bool: - pass + :return: True if manager is running and False if not. + """ + redis_connection = CeleryManager.get_worker_args_redis_connection() + manager_status = redis_connection.hgetall("manager") + redis_connection.quit() + return manager_status["status"] == WorkerStatus.running and psutil.pid_exists(manager_status["pid"]) + +def run_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: + celerymanager = CeleryManager(query_frequency=query_frequency, + query_timeout=query_timeout, + worker_timeout=worker_timeout) + celerymanager.run() + + +def start_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: + process = subprocess.Popen(f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}".split(), + start_new_session=True, + close_fds=True, + stdout=subprocess.PIPE, + ) + redis_connection = CeleryManager.get_worker_args_redis_connection() + redis_connection.hset("manager", "pid", process.pid) + redis_connection.quit() + return True def stop_manager() -> bool: - pass + redis_connection = CeleryManager.get_worker_args_redis_connection() + manager_pid = redis_connection.hget("manager", "pid") + manager_status = redis_connection.hget("manager", "status") + redis_connection.quit() + + # Check to make sure that the manager is running and the pid exists + if manager_status == WorkerStatus.running and psutil.pid_exists(manager_pid): + psutil.Process(manager_pid).terminate() + return True + return False From eca74ac53fb4ff91b71c64930888413d2dd6eb17 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 18:40:39 -0700 Subject: [PATCH 099/201] Made fix for subprocess to start new shell and fixed manager start and stop --- merlin/study/celerymanageradapter.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index f703b099f..c98f801a1 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -83,22 +83,21 @@ def run_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_time def start_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: - process = subprocess.Popen(f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}".split(), - start_new_session=True, + process = subprocess.Popen(f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", + shell=True, close_fds=True, stdout=subprocess.PIPE, ) - redis_connection = CeleryManager.get_worker_args_redis_connection() - redis_connection.hset("manager", "pid", process.pid) - redis_connection.quit() return True def stop_manager() -> bool: - redis_connection = CeleryManager.get_worker_args_redis_connection() - manager_pid = redis_connection.hget("manager", "pid") + redis_connection = CeleryManager.get_worker_status_redis_connection() + manager_pid = int(redis_connection.hget("manager", "pid")) manager_status = redis_connection.hget("manager", "status") + print(redis_connection.hgetall("manager")) redis_connection.quit() + print(manager_status, psutil.pid_exists(manager_pid)) # Check to make sure that the manager is running and the pid exists if manager_status == WorkerStatus.running and psutil.pid_exists(manager_pid): psutil.Process(manager_pid).terminate() From ec8aa789953781faf586e1fbd46943cac7ca36f9 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 18:55:33 -0700 Subject: [PATCH 100/201] Added comments and update changelog --- CHANGELOG.md | 4 ++++ merlin/study/celerymanager.py | 19 +++++++++++++++++++ merlin/study/celerymanageradapter.py | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b4427b1..f0760df79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Added +- Merlin manager capability to monitor celery workers. + ## [1.12.2b1] ### Added - Conflict handler option to the `dict_deep_merge` function in `utils.py` diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 0a2ffaccf..41bbc1cdc 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -66,14 +66,26 @@ def __init__(self, query_frequency:int=60, query_timeout:float=0.5, worker_timeo @staticmethod def get_worker_status_redis_connection(): + """ + Get the redis connection for info regarding the worker and manager status. + """ return CeleryManager.get_redis_connection(1) @staticmethod def get_worker_args_redis_connection(): + """ + Get the redis connection for info regarding the args used to generate each worker. + """ return CeleryManager.get_redis_connection(2) @staticmethod def get_redis_connection(db_num): + """ + Generic redis connection function to get the results backend redis server with a given db number increment. + :param int db_num: Increment number for the db from the one provided in the config file. + + :return Redis: Redis connections object that can be used to access values for the manager. + """ password_file = CONFIG.results_backend.password try: password = get_backend_password(password_file) @@ -87,6 +99,13 @@ def get_redis_connection(db_num): decode_responses=True) def get_celery_workers_status(self, workers): + """ + Get the worker status of a current worker that is being managed + :param CeleryManager self: CeleryManager attempting the stop. + :param list workers: Workers that are checked. + + :return dict: The result dictionary for each worker and the response. + """ from merlin.celery import app celery_app = app.control diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index c98f801a1..e6fba6ca8 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -76,6 +76,10 @@ def is_manager_runnning() -> bool: return manager_status["status"] == WorkerStatus.running and psutil.pid_exists(manager_status["pid"]) def run_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: + """ + A process locking function that calls the celery manager with proper arguments. + :params: See CeleryManager for more information regarding the parameters + """ celerymanager = CeleryManager(query_frequency=query_frequency, query_timeout=query_timeout, worker_timeout=worker_timeout) @@ -83,7 +87,13 @@ def run_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_time def start_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: - process = subprocess.Popen(f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", + """ + A Non-locking function that calls the celery manager with proper arguments. + :params: See CeleryManager for more information regarding the parameters + + :return bool: True if the manager was started successfully. + """ + subprocess.Popen(f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", shell=True, close_fds=True, stdout=subprocess.PIPE, @@ -91,6 +101,11 @@ def start_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_ti return True def stop_manager() -> bool: + """ + Stop the manager process using it's pid. + + :return bool: True if the manager was stopped successfully and False otherwise. + """ redis_connection = CeleryManager.get_worker_status_redis_connection() manager_pid = int(redis_connection.hget("manager", "pid")) manager_status = redis_connection.hget("manager", "status") From 3f04d24cfcea09a3c2be86b12ebbb56ebc962c7f Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 19:00:49 -0700 Subject: [PATCH 101/201] Include style fixes --- merlin/study/celerymanager.py | 10 ++++++---- merlin/study/celerymanageradapter.py | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 41bbc1cdc..3d33f5333 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -28,14 +28,16 @@ # SOFTWARE. ############################################################################### -from merlin.config.configfile import CONFIG -from merlin.config.results_backend import get_backend_password import os -import psutil -import redis import subprocess import time +import psutil +import redis + +from merlin.config.configfile import CONFIG +from merlin.config.results_backend import get_backend_password + class WorkerStatus: running = "Running" diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index e6fba6ca8..6ec94f5bd 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -27,10 +27,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### -from merlin.study.celerymanager import CeleryManager, WORKER_INFO, WorkerStatus -import psutil import subprocess +import psutil + +from merlin.study.celerymanager import WORKER_INFO, CeleryManager, WorkerStatus + + def add_monitor_workers(workers: list): """ Adds workers to be monitored by the celery manager. From 5538f4ba6d3898e716748ff1f32aab3e2f8a42d2 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Aug 2024 19:07:25 -0700 Subject: [PATCH 102/201] Fix style for black --- .../null_spec/scripts/launch_jobs.py | 4 +- merlin/main.py | 15 +++---- merlin/study/celeryadapter.py | 8 ++-- merlin/study/celerymanager.py | 40 ++++++++++--------- merlin/study/celerymanageradapter.py | 24 ++++++----- 5 files changed, 46 insertions(+), 45 deletions(-) diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index a6b6d1372..99c3c3d6a 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -78,9 +78,7 @@ if real_time > 1440: real_time = 1440 submit: str = "submit.sbatch" - command: str = ( - f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" - ) + command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) diff --git a/merlin/main.py b/merlin/main.py index aba27251c..e6bb7c0ce 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -401,15 +401,13 @@ def process_example(args: Namespace) -> None: setup_example(args.workflow, args.path) -def process_manager(args : Namespace): +def process_manager(args: Namespace): if args.command == "run": - run_manager(query_frequency=args.query_frequency, - query_timeout=args.query_timeout, - worker_timeout=args.worker_timeout) + run_manager(query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout) elif args.command == "start": - if start_manager(query_frequency=args.query_frequency, - query_timeout=args.query_timeout, - worker_timeout=args.worker_timeout): + if start_manager( + query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout + ): LOG.info("Manager started successfully.") elif args.command == "stop": if stop_manager(): @@ -420,7 +418,6 @@ def process_manager(args : Namespace): print("Run manager with a command. Try 'merlin manager -h' for more details") - def process_monitor(args): """ CLI command to monitor merlin workers and queues to keep @@ -919,7 +916,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: ) # merlin manager - manager : ArgumentParser = subparsers.add_parser( + manager: ArgumentParser = subparsers.add_parser( "manager", help="Watchdog application to manage workers", description="A daemon process that helps to restart and communicate with workers while running.", diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 860b30b41..6c09590a4 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -766,7 +766,7 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): process = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 # Get the worker name from worker_cmd and add to be monitored by celery manager worker_cmd_list = worker_cmd.split() - worker_name = worker_cmd_list[worker_cmd_list.index("-n")+1].replace("%h", kwargs["env"]["HOSTNAME"]) + worker_name = worker_cmd_list[worker_cmd_list.index("-n") + 1].replace("%h", kwargs["env"]["HOSTNAME"]) worker_name = "celery@" + worker_name worker_list.append(worker_cmd) @@ -780,9 +780,9 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): # further nesting can be accomplished by making this recursive. for key in kwargs: if type(kwargs[key]) is dict: - key_name = worker_name+"_"+key + key_name = worker_name + "_" + key redis_connection.hmset(name=key_name, mapping=kwargs[key]) - args[key] = "link:"+key_name + args[key] = "link:" + key_name if type(kwargs[key]) is bool: if kwargs[key]: args[key] = "True" @@ -792,7 +792,7 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): redis_connection.quit() # Adding the worker to redis db to be monitored - add_monitor_workers(workers=((worker_name, process.pid), )) + add_monitor_workers(workers=((worker_name, process.pid),)) LOG.info(f"Added {worker_name} to be monitored") except Exception as e: # pylint: disable=C0103 LOG.error(f"Cannot start celery workers, {e}") diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 3d33f5333..795ca10fb 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -45,27 +45,28 @@ class WorkerStatus: stopped = "Stopped" rebooting = "Rebooting" + WORKER_INFO = { - "status" : WorkerStatus.running, + "status": WorkerStatus.running, "pid": -1, "monitored": 1, "num_unresponsive": 0, } -class CeleryManager(): - def __init__(self, query_frequency:int=60, query_timeout:float=0.5, worker_timeout:int=180): +class CeleryManager: + def __init__(self, query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180): """ Initializer for Celery Manager @param int query_frequency: The frequency at which workers will be queried with ping commands @param float query_timeout: The timeout for the query pings that are sent to workers - @param int worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. + @param int worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. """ self.redis_connection = self.get_worker_status_redis_connection() self.query_frequency = query_frequency self.query_timeout = query_timeout self.worker_timeout = worker_timeout - + @staticmethod def get_worker_status_redis_connection(): """ @@ -93,12 +94,14 @@ def get_redis_connection(db_num): password = get_backend_password(password_file) except IOError: password = CONFIG.results_backend.password - return redis.Redis(host=CONFIG.results_backend.server, - port=CONFIG.results_backend.port, - db=CONFIG.results_backend.db_num+db_num, #Increment db_num to avoid conflicts - username=CONFIG.results_backend.username, - password=password, - decode_responses=True) + return redis.Redis( + host=CONFIG.results_backend.server, + port=CONFIG.results_backend.port, + db=CONFIG.results_backend.db_num + db_num, # Increment db_num to avoid conflicts + username=CONFIG.results_backend.username, + password=password, + decode_responses=True, + ) def get_celery_workers_status(self, workers): """ @@ -108,7 +111,7 @@ def get_celery_workers_status(self, workers): :return dict: The result dictionary for each worker and the response. """ - from merlin.celery import app + from merlin.celery import app celery_app = app.control ping_result = celery_app.ping(workers, timeout=self.query_timeout) @@ -176,7 +179,7 @@ def restart_celery_worker(self, worker): return True - #TODO add some logs + # TODO add some logs def run(self): """ Main manager loop @@ -188,7 +191,7 @@ def run(self): } self.redis_connection.hmset(name="manager", mapping=manager_info) - while True: #TODO Make it so that it will stop after a list of workers is stopped + while True: # TODO Make it so that it will stop after a list of workers is stopped # Get the list of running workers workers = self.redis_connection.keys() workers.remove("manager") @@ -207,8 +210,8 @@ def run(self): for worker in workers: if worker not in worker_results: # If time where the worker is unresponsive is less than the worker time out then just increment - num_unresponsive = int(self.redis_connection.hget(worker, "num_unresponsive"))+1 - if num_unresponsive*self.query_frequency < self.worker_timeout: + num_unresponsive = int(self.redis_connection.hget(worker, "num_unresponsive")) + 1 + if num_unresponsive * self.query_frequency < self.worker_timeout: # Attempt to restart worker if self.restart_celery_worker(worker): # If successful set the status to running and reset num_unresponsive @@ -220,7 +223,8 @@ def run(self): self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) # Sleep for the query_frequency for the next iteration time.sleep(self.query_frequency) - + + if __name__ == "__main__": cm = CeleryManager() - cm.run() \ No newline at end of file + cm.run() diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index 6ec94f5bd..a433a8cac 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -41,7 +41,7 @@ def add_monitor_workers(workers: list): """ if workers is None or len(workers) <= 0: return - + redis_connection = CeleryManager.get_worker_status_redis_connection() for worker in workers: if redis_connection.exists(worker[0]): @@ -52,6 +52,7 @@ def add_monitor_workers(workers: list): redis_connection.hmset(name=worker[0], mapping=worker_info) redis_connection.quit() + def remove_monitor_workers(workers: list): """ Remove workers from being monitored by the celery manager. @@ -64,9 +65,10 @@ def remove_monitor_workers(workers: list): if redis_connection.exists(worker): redis_connection.hset(worker, "monitored", 0) redis_connection.hset(worker, "status", WorkerStatus.stopped) - + redis_connection.quit() + def is_manager_runnning() -> bool: """ Check to see if the manager is running @@ -78,31 +80,32 @@ def is_manager_runnning() -> bool: redis_connection.quit() return manager_status["status"] == WorkerStatus.running and psutil.pid_exists(manager_status["pid"]) -def run_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: + +def run_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180) -> bool: """ A process locking function that calls the celery manager with proper arguments. :params: See CeleryManager for more information regarding the parameters """ - celerymanager = CeleryManager(query_frequency=query_frequency, - query_timeout=query_timeout, - worker_timeout=worker_timeout) + celerymanager = CeleryManager(query_frequency=query_frequency, query_timeout=query_timeout, worker_timeout=worker_timeout) celerymanager.run() - -def start_manager(query_frequency:int = 60, query_timeout:float = 0.5, worker_timeout:int = 180) -> bool: + +def start_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180) -> bool: """ A Non-locking function that calls the celery manager with proper arguments. :params: See CeleryManager for more information regarding the parameters :return bool: True if the manager was started successfully. """ - subprocess.Popen(f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", + subprocess.Popen( + f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", shell=True, close_fds=True, stdout=subprocess.PIPE, ) return True + def stop_manager() -> bool: """ Stop the manager process using it's pid. @@ -114,11 +117,10 @@ def stop_manager() -> bool: manager_status = redis_connection.hget("manager", "status") print(redis_connection.hgetall("manager")) redis_connection.quit() - + print(manager_status, psutil.pid_exists(manager_pid)) # Check to make sure that the manager is running and the pid exists if manager_status == WorkerStatus.running and psutil.pid_exists(manager_pid): psutil.Process(manager_pid).terminate() return True return False - From b6bcd3323256ef44d19fa0378569c03a9e877305 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 7 Aug 2024 14:00:04 -0700 Subject: [PATCH 103/201] Revert launch_job script that was edited when doing automated lint --- merlin/examples/workflows/null_spec/scripts/launch_jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index 99c3c3d6a..a6b6d1372 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -78,7 +78,9 @@ if real_time > 1440: real_time = 1440 submit: str = "submit.sbatch" - command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" + command: str = ( + f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" + ) shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) From 9b97f8bcc4794c8efed5dc98e52eb56db9f4ee99 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 7 Aug 2024 14:07:15 -0700 Subject: [PATCH 104/201] Move importing of CONFIG to be within redis_connection due to error of config not being created yet --- merlin/study/celerymanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 795ca10fb..7335ae0a5 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -35,7 +35,6 @@ import psutil import redis -from merlin.config.configfile import CONFIG from merlin.config.results_backend import get_backend_password @@ -89,6 +88,7 @@ def get_redis_connection(db_num): :return Redis: Redis connections object that can be used to access values for the manager. """ + from merlin.config.configfile import CONFIG password_file = CONFIG.results_backend.password try: password = get_backend_password(password_file) From c9dfd312f4bcc08b1444a8193f8dad62ff31215f Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 7 Aug 2024 14:08:55 -0700 Subject: [PATCH 105/201] Added space to fix style --- merlin/examples/workflows/null_spec/scripts/launch_jobs.py | 4 +--- merlin/study/celerymanager.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index a6b6d1372..99c3c3d6a 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -78,9 +78,7 @@ if real_time > 1440: real_time = 1440 submit: str = "submit.sbatch" - command: str = ( - f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" - ) + command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 7335ae0a5..ebc1924f4 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -89,6 +89,7 @@ def get_redis_connection(db_num): :return Redis: Redis connections object that can be used to access values for the manager. """ from merlin.config.configfile import CONFIG + password_file = CONFIG.results_backend.password try: password = get_backend_password(password_file) From a9bd86564491295858de1dd5ee1a7ca2a619c93b Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 7 Aug 2024 14:09:43 -0700 Subject: [PATCH 106/201] Revert launch_jobs.py: --- merlin/examples/workflows/null_spec/scripts/launch_jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index 99c3c3d6a..a6b6d1372 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -78,7 +78,9 @@ if real_time > 1440: real_time = 1440 submit: str = "submit.sbatch" - command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" + command: str = ( + f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" + ) shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) From ddc76142b61c4d2837a60339965d8a8d004ed95f Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 7 Aug 2024 14:13:05 -0700 Subject: [PATCH 107/201] Update import of all merlin.config to be in the function --- merlin/study/celerymanager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index ebc1924f4..6f2e697a6 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -35,8 +35,6 @@ import psutil import redis -from merlin.config.results_backend import get_backend_password - class WorkerStatus: running = "Running" @@ -89,6 +87,7 @@ def get_redis_connection(db_num): :return Redis: Redis connections object that can be used to access values for the manager. """ from merlin.config.configfile import CONFIG + from merlin.config.results_backend import get_backend_password password_file = CONFIG.results_backend.password try: From 353a66b8497d43771bc03cf8d48cd68a1b62f590 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 16 Aug 2024 17:12:47 -0700 Subject: [PATCH 108/201] suggested changes plus beginning work on monitor/manager collab --- merlin/main.py | 96 +++++----- merlin/study/celeryadapter.py | 58 +++--- merlin/study/celerymanager.py | 268 +++++++++++++++------------ merlin/study/celerymanageradapter.py | 72 ++++--- 4 files changed, 276 insertions(+), 218 deletions(-) diff --git a/merlin/main.py b/merlin/main.py index e6bb7c0ce..46683d273 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -402,6 +402,15 @@ def process_example(args: Namespace) -> None: def process_manager(args: Namespace): + """ + Process the command for managing the workers. + + This function interprets the command provided in the `args` namespace and + executes the corresponding manager function. It supports three commands: + "run", "start", and "stop". + + :param args: parsed CLI arguments + """ if args.command == "run": run_manager(query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout) elif args.command == "start": @@ -409,6 +418,8 @@ def process_manager(args: Namespace): query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout ): LOG.info("Manager started successfully.") + else: + LOG.error("Unable to start manager") elif args.command == "stop": if stop_manager(): LOG.info("Manager stopped successfully.") @@ -924,6 +935,41 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: ) manager.set_defaults(func=process_manager) + def add_manager_options(manager_parser: ArgumentParser): + """ + Add shared options for manager subcommands. + + The `manager run` and `manager start` subcommands have the same options. + Rather than writing duplicate code for these we'll use this function + to add the arguments to these subcommands. + + :param manager_parser: The ArgumentParser object to add these options to + """ + manager_parser.add_argument( + "-qf", + "--query_frequency", + action="store", + type=int, + default=60, + help="The frequency at which workers will be queried for response.", + ) + manager_parser.add_argument( + "-qt", + "--query_timeout", + action="store", + type=float, + default=0.5, + help="The timeout for the query response that are sent to workers.", + ) + manager_parser.add_argument( + "-wt", + "--worker_timeout", + action="store", + type=int, + default=180, + help="The sum total (query_frequency*tries) time before an attempt is made to restart worker.", + ) + manager_commands: ArgumentParser = manager.add_subparsers(dest="command") manager_run = manager_commands.add_parser( "run", @@ -931,30 +977,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: description="Run manager", formatter_class=ArgumentDefaultsHelpFormatter, ) - manager_run.add_argument( - "-qf", - "--query_frequency", - action="store", - type=int, - default=60, - help="The frequency at which workers will be queried for response.", - ) - manager_run.add_argument( - "-qt", - "--query_timeout", - action="store", - type=float, - default=0.5, - help="The timeout for the query response that are sent to workers.", - ) - manager_run.add_argument( - "-wt", - "--worker_timeout", - action="store", - type=int, - default=180, - help="The sum total(query_frequency*tries) time before an attempt is made to restart worker.", - ) + add_manager_options(manager_run) manager_run.set_defaults(func=process_manager) manager_start = manager_commands.add_parser( "start", @@ -962,30 +985,7 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: description="Start manager", formatter_class=ArgumentDefaultsHelpFormatter, ) - manager_start.add_argument( - "-qf", - "--query_frequency", - action="store", - type=int, - default=60, - help="The frequency at which workers will be queried for response.", - ) - manager_start.add_argument( - "-qt", - "--query_timeout", - action="store", - type=float, - default=0.5, - help="The timeout for the query response that are sent to workers.", - ) - manager_start.add_argument( - "-wt", - "--worker_timeout", - action="store", - type=int, - default=180, - help="The sum total(query_frequency*tries) time before an attempt is made to restart worker.", - ) + add_manager_options(manager_start) manager_start.set_defaults(func=process_manager) manager_stop = manager_commands.add_parser( "stop", diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 6c09590a4..e392b1795 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -502,15 +502,22 @@ def check_celery_workers_processing(queues_in_spec: List[str], app: Celery) -> b """ # Query celery for active tasks active_tasks = app.control.inspect().active() + result = False - # Search for the queues we provided if necessary - if active_tasks is not None: - for tasks in active_tasks.values(): - for task in tasks: - if task["delivery_info"]["routing_key"] in queues_in_spec: - return True + with CeleryManager.get_worker_status_redis_connection() as redis_connection: + # Search for the queues we provided if necessary + if active_tasks is not None: + for worker, tasks in active_tasks.items(): + for task in tasks: + if task["delivery_info"]["routing_key"] in queues_in_spec: + result = True - return False + # Set the entry in the Redis DB for the manager to signify if the worker + # is still doing work + worker_still_processing = 1 if result else 0 + redis_connection.hset(worker, "processing_work", worker_still_processing) + + return result def _get_workers_to_start(spec, steps): @@ -771,25 +778,24 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): worker_list.append(worker_cmd) # Adding the worker args to redis db - redis_connection = CeleryManager.get_worker_args_redis_connection() - args = kwargs - # Save worker command with the arguements - args["worker_cmd"] = worker_cmd - # Store the nested dictionaries into a separate key with a link. - # Note: This only support single nested dicts(for simplicity) and - # further nesting can be accomplished by making this recursive. - for key in kwargs: - if type(kwargs[key]) is dict: - key_name = worker_name + "_" + key - redis_connection.hmset(name=key_name, mapping=kwargs[key]) - args[key] = "link:" + key_name - if type(kwargs[key]) is bool: - if kwargs[key]: - args[key] = "True" - else: - args[key] = "False" - redis_connection.hmset(name=worker_name, mapping=args) - redis_connection.quit() + with CeleryManager.get_worker_args_redis_connection() as redis_connection: + args = kwargs + # Save worker command with the arguements + args["worker_cmd"] = worker_cmd + # Store the nested dictionaries into a separate key with a link. + # Note: This only support single nested dicts(for simplicity) and + # further nesting can be accomplished by making this recursive. + for key in kwargs: + if type(kwargs[key]) is dict: + key_name = worker_name + "_" + key + redis_connection.hmset(name=key_name, mapping=kwargs[key]) + args[key] = "link:" + key_name + if type(kwargs[key]) is bool: + if kwargs[key]: + args[key] = "True" + else: + args[key] = "False" + redis_connection.hmset(name=worker_name, mapping=args) # Adding the worker to redis db to be monitored add_monitor_workers(workers=((worker_name, process.pid),)) diff --git a/merlin/study/celerymanager.py b/merlin/study/celerymanager.py index 6f2e697a6..ddefab02c 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/study/celerymanager.py @@ -27,7 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### - +import logging import os import subprocess import time @@ -36,6 +36,9 @@ import redis +LOG = logging.getLogger(__name__) + + class WorkerStatus: running = "Running" stalled = "Stalled" @@ -48,68 +51,89 @@ class WorkerStatus: "pid": -1, "monitored": 1, "num_unresponsive": 0, + "processing_work": 1, } +class RedisConnectionManager: + """ + A context manager for handling redis connections. + This will ensure safe opening and closing of Redis connections. + """ + + def __init__(self, db_num: int): + self.db_num = db_num + self.connection = None + + def __enter__(self): + self.connection = self.get_redis_connection() + return self.connection + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.connection: + LOG.debug(f"MANAGER: Closing connection at db_num: {self.db_num}") + self.connection.close() + + def get_redis_connection(self) -> redis.Redis: + """ + Generic redis connection function to get the results backend redis server with a given db number increment. + + :return: Redis connection object that can be used to access values for the manager. + """ + # from merlin.config.results_backend import get_backend_password + from merlin.config import results_backend + from merlin.config.configfile import CONFIG + + conn_string = results_backend.get_connection_string() + base, _ = conn_string.rsplit("/", 1) + new_db_num = CONFIG.results_backend.db_num + self.db_num + conn_string = f"{base}/{new_db_num}" + LOG.debug(f"MANAGER: Connecting to redis at db_num: {new_db_num}") + return redis.from_url(conn_string, decode_responses=True) + # password_file = CONFIG.results_backend.password + # try: + # password = get_backend_password(password_file) + # except IOError: + # password = CONFIG.results_backend.password + # return redis.Redis( + # host=CONFIG.results_backend.server, + # port=CONFIG.results_backend.port, + # db=CONFIG.results_backend.db_num + self.db_num, # Increment db_num to avoid conflicts + # username=CONFIG.results_backend.username, + # password=password, + # decode_responses=True, + # ) + + class CeleryManager: def __init__(self, query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180): """ Initializer for Celery Manager - @param int query_frequency: The frequency at which workers will be queried with ping commands - @param float query_timeout: The timeout for the query pings that are sent to workers - @param int worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. + + :param query_frequency: The frequency at which workers will be queried with ping commands + :param query_timeout: The timeout for the query pings that are sent to workers + :param worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. """ - self.redis_connection = self.get_worker_status_redis_connection() self.query_frequency = query_frequency self.query_timeout = query_timeout self.worker_timeout = worker_timeout @staticmethod - def get_worker_status_redis_connection(): - """ - Get the redis connection for info regarding the worker and manager status. - """ - return CeleryManager.get_redis_connection(1) - - @staticmethod - def get_worker_args_redis_connection(): - """ - Get the redis connection for info regarding the args used to generate each worker. - """ - return CeleryManager.get_redis_connection(2) + def get_worker_status_redis_connection() -> RedisConnectionManager: + """Get the redis connection for info regarding the worker and manager status.""" + return RedisConnectionManager(1) @staticmethod - def get_redis_connection(db_num): - """ - Generic redis connection function to get the results backend redis server with a given db number increment. - :param int db_num: Increment number for the db from the one provided in the config file. + def get_worker_args_redis_connection() -> RedisConnectionManager: + """Get the redis connection for info regarding the args used to generate each worker.""" + return RedisConnectionManager(2) - :return Redis: Redis connections object that can be used to access values for the manager. - """ - from merlin.config.configfile import CONFIG - from merlin.config.results_backend import get_backend_password - - password_file = CONFIG.results_backend.password - try: - password = get_backend_password(password_file) - except IOError: - password = CONFIG.results_backend.password - return redis.Redis( - host=CONFIG.results_backend.server, - port=CONFIG.results_backend.port, - db=CONFIG.results_backend.db_num + db_num, # Increment db_num to avoid conflicts - username=CONFIG.results_backend.username, - password=password, - decode_responses=True, - ) - - def get_celery_workers_status(self, workers): + def get_celery_workers_status(self, workers: list) -> dict: """ Get the worker status of a current worker that is being managed - :param CeleryManager self: CeleryManager attempting the stop. - :param list workers: Workers that are checked. - :return dict: The result dictionary for each worker and the response. + :param workers: Workers that are checked. + :return: The result dictionary for each worker and the response. """ from merlin.celery import app @@ -118,20 +142,19 @@ def get_celery_workers_status(self, workers): worker_results = {worker: status for d in ping_result for worker, status in d.items()} return worker_results - def stop_celery_worker(self, worker): + def stop_celery_worker(self, worker: str) -> bool: """ Stop a celery worker by kill the worker with pid - :param CeleryManager self: CeleryManager attempting the stop. - :param str worker: Worker that is being stopped. - :return bool: The result of whether a worker was stopped. + :param worker: Worker that is being stopped. + :return: The result of whether a worker was stopped. """ - # Get the PID associated with the pid - worker_status_connect = self.get_worker_status_redis_connection() - worker_pid = int(worker_status_connect.hget(worker, "pid")) - worker_status = worker_status_connect.hget(worker, "status") - worker_status_connect.quit() + # Get the PID associated with the worker + with self.get_worker_status_redis_connection() as worker_status_connect: + worker_pid = int(worker_status_connect.hget(worker, "pid")) + worker_status = worker_status_connect.hget(worker, "status") + # Check to see if the pid exists and worker is set as running if worker_status == WorkerStatus.running and psutil.pid_exists(worker_pid): # Check to see if the pid is associated with celery @@ -142,87 +165,100 @@ def stop_celery_worker(self, worker): return True return False - def restart_celery_worker(self, worker): + def restart_celery_worker(self, worker: str) -> bool: """ Restart a celery worker with the same arguements and parameters during its creation - :param CeleryManager self: CeleryManager attempting the stop. - :param str worker: Worker that is being restarted. - :return bool: The result of whether a worker was restarted. + :param worker: Worker that is being restarted. + :return: The result of whether a worker was restarted. """ # Stop the worker that is currently running if not self.stop_celery_worker(worker): return False + # Start the worker again with the args saved in redis db - worker_args_connect = self.get_worker_args_redis_connection() - worker_status_connect = self.get_worker_status_redis_connection() - # Get the args and remove the worker_cmd from the hash set - args = worker_args_connect.hgetall(worker) - worker_cmd = args["worker_cmd"] - del args["worker_cmd"] - kwargs = args - for key in args: - if args[key].startswith("link:"): - kwargs[key] = worker_args_connect.hgetall(args[key].split(":", 1)[1]) - elif args[key] == "True": - kwargs[key] = True - elif args[key] == "False": - kwargs[key] = False - - # Run the subprocess for the worker and save the PID - process = subprocess.Popen(worker_cmd, **kwargs) - worker_status_connect.hset(worker, "pid", process.pid) - - worker_args_connect.quit() - worker_status_connect.quit() + with ( + self.get_worker_args_redis_connection() as worker_args_connect, + self.get_worker_status_redis_connection() as worker_status_connect, + ): + # Get the args and remove the worker_cmd from the hash set + args = worker_args_connect.hgetall(worker) + worker_cmd = args["worker_cmd"] + del args["worker_cmd"] + kwargs = args + for key in args: + if args[key].startswith("link:"): + kwargs[key] = worker_args_connect.hgetall(args[key].split(":", 1)[1]) + elif args[key] == "True": + kwargs[key] = True + elif args[key] == "False": + kwargs[key] = False + + # Run the subprocess for the worker and save the PID + process = subprocess.Popen(worker_cmd, **kwargs) + worker_status_connect.hset(worker, "pid", process.pid) return True - # TODO add some logs def run(self): """ - Main manager loop - """ + Main manager loop for monitoring and managing Celery workers. + This method continuously monitors the status of Celery workers by + checking their health and attempting to restart any that are + unresponsive. It updates the Redis database with the current + status of the manager and the workers. + """ manager_info = { "status": "Running", - "process id": os.getpid(), + "pid": os.getpid(), } - self.redis_connection.hmset(name="manager", mapping=manager_info) - - while True: # TODO Make it so that it will stop after a list of workers is stopped - # Get the list of running workers - workers = self.redis_connection.keys() - workers.remove("manager") - workers = [worker for worker in workers if int(self.redis_connection.hget(worker, "monitored"))] - print(f"Monitoring {workers} workers") - - # Check/ Ping each worker to see if they are still running - if workers: - worker_results = self.get_celery_workers_status(workers) - - # If running set the status on redis that it is running - for worker in list(worker_results.keys()): - self.redis_connection.hset(worker, "status", WorkerStatus.running) - - # If not running attempt to restart it - for worker in workers: - if worker not in worker_results: - # If time where the worker is unresponsive is less than the worker time out then just increment - num_unresponsive = int(self.redis_connection.hget(worker, "num_unresponsive")) + 1 - if num_unresponsive * self.query_frequency < self.worker_timeout: - # Attempt to restart worker - if self.restart_celery_worker(worker): - # If successful set the status to running and reset num_unresponsive - self.redis_connection.hset(worker, "status", WorkerStatus.running) - self.redis_connection.hset(worker, "num_unresponsive", 0) - # If failed set the status to stalled - self.redis_connection.hset(worker, "status", WorkerStatus.stalled) - else: - self.redis_connection.hset(worker, "num_unresponsive", num_unresponsive) - # Sleep for the query_frequency for the next iteration - time.sleep(self.query_frequency) + + with self.get_worker_status_redis_connection() as redis_connection: + LOG.debug(f"MANAGER: setting manager key in redis to hold the following info {manager_info}") + redis_connection.hmset(name="manager", mapping=manager_info) + + # TODO figure out what to do with "processing_work" entry for the merlin monitor + while True: # TODO Make it so that it will stop after a list of workers is stopped + # Get the list of running workers + workers = redis_connection.keys() + LOG.debug(f"MANAGER: workers: {workers}") + workers.remove("manager") + workers = [worker for worker in workers if int(redis_connection.hget(worker, "monitored"))] + LOG.info(f"MANAGER: Monitoring {workers} workers") + + # Check/ Ping each worker to see if they are still running + if workers: + worker_results = self.get_celery_workers_status(workers) + + # If running set the status on redis that it is running + LOG.info(f"MANAGER: Responsive workers: {worker_results.keys()}") + for worker in list(worker_results.keys()): + redis_connection.hset(worker, "status", WorkerStatus.running) + + # If not running attempt to restart it + for worker in workers: + if worker not in worker_results: + LOG.info(f"MANAGER: Worker '{worker}' is unresponsive.") + # If time where the worker is unresponsive is less than the worker time out then just increment + num_unresponsive = int(redis_connection.hget(worker, "num_unresponsive")) + 1 + if num_unresponsive * self.query_frequency < self.worker_timeout: + # Attempt to restart worker + LOG.info(f"MANAGER: Attempting to restart worker '{worker}'...") + if self.restart_celery_worker(worker): + # If successful set the status to running and reset num_unresponsive + redis_connection.hset(worker, "status", WorkerStatus.running) + redis_connection.hset(worker, "num_unresponsive", 0) + # If failed set the status to stalled + redis_connection.hset(worker, "status", WorkerStatus.stalled) + LOG.info(f"MANAGER: Worker '{worker}' restarted.") + else: + LOG.error(f"MANAGER: Could not restart worker '{worker}'.") + else: + redis_connection.hset(worker, "num_unresponsive", num_unresponsive) + # Sleep for the query_frequency for the next iteration + time.sleep(self.query_frequency) if __name__ == "__main__": diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index a433a8cac..d195eb966 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -27,6 +27,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### +import logging import subprocess import psutil @@ -34,6 +35,9 @@ from merlin.study.celerymanager import WORKER_INFO, CeleryManager, WorkerStatus +LOG = logging.getLogger(__name__) + + def add_monitor_workers(workers: list): """ Adds workers to be monitored by the celery manager. @@ -42,15 +46,25 @@ def add_monitor_workers(workers: list): if workers is None or len(workers) <= 0: return - redis_connection = CeleryManager.get_worker_status_redis_connection() - for worker in workers: - if redis_connection.exists(worker[0]): - redis_connection.hset(worker[0], "monitored", 1) - redis_connection.hset(worker[0], "pid", worker[1]) - worker_info = WORKER_INFO - worker_info["pid"] = worker[1] - redis_connection.hmset(name=worker[0], mapping=worker_info) - redis_connection.quit() + LOG.info( + f"MANAGER: Attempting to have the manager monitor the following workers {[worker_name for worker_name, _ in workers]}." + ) + monitored_workers = [] + + with CeleryManager.get_worker_status_redis_connection() as redis_connection: + for worker in workers: + LOG.debug(f"MANAGER: Checking if connection for worker '{worker}' exists...") + if redis_connection.exists(worker[0]): + LOG.debug(f"MANAGER: Connection for worker '{worker}' exists. Setting this worker to be monitored") + redis_connection.hset(worker[0], "monitored", 1) + redis_connection.hset(worker[0], "pid", worker[1]) + monitored_workers.append(worker[0]) + else: + LOG.debug(f"MANAGER: Connection for worker '{worker}' does not exist. Not monitoring this worker.") + worker_info = WORKER_INFO + worker_info["pid"] = worker[1] + redis_connection.hmset(name=worker[0], mapping=worker_info) + LOG.info(f"MANAGER: Manager is monitoring the following workers {monitored_workers}.") def remove_monitor_workers(workers: list): @@ -60,13 +74,11 @@ def remove_monitor_workers(workers: list): """ if workers is None or len(workers) <= 0: return - redis_connection = CeleryManager.get_worker_status_redis_connection() - for worker in workers: - if redis_connection.exists(worker): - redis_connection.hset(worker, "monitored", 0) - redis_connection.hset(worker, "status", WorkerStatus.stopped) - - redis_connection.quit() + with CeleryManager.get_worker_status_redis_connection() as redis_connection: + for worker in workers: + if redis_connection.exists(worker): + redis_connection.hset(worker, "monitored", 0) + redis_connection.hset(worker, "status", WorkerStatus.stopped) def is_manager_runnning() -> bool: @@ -75,16 +87,18 @@ def is_manager_runnning() -> bool: :return: True if manager is running and False if not. """ - redis_connection = CeleryManager.get_worker_args_redis_connection() - manager_status = redis_connection.hgetall("manager") - redis_connection.quit() + with CeleryManager.get_worker_args_redis_connection() as redis_connection: + manager_status = redis_connection.hgetall("manager") return manager_status["status"] == WorkerStatus.running and psutil.pid_exists(manager_status["pid"]) def run_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180) -> bool: """ A process locking function that calls the celery manager with proper arguments. - :params: See CeleryManager for more information regarding the parameters + + :param query_frequency: The frequency at which workers will be queried with ping commands + :param query_timeout: The timeout for the query pings that are sent to workers + :param worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. """ celerymanager = CeleryManager(query_frequency=query_frequency, query_timeout=query_timeout, worker_timeout=worker_timeout) celerymanager.run() @@ -93,9 +107,11 @@ def run_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_ti def start_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180) -> bool: """ A Non-locking function that calls the celery manager with proper arguments. - :params: See CeleryManager for more information regarding the parameters - :return bool: True if the manager was started successfully. + :param query_frequency: The frequency at which workers will be queried with ping commands + :param query_timeout: The timeout for the query pings that are sent to workers + :param worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. + :return bool: True if the manager was started successfully. """ subprocess.Popen( f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", @@ -112,13 +128,13 @@ def stop_manager() -> bool: :return bool: True if the manager was stopped successfully and False otherwise. """ - redis_connection = CeleryManager.get_worker_status_redis_connection() - manager_pid = int(redis_connection.hget("manager", "pid")) - manager_status = redis_connection.hget("manager", "status") - print(redis_connection.hgetall("manager")) - redis_connection.quit() + with CeleryManager.get_worker_status_redis_connection() as redis_connection: + LOG.debug(f"MANAGER: manager keys: {redis_connection.hgetall('manager')}") + manager_pid = int(redis_connection.hget("manager", "pid")) + manager_status = redis_connection.hget("manager", "status") + LOG.debug(f"MANAGER: manager_status: {manager_status}") + LOG.debug(f"MANAGER: pid exists: {psutil.pid_exists(manager_pid)}") - print(manager_status, psutil.pid_exists(manager_pid)) # Check to make sure that the manager is running and the pid exists if manager_status == WorkerStatus.running and psutil.pid_exists(manager_pid): psutil.Process(manager_pid).terminate() From 1a4d416f97ead9725e0ca636d9b0f61141e370dd Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 22 Aug 2024 14:49:02 -0700 Subject: [PATCH 109/201] move managers to their own folder and fix ssl problems --- merlin/managers/__init__.py | 0 merlin/{study => managers}/celerymanager.py | 97 +++++++++++---------- merlin/managers/redis_connection.py | 81 +++++++++++++++++ merlin/study/celeryadapter.py | 2 +- merlin/study/celerymanageradapter.py | 2 +- 5 files changed, 132 insertions(+), 50 deletions(-) create mode 100644 merlin/managers/__init__.py rename merlin/{study => managers}/celerymanager.py (81%) create mode 100644 merlin/managers/redis_connection.py diff --git a/merlin/managers/__init__.py b/merlin/managers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/merlin/study/celerymanager.py b/merlin/managers/celerymanager.py similarity index 81% rename from merlin/study/celerymanager.py rename to merlin/managers/celerymanager.py index ddefab02c..99914545c 100644 --- a/merlin/study/celerymanager.py +++ b/merlin/managers/celerymanager.py @@ -35,6 +35,7 @@ import psutil import redis +from merlin.managers.redis_connection import RedisConnectionManager LOG = logging.getLogger(__name__) @@ -55,54 +56,54 @@ class WorkerStatus: } -class RedisConnectionManager: - """ - A context manager for handling redis connections. - This will ensure safe opening and closing of Redis connections. - """ - - def __init__(self, db_num: int): - self.db_num = db_num - self.connection = None - - def __enter__(self): - self.connection = self.get_redis_connection() - return self.connection - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.connection: - LOG.debug(f"MANAGER: Closing connection at db_num: {self.db_num}") - self.connection.close() - - def get_redis_connection(self) -> redis.Redis: - """ - Generic redis connection function to get the results backend redis server with a given db number increment. - - :return: Redis connection object that can be used to access values for the manager. - """ - # from merlin.config.results_backend import get_backend_password - from merlin.config import results_backend - from merlin.config.configfile import CONFIG - - conn_string = results_backend.get_connection_string() - base, _ = conn_string.rsplit("/", 1) - new_db_num = CONFIG.results_backend.db_num + self.db_num - conn_string = f"{base}/{new_db_num}" - LOG.debug(f"MANAGER: Connecting to redis at db_num: {new_db_num}") - return redis.from_url(conn_string, decode_responses=True) - # password_file = CONFIG.results_backend.password - # try: - # password = get_backend_password(password_file) - # except IOError: - # password = CONFIG.results_backend.password - # return redis.Redis( - # host=CONFIG.results_backend.server, - # port=CONFIG.results_backend.port, - # db=CONFIG.results_backend.db_num + self.db_num, # Increment db_num to avoid conflicts - # username=CONFIG.results_backend.username, - # password=password, - # decode_responses=True, - # ) +# class RedisConnectionManager: +# """ +# A context manager for handling redis connections. +# This will ensure safe opening and closing of Redis connections. +# """ + +# def __init__(self, db_num: int): +# self.db_num = db_num +# self.connection = None + +# def __enter__(self): +# self.connection = self.get_redis_connection() +# return self.connection + +# def __exit__(self, exc_type, exc_val, exc_tb): +# if self.connection: +# LOG.debug(f"MANAGER: Closing connection at db_num: {self.db_num}") +# self.connection.close() + +# def get_redis_connection(self) -> redis.Redis: +# """ +# Generic redis connection function to get the results backend redis server with a given db number increment. + +# :return: Redis connection object that can be used to access values for the manager. +# """ +# # from merlin.config.results_backend import get_backend_password +# from merlin.config import results_backend +# from merlin.config.configfile import CONFIG + +# conn_string = results_backend.get_connection_string() +# base, _ = conn_string.rsplit("/", 1) +# new_db_num = CONFIG.results_backend.db_num + self.db_num +# conn_string = f"{base}/{new_db_num}" +# LOG.debug(f"MANAGER: Connecting to redis at db_num: {new_db_num}") +# return redis.from_url(conn_string, decode_responses=True) +# # password_file = CONFIG.results_backend.password +# # try: +# # password = get_backend_password(password_file) +# # except IOError: +# # password = CONFIG.results_backend.password +# # return redis.Redis( +# # host=CONFIG.results_backend.server, +# # port=CONFIG.results_backend.port, +# # db=CONFIG.results_backend.db_num + self.db_num, # Increment db_num to avoid conflicts +# # username=CONFIG.results_backend.username, +# # password=password, +# # decode_responses=True, +# # ) class CeleryManager: diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py new file mode 100644 index 000000000..8749fcb64 --- /dev/null +++ b/merlin/managers/redis_connection.py @@ -0,0 +1,81 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.12.2b1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +This module stores a manager for redis connections. +""" +import logging +import redis + +LOG = logging.getLogger(__name__) + + +class RedisConnectionManager: + """ + A context manager for handling redis connections. + This will ensure safe opening and closing of Redis connections. + """ + + def __init__(self, db_num: int): + self.db_num = db_num + self.connection = None + + def __enter__(self): + self.connection = self.get_redis_connection() + return self.connection + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.connection: + LOG.debug(f"MANAGER: Closing connection at db_num: {self.db_num}") + self.connection.close() + + def get_redis_connection(self) -> redis.Redis: + """ + Generic redis connection function to get the results backend redis server with a given db number increment. + + :return: Redis connection object that can be used to access values for the manager. + """ + from merlin.config.results_backend import get_backend_password + from merlin.config.configfile import CONFIG + + password_file = CONFIG.results_backend.password + try: + password = get_backend_password(password_file) + except IOError: + password = CONFIG.results_backend.password + return redis.Redis( + host=CONFIG.results_backend.server, + port=CONFIG.results_backend.port, + db=CONFIG.results_backend.db_num + self.db_num, # Increment db_num to avoid conflicts + username=CONFIG.results_backend.username, + password=password, + decode_responses=True, + ssl=True, + ssl_cert_reqs=CONFIG.results_backend.cert_reqs, + ) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index e392b1795..651cd7a4b 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -46,8 +46,8 @@ from merlin.common.dumper import dump_handler from merlin.config import Config +from merlin.managers.celerymanager import CeleryManager from merlin.study.batch import batch_check_parallel, batch_worker_launch -from merlin.study.celerymanager import CeleryManager from merlin.study.celerymanageradapter import add_monitor_workers, remove_monitor_workers from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index d195eb966..31072d23e 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -32,7 +32,7 @@ import psutil -from merlin.study.celerymanager import WORKER_INFO, CeleryManager, WorkerStatus +from merlin.managers.celerymanager import WORKER_INFO, CeleryManager, WorkerStatus LOG = logging.getLogger(__name__) From 875f13792f811dd693143f30b6ab192c8abefc32 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 3 Sep 2024 14:11:29 -0700 Subject: [PATCH 110/201] final PR touch ups --- merlin/main.py | 13 +++--- merlin/managers/celerymanager.py | 64 +++------------------------- merlin/router.py | 5 ++- merlin/study/celeryadapter.py | 7 +-- merlin/study/celerymanageradapter.py | 11 ++++- 5 files changed, 30 insertions(+), 70 deletions(-) diff --git a/merlin/main.py b/merlin/main.py index 46683d273..56fcbd5a8 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -360,7 +360,7 @@ def stop_workers(args): LOG.warning(f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?") # Send stop command to router - router.stop_workers(args.task_server, worker_names, args.queues, args.workers) + router.stop_workers(args.task_server, worker_names, args.queues, args.workers, args.level.upper()) def print_info(args): @@ -414,12 +414,13 @@ def process_manager(args: Namespace): if args.command == "run": run_manager(query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout) elif args.command == "start": - if start_manager( - query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout - ): + try: + start_manager( + query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout + ) LOG.info("Manager started successfully.") - else: - LOG.error("Unable to start manager") + except Exception as e: + LOG.error(f"Unable to start manager.\n{e}") elif args.command == "stop": if stop_manager(): LOG.info("Manager stopped successfully.") diff --git a/merlin/managers/celerymanager.py b/merlin/managers/celerymanager.py index 99914545c..e6b820850 100644 --- a/merlin/managers/celerymanager.py +++ b/merlin/managers/celerymanager.py @@ -50,62 +50,12 @@ class WorkerStatus: WORKER_INFO = { "status": WorkerStatus.running, "pid": -1, - "monitored": 1, + "monitored": 1, # This setting is for debug mode "num_unresponsive": 0, "processing_work": 1, } -# class RedisConnectionManager: -# """ -# A context manager for handling redis connections. -# This will ensure safe opening and closing of Redis connections. -# """ - -# def __init__(self, db_num: int): -# self.db_num = db_num -# self.connection = None - -# def __enter__(self): -# self.connection = self.get_redis_connection() -# return self.connection - -# def __exit__(self, exc_type, exc_val, exc_tb): -# if self.connection: -# LOG.debug(f"MANAGER: Closing connection at db_num: {self.db_num}") -# self.connection.close() - -# def get_redis_connection(self) -> redis.Redis: -# """ -# Generic redis connection function to get the results backend redis server with a given db number increment. - -# :return: Redis connection object that can be used to access values for the manager. -# """ -# # from merlin.config.results_backend import get_backend_password -# from merlin.config import results_backend -# from merlin.config.configfile import CONFIG - -# conn_string = results_backend.get_connection_string() -# base, _ = conn_string.rsplit("/", 1) -# new_db_num = CONFIG.results_backend.db_num + self.db_num -# conn_string = f"{base}/{new_db_num}" -# LOG.debug(f"MANAGER: Connecting to redis at db_num: {new_db_num}") -# return redis.from_url(conn_string, decode_responses=True) -# # password_file = CONFIG.results_backend.password -# # try: -# # password = get_backend_password(password_file) -# # except IOError: -# # password = CONFIG.results_backend.password -# # return redis.Redis( -# # host=CONFIG.results_backend.server, -# # port=CONFIG.results_backend.port, -# # db=CONFIG.results_backend.db_num + self.db_num, # Increment db_num to avoid conflicts -# # username=CONFIG.results_backend.username, -# # password=password, -# # decode_responses=True, -# # ) - - class CeleryManager: def __init__(self, query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180): """ @@ -156,6 +106,7 @@ def stop_celery_worker(self, worker: str) -> bool: worker_pid = int(worker_status_connect.hget(worker, "pid")) worker_status = worker_status_connect.hget(worker, "status") + # TODO be wary of stalled state workers (should not happen since we use psutil.Process.kill()) # Check to see if the pid exists and worker is set as running if worker_status == WorkerStatus.running and psutil.pid_exists(worker_pid): # Check to see if the pid is associated with celery @@ -174,9 +125,8 @@ def restart_celery_worker(self, worker: str) -> bool: :return: The result of whether a worker was restarted. """ - # Stop the worker that is currently running - if not self.stop_celery_worker(worker): - return False + # Stop the worker that is currently running (if possible) + self.stop_celery_worker(worker) # Start the worker again with the args saved in redis db with ( @@ -218,7 +168,7 @@ def run(self): with self.get_worker_status_redis_connection() as redis_connection: LOG.debug(f"MANAGER: setting manager key in redis to hold the following info {manager_info}") - redis_connection.hmset(name="manager", mapping=manager_info) + redis_connection.hset("manager", mapping=manager_info) # TODO figure out what to do with "processing_work" entry for the merlin monitor while True: # TODO Make it so that it will stop after a list of workers is stopped @@ -251,10 +201,10 @@ def run(self): # If successful set the status to running and reset num_unresponsive redis_connection.hset(worker, "status", WorkerStatus.running) redis_connection.hset(worker, "num_unresponsive", 0) - # If failed set the status to stalled - redis_connection.hset(worker, "status", WorkerStatus.stalled) LOG.info(f"MANAGER: Worker '{worker}' restarted.") else: + # If failed set the status to stalled + redis_connection.hset(worker, "status", WorkerStatus.stalled) LOG.error(f"MANAGER: Could not restart worker '{worker}'.") else: redis_connection.hset(worker, "num_unresponsive", num_unresponsive) diff --git a/merlin/router.py b/merlin/router.py index d9114bbcd..9747b7c45 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -190,7 +190,7 @@ def get_workers(task_server): return [] -def stop_workers(task_server, spec_worker_names, queues, workers_regex): +def stop_workers(task_server, spec_worker_names, queues, workers_regex, debug_lvl): """ Stops workers. @@ -198,12 +198,13 @@ def stop_workers(task_server, spec_worker_names, queues, workers_regex): :param `spec_worker_names`: Worker names to stop, drawn from a spec. :param `queues` : The queues to stop :param `workers_regex` : Regex for workers to stop + :param debug_lvl: The debug level to use (INFO, DEBUG, ERROR, etc.) """ LOG.info("Stopping workers...") if task_server == "celery": # pylint: disable=R1705 # Stop workers - stop_celery_workers(queues, spec_worker_names, workers_regex) + stop_celery_workers(queues, spec_worker_names, workers_regex, debug_lvl) else: LOG.error("Celery is not specified as the task server!") diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 651cd7a4b..510e5a04e 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -46,7 +46,7 @@ from merlin.common.dumper import dump_handler from merlin.config import Config -from merlin.managers.celerymanager import CeleryManager +from merlin.managers.celerymanager import CeleryManager, WorkerStatus from merlin.study.batch import batch_check_parallel, batch_worker_launch from merlin.study.celerymanageradapter import add_monitor_workers, remove_monitor_workers from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running @@ -838,7 +838,7 @@ def purge_celery_tasks(queues, force): return subprocess.run(purge_command, shell=True).returncode -def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): # pylint: disable=R0912 +def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None, debug_lvl="INFO"): # pylint: disable=R0912 """Send a stop command to celery workers. Default behavior is to stop all connected workers. @@ -903,7 +903,8 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): if workers_to_stop: LOG.info(f"Sending stop to these workers: {workers_to_stop}") app.control.broadcast("shutdown", destination=workers_to_stop) - remove_monitor_workers(workers=workers_to_stop) + remove_entry = False if debug_lvl == "DEBUG" else True + remove_monitor_workers(workers=workers_to_stop, worker_status=WorkerStatus.stopped, remove_entry=remove_entry) else: LOG.warning("No workers found to stop") diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index 31072d23e..81f1a3240 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -67,7 +67,11 @@ def add_monitor_workers(workers: list): LOG.info(f"MANAGER: Manager is monitoring the following workers {monitored_workers}.") -def remove_monitor_workers(workers: list): +def remove_monitor_workers( + workers: list, + worker_status: WorkerStatus = None, + remove_entry: bool = True +): """ Remove workers from being monitored by the celery manager. :param list workers: A worker names @@ -78,7 +82,10 @@ def remove_monitor_workers(workers: list): for worker in workers: if redis_connection.exists(worker): redis_connection.hset(worker, "monitored", 0) - redis_connection.hset(worker, "status", WorkerStatus.stopped) + if worker_status is not None: + redis_connection.hset(worker, "status", worker_status) + if remove_entry: + redis_connection.delete(worker) def is_manager_runnning() -> bool: From 58da9bc173c0983f2b19d40ca21b12e8a79093d4 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 3 Sep 2024 14:16:42 -0700 Subject: [PATCH 111/201] Fix lint style changes --- merlin/managers/celerymanager.py | 8 ++++---- merlin/managers/redis_connection.py | 4 +++- merlin/study/celerymanageradapter.py | 6 +----- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/merlin/managers/celerymanager.py b/merlin/managers/celerymanager.py index e6b820850..2f1b99c00 100644 --- a/merlin/managers/celerymanager.py +++ b/merlin/managers/celerymanager.py @@ -33,10 +33,10 @@ import time import psutil -import redis from merlin.managers.redis_connection import RedisConnectionManager + LOG = logging.getLogger(__name__) @@ -156,9 +156,9 @@ def run(self): """ Main manager loop for monitoring and managing Celery workers. - This method continuously monitors the status of Celery workers by - checking their health and attempting to restart any that are - unresponsive. It updates the Redis database with the current + This method continuously monitors the status of Celery workers by + checking their health and attempting to restart any that are + unresponsive. It updates the Redis database with the current status of the manager and the workers. """ manager_info = { diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index 8749fcb64..e9e947dff 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -31,8 +31,10 @@ This module stores a manager for redis connections. """ import logging + import redis + LOG = logging.getLogger(__name__) @@ -61,8 +63,8 @@ def get_redis_connection(self) -> redis.Redis: :return: Redis connection object that can be used to access values for the manager. """ - from merlin.config.results_backend import get_backend_password from merlin.config.configfile import CONFIG + from merlin.config.results_backend import get_backend_password password_file = CONFIG.results_backend.password try: diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py index 81f1a3240..6dc07bab2 100644 --- a/merlin/study/celerymanageradapter.py +++ b/merlin/study/celerymanageradapter.py @@ -67,11 +67,7 @@ def add_monitor_workers(workers: list): LOG.info(f"MANAGER: Manager is monitoring the following workers {monitored_workers}.") -def remove_monitor_workers( - workers: list, - worker_status: WorkerStatus = None, - remove_entry: bool = True -): +def remove_monitor_workers(workers: list, worker_status: WorkerStatus = None, remove_entry: bool = True): """ Remove workers from being monitored by the celery manager. :param list workers: A worker names From e75dcc2678116f9318afec70e0b6606416c59565 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 4 Sep 2024 10:52:31 -0700 Subject: [PATCH 112/201] Fixed issue with context manager --- merlin/examples/workflows/null_spec/scripts/launch_jobs.py | 4 +--- merlin/managers/celerymanager.py | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index a6b6d1372..99c3c3d6a 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -78,9 +78,7 @@ if real_time > 1440: real_time = 1440 submit: str = "submit.sbatch" - command: str = ( - f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" - ) + command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) diff --git a/merlin/managers/celerymanager.py b/merlin/managers/celerymanager.py index 2f1b99c00..fe136d1ec 100644 --- a/merlin/managers/celerymanager.py +++ b/merlin/managers/celerymanager.py @@ -129,10 +129,8 @@ def restart_celery_worker(self, worker: str) -> bool: self.stop_celery_worker(worker) # Start the worker again with the args saved in redis db - with ( - self.get_worker_args_redis_connection() as worker_args_connect, - self.get_worker_status_redis_connection() as worker_status_connect, - ): + with self.get_worker_args_redis_connection() as worker_args_connect, self.get_worker_status_redis_connection() as worker_status_connect: + # Get the args and remove the worker_cmd from the hash set args = worker_args_connect.hgetall(worker) worker_cmd = args["worker_cmd"] From 11f9e7ca4dc2ffd3e2417d6364d7deb92bbc49ab Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 4 Sep 2024 10:57:53 -0700 Subject: [PATCH 113/201] Reset file that was incorrect changed --- merlin/examples/workflows/null_spec/scripts/launch_jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py index 99c3c3d6a..a6b6d1372 100644 --- a/merlin/examples/workflows/null_spec/scripts/launch_jobs.py +++ b/merlin/examples/workflows/null_spec/scripts/launch_jobs.py @@ -78,7 +78,9 @@ if real_time > 1440: real_time = 1440 submit: str = "submit.sbatch" - command: str = f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" + command: str = ( + f"sbatch -J c{concurrency}s{sample}r{args.run_id} --time {real_time} -N {nodes[ii]} -p {partition} -A {account} {submit} {sample} {int(concurrency/nodes[ii])} {args.run_id} {concurrency}" + ) shutil.copyfile(os.path.join(submit_path, submit), submit) shutil.copyfile(args.spec_path, "spec.yaml") shutil.copyfile(args.script_path, os.path.join("scripts", "make_samples.py")) From 7204e46cdd07620eb9b0d29beb7ed90a742a2665 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 4 Sep 2024 11:52:21 -0700 Subject: [PATCH 114/201] Check for ssl cert before applying to Redis connection --- merlin/managers/redis_connection.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index e9e947dff..5b3f4c723 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -71,6 +71,12 @@ def get_redis_connection(self) -> redis.Redis: password = get_backend_password(password_file) except IOError: password = CONFIG.results_backend.password + + has_ssl = hasattr(CONFIG.results_backend, "cert_reqs") + ssl_cert_reqs = "required" + if has_ssl: + ssl_cert_reqs = CONFIG.results_backend.cert_reqs + return redis.Redis( host=CONFIG.results_backend.server, port=CONFIG.results_backend.port, @@ -78,6 +84,6 @@ def get_redis_connection(self) -> redis.Redis: username=CONFIG.results_backend.username, password=password, decode_responses=True, - ssl=True, - ssl_cert_reqs=CONFIG.results_backend.cert_reqs, + ssl=has_ssl, + ssl_cert_reqs=ssl_cert_reqs, ) From 53d8f32e994a19c0ed8b4c3b31d7eca4fab0df35 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Wed, 4 Sep 2024 12:53:11 -0700 Subject: [PATCH 115/201] Comment out Active tests for celerymanager --- tests/unit/study/test_celeryadapter.py | 288 ++++++++++++------------- 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py index 0572d6c66..44772de3e 100644 --- a/tests/unit/study/test_celeryadapter.py +++ b/tests/unit/study/test_celeryadapter.py @@ -49,150 +49,150 @@ from tests.unit.study.status_test_files.status_test_variables import SPEC_PATH -@pytest.mark.order(before="TestInactive") -class TestActive: - """ - This class will test functions in the celeryadapter.py module. - It will run tests where we need active queues/workers to interact with. - - NOTE: The tests in this class must be ran before the TestInactive class or else the - Celery workers needed for this class don't start - - TODO: fix the bug noted above and then check if we still need pytest-order - """ - - def test_query_celery_queues( - self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 - ): - """ - Test the query_celery_queues function by providing it with a list of active queues. - This should return a dict where keys are queue names and values are more dicts containing - the number of jobs and consumers in that queue. - - :param `celery_app`: A pytest fixture for the test Celery app - :param launch_workers: A pytest fixture that launches celery workers for us to interact with - :param worker_queue_map: A pytest fixture that returns a dict of workers and queues - """ - # Set up a dummy configuration to use in the test - dummy_config = Config({"broker": {"name": "redis"}}) - - # Get the actual output - queues_to_query = list(worker_queue_map.values()) - actual_queue_info = celeryadapter.query_celery_queues(queues_to_query, app=celery_app, config=dummy_config) - - # Ensure all 3 queues in worker_queue_map were queried before looping - assert len(actual_queue_info) == 3 - - # Ensure each queue has a worker attached - for queue_name, queue_info in actual_queue_info.items(): - assert queue_name in worker_queue_map.values() - assert queue_info == {"consumers": 1, "jobs": 0} - - def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 - """ - Test the get_running_queues function with queues active. - This should return a list of active queues. - - :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with - :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues - """ - result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) - assert sorted(result) == sorted(list(worker_queue_map.values())) - - def test_get_active_celery_queues( - self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 - ): - """ - Test the get_active_celery_queues function with queues active. - This should return a tuple where the first entry is a dict of queue info - and the second entry is a list of worker names. - - :param `celery_app`: A pytest fixture for the test Celery app - :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with - :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues - """ - # Start the queues and run the test - queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) - - # Ensure we got output before looping - assert len(queue_result) == len(worker_result) == 3 - - for worker, queue in worker_queue_map.items(): - # Check that the entry in the queue_result dict for this queue is correct - assert queue in queue_result - assert len(queue_result[queue]) == 1 - assert worker in queue_result[queue][0] - - # Remove this entry from the queue_result dict - del queue_result[queue] - - # Check that this worker was added to the worker_result list - worker_found = False - for worker_name in worker_result[:]: - if worker in worker_name: - worker_found = True - worker_result.remove(worker_name) - break - assert worker_found - - # Ensure there was no extra output that we weren't expecting - assert queue_result == {} - assert worker_result == [] - - def test_build_set_of_queues( - self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 - ): - """ - Test the build_set_of_queues function with queues active. - This should return a set of queues (the queues defined in setUp). - """ - # Run the test - result = celeryadapter.build_set_of_queues( - steps=["all"], spec=None, specific_queues=None, verbose=False, app=celery_app - ) - assert result == set(worker_queue_map.values()) - - @pytest.mark.order(index=1) - def test_check_celery_workers_processing_tasks( - self, - celery_app: Celery, - sleep_sig: Signature, - launch_workers: "Fixture", # noqa: F821 - ): - """ - Test the check_celery_workers_processing function with workers active and a task in a queue. - This function will query workers for any tasks they're still processing. We'll send a - a task that sleeps for 3 seconds to our workers before we run this test so that there should be - a task for this function to find. - - NOTE: the celery app fixture shows strange behavior when using app.control.inspect() calls (which - check_celery_workers_processing uses) so we have to run this test first in this class in order to - have it run properly. - - :param celery_app: A pytest fixture for the test Celery app - :param sleep_sig: A pytest fixture for a celery signature of a task that sleeps for 3 sec - :param launch_workers: A pytest fixture that launches celery workers for us to interact with - """ - # Our active workers/queues are test_worker_[0-2]/test_queue_[0-2] so we're - # sending this to test_queue_0 for test_worker_0 to process - queue_for_signature = "test_queue_0" - sleep_sig.set(queue=queue_for_signature) - result = sleep_sig.delay() - - # We need to give the task we just sent to the server a second to get picked up by the worker - sleep(1) - - # Run the test now that the task should be getting processed - active_queue_test = celeryadapter.check_celery_workers_processing([queue_for_signature], celery_app) - assert active_queue_test is True - - # Now test that a queue without any tasks returns false - # We sent the signature to task_queue_0 so task_queue_1 shouldn't have any tasks to find - non_active_queue_test = celeryadapter.check_celery_workers_processing(["test_queue_1"], celery_app) - assert non_active_queue_test is False - - # Wait for the worker to finish running the task - result.get() +# @pytest.mark.order(before="TestInactive") +# class TestActive: +# """ +# This class will test functions in the celeryadapter.py module. +# It will run tests where we need active queues/workers to interact with. + +# NOTE: The tests in this class must be ran before the TestInactive class or else the +# Celery workers needed for this class don't start + +# TODO: fix the bug noted above and then check if we still need pytest-order +# """ + +# def test_query_celery_queues( +# self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 +# ): +# """ +# Test the query_celery_queues function by providing it with a list of active queues. +# This should return a dict where keys are queue names and values are more dicts containing +# the number of jobs and consumers in that queue. + +# :param `celery_app`: A pytest fixture for the test Celery app +# :param launch_workers: A pytest fixture that launches celery workers for us to interact with +# :param worker_queue_map: A pytest fixture that returns a dict of workers and queues +# """ +# # Set up a dummy configuration to use in the test +# dummy_config = Config({"broker": {"name": "redis"}}) + +# # Get the actual output +# queues_to_query = list(worker_queue_map.values()) +# actual_queue_info = celeryadapter.query_celery_queues(queues_to_query, app=celery_app, config=dummy_config) + +# # Ensure all 3 queues in worker_queue_map were queried before looping +# assert len(actual_queue_info) == 3 + +# # Ensure each queue has a worker attached +# for queue_name, queue_info in actual_queue_info.items(): +# assert queue_name in worker_queue_map.values() +# assert queue_info == {"consumers": 1, "jobs": 0} + +# def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 +# """ +# Test the get_running_queues function with queues active. +# This should return a list of active queues. + +# :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with +# :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues +# """ +# result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) +# assert sorted(result) == sorted(list(worker_queue_map.values())) + +# def test_get_active_celery_queues( +# self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 +# ): +# """ +# Test the get_active_celery_queues function with queues active. +# This should return a tuple where the first entry is a dict of queue info +# and the second entry is a list of worker names. + +# :param `celery_app`: A pytest fixture for the test Celery app +# :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with +# :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues +# """ +# # Start the queues and run the test +# queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) + +# # Ensure we got output before looping +# assert len(queue_result) == len(worker_result) == 3 + +# for worker, queue in worker_queue_map.items(): +# # Check that the entry in the queue_result dict for this queue is correct +# assert queue in queue_result +# assert len(queue_result[queue]) == 1 +# assert worker in queue_result[queue][0] + +# # Remove this entry from the queue_result dict +# del queue_result[queue] + +# # Check that this worker was added to the worker_result list +# worker_found = False +# for worker_name in worker_result[:]: +# if worker in worker_name: +# worker_found = True +# worker_result.remove(worker_name) +# break +# assert worker_found + +# # Ensure there was no extra output that we weren't expecting +# assert queue_result == {} +# assert worker_result == [] + +# def test_build_set_of_queues( +# self, celery_app: Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 +# ): +# """ +# Test the build_set_of_queues function with queues active. +# This should return a set of queues (the queues defined in setUp). +# """ +# # Run the test +# result = celeryadapter.build_set_of_queues( +# steps=["all"], spec=None, specific_queues=None, verbose=False, app=celery_app +# ) +# assert result == set(worker_queue_map.values()) + +# @pytest.mark.order(index=1) +# def test_check_celery_workers_processing_tasks( +# self, +# celery_app: Celery, +# sleep_sig: Signature, +# launch_workers: "Fixture", # noqa: F821 +# ): +# """ +# Test the check_celery_workers_processing function with workers active and a task in a queue. +# This function will query workers for any tasks they're still processing. We'll send a +# a task that sleeps for 3 seconds to our workers before we run this test so that there should be +# a task for this function to find. + +# NOTE: the celery app fixture shows strange behavior when using app.control.inspect() calls (which +# check_celery_workers_processing uses) so we have to run this test first in this class in order to +# have it run properly. + +# :param celery_app: A pytest fixture for the test Celery app +# :param sleep_sig: A pytest fixture for a celery signature of a task that sleeps for 3 sec +# :param launch_workers: A pytest fixture that launches celery workers for us to interact with +# """ +# # Our active workers/queues are test_worker_[0-2]/test_queue_[0-2] so we're +# # sending this to test_queue_0 for test_worker_0 to process +# queue_for_signature = "test_queue_0" +# sleep_sig.set(queue=queue_for_signature) +# result = sleep_sig.delay() + +# # We need to give the task we just sent to the server a second to get picked up by the worker +# sleep(1) + +# # Run the test now that the task should be getting processed +# active_queue_test = celeryadapter.check_celery_workers_processing([queue_for_signature], celery_app) +# assert active_queue_test is True + +# # Now test that a queue without any tasks returns false +# # We sent the signature to task_queue_0 so task_queue_1 shouldn't have any tasks to find +# non_active_queue_test = celeryadapter.check_celery_workers_processing(["test_queue_1"], celery_app) +# assert non_active_queue_test is False + +# # Wait for the worker to finish running the task +# result.get() class TestInactive: From 6bf1fe63ca8ec72c576846b9070ee175b0f697d1 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 5 Sep 2024 15:59:55 -0700 Subject: [PATCH 116/201] split up create_server_config and write tests for it --- merlin/server/server_config.py | 42 ++++-- tests/unit/server/test_server_config.py | 183 ++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 14 deletions(-) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 5b89f2f6a..9f6047a17 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -136,6 +136,31 @@ def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: return False, "Reached end of redis output without seeing 'Ready to accept connections'" +def write_container_command_files(config_dir: str) -> bool: + """ + Write the yaml files that contain instructions on how to + run certain commands for each container type. + + :param config_dir: The path to the configuration dir where we'll write files + :returns: True if successful. False otherwise. + """ + files = [i + ".yaml" for i in CONTAINER_TYPES] + for file in files: + file_path = os.path.join(config_dir, file) + if os.path.exists(file_path): + LOG.info(f"{file} already exists.") + continue + LOG.info(f"Copying file {file} to configuration directory.") + try: + with resources.path("merlin.server", file) as config_file: + with open(file_path, "w") as outfile, open(config_file, "r") as infile: + outfile.write(infile.read()) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + return True + + def create_server_config() -> bool: """ Create main configuration file for merlin server in the @@ -158,20 +183,8 @@ def create_server_config() -> bool: LOG.error(err) return False - files = [i + ".yaml" for i in CONTAINER_TYPES] - for file in files: - file_path = os.path.join(config_dir, file) - if os.path.exists(file_path): - LOG.info(f"{file} already exists.") - continue - LOG.info(f"Copying file {file} to configuration directory.") - try: - with resources.path("merlin.server", file) as config_file: - with open(file_path, "w") as outfile, open(config_file, "r") as infile: - outfile.write(infile.read()) - except OSError: - LOG.error(f"Destination location {config_dir} is not writable.") - return False + if not write_container_command_files(config_dir): + return False # Load Merlin Server Configuration and apply it to app.yaml with resources.path("merlin.server", MERLIN_SERVER_CONFIG) as merlin_server_config: @@ -189,6 +202,7 @@ def create_server_config() -> bool: return False if not os.path.exists(server_config.container.get_config_dir()): + print("inside get_config_dir if statement") LOG.info("Creating merlin server directory.") os.mkdir(server_config.container.get_config_dir()) diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 035e70d60..eaac6bc99 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -3,11 +3,14 @@ """ import io +import logging +import os import string from typing import Dict, Tuple, Union import pytest +from merlin.server.server_util import CONTAINER_TYPES, ServerConfig from merlin.server.server_config import ( PASSWORD_LENGTH, check_process_file_format, @@ -20,6 +23,7 @@ pull_process_file, pull_server_config, pull_server_image, + write_container_command_files, ) @@ -106,3 +110,182 @@ def test_parse_redis_output_with_vars(lines: bytes, expected_config: Tuple[bool, reader_input = io.BufferedReader(buffer) _, actual_vars = parse_redis_output(reader_input) assert expected_config == actual_vars + + +def test_write_container_command_files_with_existing_files( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Test the `write_container_command_files` function with files that already exist. + This should skip trying to create the files, log 3 "file already exists" messages, + and return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + mocker.patch('os.path.exists', return_value=True) + assert write_container_command_files(server_testing_dir) + file_names = [f"{container}.yaml" for container in CONTAINER_TYPES] + for file in file_names: + assert f"{file} already exists." in caplog.text + + +def test_write_container_command_files_with_nonexisting_files( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Test the `write_container_command_files` function with files that don't already exist. + This should create the files, log messages for each file, and return True + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + + # Mock the os.path.exists function so it returns False + mocker.patch('os.path.exists', return_value=False) + + # Mock the resources.path context manager + mock_path = mocker.patch("merlin.server.server_config.resources.path") + mock_path.return_value.__enter__.return_value = "mocked_file_path" + + # Mock the open builtin + mock_data = mocker.mock_open(read_data="mocked data") + mocker.patch("builtins.open", mock_data) + + assert write_container_command_files(server_testing_dir) + file_names = [f"{container}.yaml" for container in CONTAINER_TYPES] + for file in file_names: + assert f"Copying file {file} to configuration directory." in caplog.text + + +def test_write_container_command_files_with_oserror( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Test the `write_container_command_files` function with an OSError being raised. + This should log an error message and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + # Mock the open function to raise an OSError + mocker.patch("builtins.open", side_effect=OSError("File not writeable")) + + assert not write_container_command_files(server_testing_dir) + assert f"Destination location {server_testing_dir} is not writable." in caplog.text + + +def test_create_server_config_merlin_config_dir_nonexistent( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Tests the `create_server_config` function with MERLIN_CONFIG_DIR not existing. + This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + nonexistent_dir = f"{server_testing_dir}/merlin_config_dir" + mocker.patch('merlin.server.server_config.MERLIN_CONFIG_DIR', nonexistent_dir) + assert not create_server_config() + assert f"Unable to find main merlin configuration directory at {nonexistent_dir}" in caplog.text + + +def test_create_server_config_server_subdir_nonexistent_oserror( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Tests the `create_server_config` function with MERLIN_CONFIG_DIR/MERLIN_SERVER_SUBDIR + not existing and an OSError being raised. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + + # Mock MERLIN_CONFIG_DIR and MERLIN_SERVER_SUBDIR + nonexistent_server_subdir = "test_create_server_config_server_subdir_nonexistent" + mocker.patch('merlin.server.server_config.MERLIN_CONFIG_DIR', server_testing_dir) + mocker.patch('merlin.server.server_config.MERLIN_SERVER_SUBDIR', nonexistent_server_subdir) + + # Mock os.mkdir so it raises an OSError + err_msg = "File not writeable" + mocker.patch("os.mkdir", side_effect=OSError(err_msg)) + assert not create_server_config() + assert err_msg in caplog.text + + +def test_create_server_config_no_server_config( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Tests the `create_server_config` function with the call to `pull_server_config()` + returning None. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + + # Mock the necessary variables/functions to get us to the pull_server_config call + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mocker.patch("merlin.server.server_config.write_container_command_files", return_value=True) + mock_open_func = mocker.mock_open(read_data='key: value') + mocker.patch("builtins.open", mock_open_func) + + # Mock the pull_server_config call (what we're actually testing) and run the test + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + assert not create_server_config() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_create_server_config_no_server_dir( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, str], +): + """ + Tests the `create_server_config` function with the call to + `server_config.container.get_config_dir()` returning a non-existent path. This should + log a message and create the directory, then return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Mock the necessary variables/functions to get us to the get_config_dir call + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mocker.patch("merlin.server.server_config.write_container_command_files", return_value=True) + mock_open_func = mocker.mock_open(read_data='key: value') + mocker.patch("builtins.open", mock_open_func) + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + + # Mock the get_config_dir call to return a directory that doesn't exist yet + nonexistent_dir = f"{server_testing_dir}/merlin_server" + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value=nonexistent_dir) + + assert create_server_config() + assert os.path.exists(nonexistent_dir) + assert "Creating merlin server directory." in caplog.text From cf307bb4a2de5595d29afa58a8101c99b76fa76d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 5 Sep 2024 16:56:51 -0700 Subject: [PATCH 117/201] add tests for config_merlin_server function --- merlin/server/server_config.py | 3 - tests/unit/server/test_server_config.py | 79 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 9f6047a17..4b8790e20 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -223,9 +223,6 @@ def config_merlin_server(): if os.path.exists(pass_file): LOG.info("Password file already exists. Skipping password generation step.") else: - # if "pass_command" in server_config["container"]: - # password = generate_password(PASSWORD_LENGTH, server_config["container"]["pass_command"]) - # else: password = generate_password(PASSWORD_LENGTH) with open(pass_file, "w+") as f: # pylint: disable=C0103 diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index eaac6bc99..0ce986074 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -289,3 +289,82 @@ def test_create_server_config_no_server_dir( assert create_server_config() assert os.path.exists(nonexistent_dir) assert "Creating merlin server directory." in caplog.text + + +def test_config_merlin_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `config_merlin_server` function with the call to `pull_server_config()` + returning None. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + assert not config_merlin_server() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_config_merlin_server_pass_user_exist( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, str], +): + """ + Tests the `config_merlin_server` function with a password file and user file already + existing. This should log 2 messages and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Create the password file and user file + pass_file = f"{server_testing_dir}/existent_pass_file.txt" + user_file = f"{server_testing_dir}/existent_user_file.txt" + with open(pass_file, "w"), open(user_file, "w"): + pass + + # Mock necessary calls + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_pass_file_path", return_value=pass_file) + mocker.patch("merlin.server.server_util.ContainerConfig.get_user_file_path", return_value=user_file) + + assert config_merlin_server() is None + assert "Password file already exists. Skipping password generation step." in caplog.text + assert "User file already exists." in caplog.text + + +def test_config_merlin_server_pass_user_dont_exist( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, str], +): + """ + Tests the `config_merlin_server` function with a password file and user file that don't + already exist. This should log 2 messages, create the files, and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Create vars for the nonexistent password file and user file + pass_file = f"{server_testing_dir}/nonexistent_pass_file.txt" + user_file = f"{server_testing_dir}/nonexistent_user_file.txt" + + # Mock necessary calls + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_pass_file_path", return_value=pass_file) + mocker.patch("merlin.server.server_util.ContainerConfig.get_user_file_path", return_value=user_file) + + assert config_merlin_server() is None + assert os.path.exists(pass_file) + assert os.path.exists(user_file) + assert "Creating password file for merlin server container." in caplog.text + assert f"User {os.environ.get('USER')} created in user file for merlin server container" in caplog.text From a5ccb2d0954fa24361c5a141ae15cf07fc77eb8f Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 9 Sep 2024 12:28:14 -0700 Subject: [PATCH 118/201] Fix lint issue with unused import after commenting out Active celery tests --- tests/unit/study/test_celeryadapter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py index 44772de3e..1a4a60fd9 100644 --- a/tests/unit/study/test_celeryadapter.py +++ b/tests/unit/study/test_celeryadapter.py @@ -34,12 +34,12 @@ import json import os from datetime import datetime -from time import sleep +# from time import sleep from typing import Dict -import pytest +# import pytest from celery import Celery -from celery.canvas import Signature +# from celery.canvas import Signature from deepdiff import DeepDiff from merlin.config import Config From 2b0e8a6cff7c284c293b67b73a368d9f22dc77c5 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 9 Sep 2024 12:37:56 -0700 Subject: [PATCH 119/201] Fixed style for import --- tests/unit/study/test_celeryadapter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py index 1a4a60fd9..0cc6592dc 100644 --- a/tests/unit/study/test_celeryadapter.py +++ b/tests/unit/study/test_celeryadapter.py @@ -34,11 +34,13 @@ import json import os from datetime import datetime + # from time import sleep from typing import Dict # import pytest from celery import Celery + # from celery.canvas import Signature from deepdiff import DeepDiff From 6ba91a6889e8b5be700df185320ab1ce256fc988 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 9 Sep 2024 15:49:06 -0700 Subject: [PATCH 120/201] add tests for pull_server_config --- merlin/server/server_config.py | 1 - tests/unit/server/test_server_config.py | 148 +++++++++++++++++++++++- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 4b8790e20..fe79599bc 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -202,7 +202,6 @@ def create_server_config() -> bool: return False if not os.path.exists(server_config.container.get_config_dir()): - print("inside get_config_dir if statement") LOG.info("Creating merlin server directory.") os.mkdir(server_config.container.get_config_dir()) diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 0ce986074..6c2dd0cd0 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -10,8 +10,10 @@ import pytest -from merlin.server.server_util import CONTAINER_TYPES, ServerConfig +from merlin.server.server_util import CONTAINER_TYPES, MERLIN_SERVER_SUBDIR, ServerConfig from merlin.server.server_config import ( + LOCAL_APP_YAML, + MERLIN_CONFIG_DIR, PASSWORD_LENGTH, check_process_file_format, config_merlin_server, @@ -368,3 +370,147 @@ def test_config_merlin_server_pass_user_dont_exist( assert os.path.exists(user_file) assert "Creating password file for merlin server container." in caplog.text assert f"User {os.environ.get('USER')} created in user file for merlin server container" in caplog.text + + +def setup_pull_server_config_mock( + mocker: "Fixture", + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_server_config: Dict[str, Dict[str, str]], +): + """ + Setup the necessary mocker calls for the `pull_server_config` function. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_util.AppYaml.get_data", return_value=server_app_yaml_contents) + mocker.patch('merlin.server.server_config.MERLIN_CONFIG_DIR', server_testing_dir) + mock_data = mocker.mock_open(read_data=str(server_server_config)) + mocker.patch("builtins.open", mock_data) + + +@pytest.mark.parametrize( + "key_to_delete, expected_log_message", + [ + ("container", 'Unable to find "container" object in {default_app_yaml}'), + ("container.format", 'Unable to find "format" in {default_app_yaml}'), + ("process", 'Process config not found in {default_app_yaml}'), + ] +) +def test_pull_server_config_missing_config_keys( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_server_config: Dict[str, Dict[str, str]], + key_to_delete: str, + expected_log_message: str, +): + """ + Test the `pull_server_config` function with missing container-related keys in the + app.yaml file contents. This should log an error message and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param key_to_delete: The key to delete from the app.yaml contents + :param expected_log_message: The expected log message when the key is missing + """ + # Handle nested key deletion + keys = key_to_delete.split('.') + temp_app_yaml = server_app_yaml_contents + for key in keys[:-1]: + temp_app_yaml = temp_app_yaml[key] + del temp_app_yaml[keys[-1]] + + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + + assert pull_server_config() is None + default_app_yaml = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + assert expected_log_message.format(default_app_yaml=default_app_yaml) in caplog.text + + +@pytest.mark.parametrize("key_to_delete", ["command", "run_command", "stop_command", "pull_command"]) +def test_pull_server_config_missing_format_needed_keys( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_container_format_config_data: Dict[str, str], + server_server_config: Dict[str, Dict[str, str]], + key_to_delete: str, +): + """ + Test the `pull_server_config` function with necessary format keys missing in the + singularity.yaml file contents. This should log an error message and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param key_to_delete: The key to delete from the singularity.yaml contents + """ + del server_container_format_config_data[key_to_delete] + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + + assert pull_server_config() is None + format_file_basename = server_app_yaml_contents["container"]["format"] + ".yaml" + format_file = os.path.join(server_testing_dir, MERLIN_SERVER_SUBDIR) + format_file = os.path.join(format_file, format_file_basename) + assert f'Unable to find necessary "{key_to_delete}" value in format config file {format_file}' in caplog.text + + +@pytest.mark.parametrize("key_to_delete", ["status", "kill"]) +def test_pull_server_config_missing_process_needed_key( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_process_config_data: Dict[str, str], + server_server_config: Dict[str, Dict[str, str]], + key_to_delete: str, +): + """ + Test the `pull_server_config` function with necessary process keys missing. + This should log an error message and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param key_to_delete: The key to delete from the process config entry + """ + del server_process_config_data[key_to_delete] + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + + assert pull_server_config() is None + default_app_yaml = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + assert f'Process necessary "{key_to_delete}" command configuration not found in {default_app_yaml}' in caplog.text + + +def test_pull_server_config_no_issues( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_config` function without any problems. This should + return a ServerConfig object. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + assert isinstance(pull_server_config(), ServerConfig) From 28e50408541ee516469306ef2cae3bfdcb6f0472 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 10 Sep 2024 14:07:13 -0700 Subject: [PATCH 121/201] add tests for pull_server_image --- tests/unit/server/test_server_config.py | 179 ++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 6c2dd0cd0..b2edf4314 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -28,6 +28,11 @@ write_container_command_files, ) +try: + from importlib import resources +except ImportError: + import importlib_resources as resources + def test_generate_password_no_pass_command(): """ @@ -514,3 +519,177 @@ def test_pull_server_config_no_issues( """ setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) assert isinstance(pull_server_config(), ServerConfig) + + +def test_pull_server_image_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `pull_server_image` function with no server config being found. + This should return False and log an error message. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + assert not pull_server_image() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def setup_pull_server_image_mock( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], + config_dir: str, + config_file: str, + image_file: str, + create_config_file: bool = False, + create_image_file: bool = False, + +): + """ + Set up the necessary mock calls for the `pull_server_image` function. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + image_url = "docker://redis" + image_path = f"{server_testing_dir}/{image_file}" + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value=config_dir) + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_name", return_value=config_file) + mocker.patch("merlin.server.server_util.ContainerConfig.get_image_url", return_value=image_url) + mocker.patch("merlin.server.server_util.ContainerConfig.get_image_path", return_value=image_path) + + if create_config_file: + if not os.path.exists(config_dir): + os.mkdir(config_dir) + with open(os.path.join(config_dir, config_file), "w"): + pass + + if create_image_file: + with open(image_path, "w"): + pass + + +def test_pull_server_image_no_image_path_no_config_path( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_image` function with no image path and no configuration + path. This should run a subprocess for the image path and create the configuration + file. It should also return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + # Set up mock calls to simulate the setup of this function + config_dir = f"{server_testing_dir}/config_dir" + config_file = "nonexistent.yaml" + image_file = "nonexistent.sif" + setup_pull_server_image_mock( + mocker, + server_testing_dir, + server_server_config, + config_dir, + config_file, + image_file + ) + mocked_subprocess = mocker.patch("subprocess.run") + + # Mock the open function + read_data = "Mocked file content" + mocked_open = mocker.mock_open(read_data=read_data) + mocker.patch("builtins.open", mocked_open) + + # Call the function + assert pull_server_image() + + # Assert that the subprocess call to pull the image was called + mocked_subprocess.assert_called_once() + + # Assert that open was called with the correct arguments + mocked_open.assert_any_call(os.path.join(config_dir, config_file), "w") + with resources.path("merlin.server", config_file) as file: + mocked_open.assert_any_call(file, "r") + assert mocked_open.call_count == 2 + + # Assert that the write method was called with the expected content + mocked_open().write.assert_called_once_with(read_data) + + +def test_pull_server_image_both_paths_exist( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_image` function with both an image path and a configuration + path that both exist. This should log two messages and return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Set up mock calls to simulate the setup of this function + config_dir = f"{server_testing_dir}/config_dir" + config_file = "existent.yaml" + image_file = "existent.sif" + setup_pull_server_image_mock( + mocker, + server_testing_dir, + server_server_config, + config_dir, + config_file, + image_file, + create_config_file=True, + create_image_file=True, + ) + + assert pull_server_image() + assert f"{image_file} already exists." in caplog.text + assert "Redis configuration file already exist." in caplog.text + + +def test_pull_server_image_os_error( + mocker: "Fixture", + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_image` function with an image path but no configuration + path. We'll force this to raise an OSError when writing to the configuration file + to ensure it's handled properly. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + # Set up mock calls to simulate the setup of this function + config_dir = f"{server_testing_dir}/config_dir" + config_file = "existent.yaml" + image_file = "nonexistent.sif" + setup_pull_server_image_mock( + mocker, + server_testing_dir, + server_server_config, + config_dir, + config_file, + image_file, + create_image_file=True, + ) + + # Mock the open function + mocker.patch("builtins.open", side_effect=OSError) + + # Run the test + assert not pull_server_image() + assert f"Destination location {config_dir} is not writable." in caplog.text From 24470e5f49369f89877b4fd9e6ab727899edbdc8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 10 Sep 2024 17:17:13 -0700 Subject: [PATCH 122/201] finish writing tests for server_config.py --- tests/fixtures/server.py | 6 + tests/unit/server/test_server_config.py | 171 +++++++++++++++++++++++- 2 files changed, 171 insertions(+), 6 deletions(-) diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 156a374d7..a955ebb08 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -276,3 +276,9 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> yaml.dump(server_app_yaml_contents, ayf) return app_yaml_file + + +@pytest.fixture(scope="function") +def server_process_file_contents() -> str: + """Fixture to represent process file contents.""" + return {"parent_pid": 123, "image_pid": 456, "port": 6379, "hostname": "dummy_server"} \ No newline at end of file diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index b2edf4314..7c70d4c64 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -7,6 +7,7 @@ import os import string from typing import Dict, Tuple, Union +import yaml import pytest @@ -15,6 +16,7 @@ LOCAL_APP_YAML, MERLIN_CONFIG_DIR, PASSWORD_LENGTH, + ServerStatus, check_process_file_format, config_merlin_server, create_server_config, @@ -587,8 +589,8 @@ def test_pull_server_image_no_image_path_no_config_path( """ # Set up mock calls to simulate the setup of this function config_dir = f"{server_testing_dir}/config_dir" - config_file = "nonexistent.yaml" - image_file = "nonexistent.sif" + config_file = "pull_server_image_no_image_path_no_config_path_config_nonexistent.yaml" + image_file = "pull_server_image_no_image_path_no_config_path_image_nonexistent.sif" setup_pull_server_image_mock( mocker, server_testing_dir, @@ -639,8 +641,8 @@ def test_pull_server_image_both_paths_exist( # Set up mock calls to simulate the setup of this function config_dir = f"{server_testing_dir}/config_dir" - config_file = "existent.yaml" - image_file = "existent.sif" + config_file = "pull_server_image_both_paths_exist_config.yaml" + image_file = "pull_server_image_both_paths_exist_image.sif" setup_pull_server_image_mock( mocker, server_testing_dir, @@ -675,8 +677,8 @@ def test_pull_server_image_os_error( """ # Set up mock calls to simulate the setup of this function config_dir = f"{server_testing_dir}/config_dir" - config_file = "existent.yaml" - image_file = "nonexistent.sif" + config_file = "pull_server_image_os_error_config.yaml" + image_file = "pull_server_image_os_error_config_nonexistent.sif" setup_pull_server_image_mock( mocker, server_testing_dir, @@ -693,3 +695,160 @@ def test_pull_server_image_os_error( # Run the test assert not pull_server_image() assert f"Destination location {config_dir} is not writable." in caplog.text + + +@pytest.mark.parametrize("server_config_exists, config_exists, image_exists, pfile_exists, expected_status", [ + (False, True, True, True, ServerStatus.NOT_INITALIZED), # No server config + (True, False, True, True, ServerStatus.NOT_INITALIZED), # Config dir does not exist + (True, True, False, True, ServerStatus.MISSING_CONTAINER), # Image path does not exist + (True, True, True, False, ServerStatus.NOT_RUNNING), # Pfile path does not exist +]) +def test_get_server_status_initial_checks( + mocker: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], + server_config_exists: bool, + config_exists: bool, + image_exists: bool, + pfile_exists: bool, + expected_status: ServerStatus, +): + """ + Test the `get_server_status` function for the initial conditional checks that it looks for. + These checks include: + - no server configuration -> should return NOT_INITIALIZED + - no config directory path -> should return NOT_INITIALIZED + - no image path -> should return MISSING_CONTAINER + - no password file -> should return NOT_RUNNING + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_config_exists: A boolean to denote whether the server config exists in this test or not + :param config_exists: A boolean to denote whether the config dir exists in this test or not + :param image_exists: A boolean to denote whether the image path exists in this test or not + :param pfile_exists: A boolean to denote whether the password file exists in this test or not + :param expected_status: The status we're expecting `get_server_status` to return for this test + """ + # Mock the necessary calls + if server_config_exists: + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value="config_dir") + mocker.patch("merlin.server.server_util.ContainerConfig.get_image_path", return_value="image_path") + mocker.patch("merlin.server.server_util.ContainerConfig.get_pfile_path", return_value="pfile_path") + + # Mock os.path.exists to return the desired values + mocker.patch("os.path.exists", side_effect=lambda path: { + "config_dir": config_exists, + "image_path": image_exists, + "pfile_path": pfile_exists + }.get(path, False)) + else: + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + + # Call the function and assert the expected status + assert get_server_status() == expected_status + + +@pytest.mark.parametrize("stdout_val, expected_status", [ + (b"", ServerStatus.NOT_RUNNING), # No stdout from subprocess + (b"Successfully started", ServerStatus.RUNNING), # Stdout from subprocess exists +]) +def test_get_server_status_subprocess_check( + mocker: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], + stdout_val: bytes, + expected_status: ServerStatus, +): + """ + Test the `get_server_status` function with empty stdout return from the subprocess run. + This should return a NOT_RUNNING status. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("os.path.exists", return_value=True) + mocker.patch("merlin.server.server_config.pull_process_file", return_value={"parent_pid": 123}) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = stdout_val + + assert get_server_status() == expected_status + + +@pytest.mark.parametrize("data_to_test, expected_result", [ + ({"image_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No parent_pid entry + ({"parent_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No image_pid entry + ({"parent_pid": 123, "image_pid": 456, "hostname": "dummy_server"}, False), # No port entry + ({"parent_pid": 123, "image_pid": 123, "port": 6379}, False), # No hostname entry + ({"parent_pid": 123, "image_pid": 123, "port": 6379, "hostname": "dummy_server"}, True), # All required entries exist +]) +def test_check_process_file_format(data_to_test: Dict[str, Union[int, str]], expected_result: bool): + """ + Test the `check_process_file_format` function. The first 4 parametrized tests above should all + return False as they're all missing a required key. The final parametrized test above should return + True since it has every required key. + + :param data_to_test: The data dict that we'll pass in to the `check_process_file_format` function + :param expected_result: The return value we expect based on `data_to_test` + """ + assert check_process_file_format(data_to_test) == expected_result + + +def test_pull_process_file_valid_file(server_testing_dir: str, server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `pull_process_file` function with a valid process file. This test will create a test + process file with valid contents that `pull_process_file` will read in and return. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_process_file_contents: A fixture representing process file contents + """ + # Create the valid process file in our temp testing directory + process_filepath = f"{server_testing_dir}/valid_process_file.yaml" + with open(process_filepath, 'w') as process_file: + yaml.dump(server_process_file_contents, process_file) + + # Run the test + assert pull_process_file(process_filepath) == server_process_file_contents + + +def test_pull_process_file_invalid_file(server_testing_dir: str, server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `pull_process_file` function with an invalid process file. This test will create a test + process file with invalid contents that `pull_process_file` will try to read in. Once it sees + that the file is invalid it will return None. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_process_file_contents: A fixture representing process file contents + """ + # Remove a key from the process file contents so that it's no longer valid + del server_process_file_contents["hostname"] + + # Create the invalid process file in our temp testing directory + process_filepath = f"{server_testing_dir}/invalid_process_file.yaml" + with open(process_filepath, 'w') as process_file: + yaml.dump(server_process_file_contents, process_file) + + # Run the test + assert pull_process_file(process_filepath) is None + + +def test_dump_process_file_invalid_file(server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `dump_process_file` function with invalid process file data. This should return False. + + :param server_process_file_contents: A fixture representing process file contents + """ + # Remove a key from the process file contents so that it's no longer valid and run the test + del server_process_file_contents["parent_pid"] + assert not dump_process_file(server_process_file_contents, "some_filepath.yaml") + + +def test_dump_process_file_valid_file(server_testing_dir: str, server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `dump_process_file` function with invalid process file data. This should return False. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_process_file_contents: A fixture representing process file contents + """ + process_filepath = f"{server_testing_dir}/dumped_process_file.yaml" + assert dump_process_file(server_process_file_contents, process_filepath) + assert os.path.exists(process_filepath) From e49f378fbdccabd5aebe11f7577eff80e8e26932 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Thu, 12 Sep 2024 08:37:26 -0700 Subject: [PATCH 123/201] Fixed kwargs being modified when making a copy for saving to redis worker args. --- merlin/study/celeryadapter.py | 2 +- tests/unit/study/test_celeryadapter.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 510e5a04e..0791c2a35 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -779,7 +779,7 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): # Adding the worker args to redis db with CeleryManager.get_worker_args_redis_connection() as redis_connection: - args = kwargs + args = kwargs.copy() # Save worker command with the arguements args["worker_cmd"] = worker_cmd # Store the nested dictionaries into a separate key with a link. diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py index 0cc6592dc..60e24bb9a 100644 --- a/tests/unit/study/test_celeryadapter.py +++ b/tests/unit/study/test_celeryadapter.py @@ -34,14 +34,9 @@ import json import os from datetime import datetime - -# from time import sleep from typing import Dict -# import pytest from celery import Celery - -# from celery.canvas import Signature from deepdiff import DeepDiff from merlin.config import Config @@ -51,6 +46,9 @@ from tests.unit.study.status_test_files.status_test_variables import SPEC_PATH +# from time import sleep +# import pytest +# from celery.canvas import Signature # @pytest.mark.order(before="TestInactive") # class TestActive: # """ From 72398e6d9f5fa213dd5f79c850d39155c0ef6747 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 12:00:36 -0700 Subject: [PATCH 124/201] add tests for server_commands.py --- merlin/server/server_commands.py | 101 +++- merlin/server/server_config.py | 6 +- tests/fixtures/server.py | 26 +- tests/unit/server/test_server_commands.py | 620 ++++++++++++++++++++++ tests/unit/server/test_server_config.py | 4 +- 5 files changed, 727 insertions(+), 30 deletions(-) create mode 100644 tests/unit/server/test_server_commands.py diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index ef156f1ab..7266007d1 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -48,7 +48,7 @@ pull_server_config, pull_server_image, ) -from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers +from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers, ServerConfig LOG = logging.getLogger("merlin") @@ -70,17 +70,13 @@ def init_server() -> None: LOG.info("Merlin server initialization successful.") -# Pylint complains that there's too many branches in this function but -# it looks clean to me so we'll ignore it -def config_server(args: Namespace) -> None: # pylint: disable=R0912 +def apply_config_changes(server_config: ServerConfig, args: Namespace): """ - Process the merlin server config flags to make changes and edits to appropriate configurations - based on the input passed in by the user. + Apply any configuration changes that the user is requesting. + + :param server_config: An instance of ServerConfig containing all the necessary configuration values + :param args: An argumentparser namespace object with args from the user """ - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False redis_config = RedisConfig(server_config.container.get_config_path()) redis_config.set_ip_address(args.ipaddress) @@ -114,14 +110,31 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 else: LOG.info("Add changes to config file and exisiting containers.") - server_config = pull_server_config() - if not server_config: + +# Pylint complains that there's too many branches in this function but +# it looks clean to me so we'll ignore it +def config_server(args: Namespace) -> None: # pylint: disable=R0912 + """ + Process the merlin server config flags to make changes and edits to appropriate configurations + based on the input passed in by the user. + + :param args: An argumentparser namespace object with args from the user + """ + server_config_before_changes = pull_server_config() + if not server_config_before_changes: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + apply_config_changes(server_config_before_changes, args) + + server_config_after_changes = pull_server_config() + if not server_config_after_changes: LOG.error('Try to run "merlin server init" again to reinitialize values.') return False # Read the user from the list of avaliable users - redis_users = RedisUsers(server_config.container.get_user_file_path()) - redis_config = RedisConfig(server_config.container.get_config_path()) + redis_users = RedisUsers(server_config_after_changes.container.get_user_file_path()) + redis_config = RedisConfig(server_config_after_changes.container.get_config_path()) if args.add_user is not None: # Log the user in a file @@ -155,7 +168,7 @@ def status_server() -> None: Get the server status of the any current running containers for merlin server """ current_status = get_server_status() - if current_status == ServerStatus.NOT_INITALIZED: + if current_status == ServerStatus.NOT_INITIALIZED: LOG.info("Merlin server has not been initialized.") LOG.info("Please initalize server by running 'merlin server init'") elif current_status == ServerStatus.MISSING_CONTAINER: @@ -167,15 +180,17 @@ def status_server() -> None: LOG.info("Merlin server is running.") -def start_server() -> bool: # pylint: disable=R0911 +def check_for_not_running_server() -> bool: """ - Start a merlin server container using singularity. - :return:: True if server was successful started and False if failed. + When starting a server the status must be NOT_RUNNING. If it's any other + status we need to log an error for the user to see. + + :returns: True if the status is NOT_RUNNING. False otherwise. """ current_status = get_server_status() uninitialized_err = "Merlin server has not been intitialized. Please run 'merlin server init' first." status_errors = { - ServerStatus.NOT_INITALIZED: uninitialized_err, + ServerStatus.NOT_INITIALIZED: uninitialized_err, ServerStatus.MISSING_CONTAINER: uninitialized_err, ServerStatus.RUNNING: """Merlin server already running. Stop current server with 'merlin server stop' before attempting to start a new server.""", @@ -184,12 +199,17 @@ def start_server() -> bool: # pylint: disable=R0911 if current_status in status_errors: LOG.info(status_errors[current_status]) return False + + return True - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False +def start_redis_container(server_config: ServerConfig) -> subprocess.Popen: + """ + Given a server configuration, use it to start up a container that hosts redis. + + :param server_config: The ServerConfig instance that holds information about the redis server to start + :returns: A subprocess started with subprocess.Popen that's executing the command to start the container + """ image_path = server_config.container.get_image_path() config_path = server_config.container.get_config_path() path_errors = { @@ -200,7 +220,7 @@ def start_server() -> bool: # pylint: disable=R0911 for path in (image_path, config_path): if not os.path.exists(path): LOG.error(f"Unable to find {path_errors[path]} at {path}") - return False + return None # Pylint wants us to use with here but we don't need that process = subprocess.Popen( # pylint: disable=R1732 @@ -220,6 +240,17 @@ def start_server() -> bool: # pylint: disable=R0911 time.sleep(1) + return process + + +def server_started(process: subprocess.Popen, server_config: ServerConfig) -> bool: + """ + Check that the server spun up by `start_redis_container` was started properly. + + :param process: The subprocess that was started by `start_redis_container` + :param server_config: The ServerConfig instance that holds information about the redis server to start + :returns: True if the server started properly. False otherwise. + """ redis_start, redis_out = parse_redis_output(process.stdout) if not redis_start: @@ -241,6 +272,28 @@ def start_server() -> bool: # pylint: disable=R0911 LOG.info(f"Server started with PID {str(process.pid)}.") LOG.info(f'Merlin server operating on "{redis_out["hostname"]}" and port "{redis_out["port"]}".') + return True + + +def start_server() -> bool: # pylint: disable=R0911 + """ + Start a merlin server container using singularity. + :return:: True if server was successful started and False if failed. + """ + if not check_for_not_running_server(): + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + process = start_redis_container(server_config) + if process is None: + return False + + if not server_started(process, server_config): + return False redis_users = RedisUsers(server_config.container.get_user_file_path()) redis_config = RedisConfig(server_config.container.get_config_path()) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index fe79599bc..93e5b0baa 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -77,7 +77,7 @@ class ServerStatus(enum.Enum): """ RUNNING = 0 - NOT_INITALIZED = 1 + NOT_INITIALIZED = 1 MISSING_CONTAINER = 2 NOT_RUNNING = 3 ERROR = 4 @@ -349,10 +349,10 @@ def get_server_status(): """ server_config = pull_server_config() if not server_config: - return ServerStatus.NOT_INITALIZED + return ServerStatus.NOT_INITIALIZED if not os.path.exists(server_config.container.get_config_dir()): - return ServerStatus.NOT_INITALIZED + return ServerStatus.NOT_INITIALIZED if not os.path.exists(server_config.container.get_image_path()): return ServerStatus.MISSING_CONTAINER diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index a955ebb08..4f4a07a2c 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -3,6 +3,7 @@ """ import os +from argparse import Namespace from typing import Dict, Union import pytest @@ -281,4 +282,27 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> @pytest.fixture(scope="function") def server_process_file_contents() -> str: """Fixture to represent process file contents.""" - return {"parent_pid": 123, "image_pid": 456, "port": 6379, "hostname": "dummy_server"} \ No newline at end of file + return {"parent_pid": 123, "image_pid": 456, "port": 6379, "hostname": "dummy_server"} + + +@pytest.fixture(scope="function") +def server_config_server_args() -> Namespace: + """ + Setup an argparse Namespace with all args that the `config_server` + function will need. These can be modified on a test-by-test basis. + + :returns: An argparse Namespace with args needed by `config_server` + """ + return Namespace( + ipaddress=None, + port=None, + password=None, + directory=None, + snapshot_seconds=None, + snapshot_changes=None, + snapshot_file=None, + append_mode=None, + append_file=None, + add_user=None, + remove_user=None, + ) diff --git a/tests/unit/server/test_server_commands.py b/tests/unit/server/test_server_commands.py new file mode 100644 index 000000000..2deb64c80 --- /dev/null +++ b/tests/unit/server/test_server_commands.py @@ -0,0 +1,620 @@ +""" +Tests for the `server_commands.py` module. +""" +import logging +import os +import pytest +import subprocess +from argparse import Namespace +from typing import Dict, List + +from merlin.server.server_commands import ( + check_for_not_running_server, + config_server, + init_server, + restart_server, + server_started, + start_redis_container, + start_server, + status_server, + stop_server +) +from merlin.server.server_config import ServerStatus +from merlin.server.server_util import ServerConfig + + +def test_init_server_create_server_fail(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `init_server` function with `create_server_config` returning False. + This should log a failure message. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + caplog.set_level(logging.INFO) + create_server_mock = mocker.patch("merlin.server.server_commands.create_server_config", return_value=False) + init_server() + create_server_mock.assert_called_once() + assert "Merlin server initialization failed." in caplog.text + + +def test_init_server_create_server_success(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `init_server` function with `create_server_config` returning True. + This should log a sucess message. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + caplog.set_level(logging.INFO) + create_server_mock = mocker.patch("merlin.server.server_commands.create_server_config", return_value=True) + pull_server_mock = mocker.patch("merlin.server.server_commands.pull_server_image", return_value=True) + config_merlin_mock = mocker.patch("merlin.server.server_commands.config_merlin_server", return_value=True) + init_server() + create_server_mock.assert_called_once() + pull_server_mock.assert_called_once() + config_merlin_mock.assert_called_once() + assert "Merlin server initialization successful." in caplog.text + + +def test_config_server_no_server_config( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_config_server_args: Namespace, +): + """ + Test the `config_server` function with no server config. This should log an error + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_config_server_args: An argparse Namespace with args needed by `config_server` + """ + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=None) + assert not config_server(server_config_server_args) + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +@pytest.mark.parametrize("server_status, status_name", [ + (ServerStatus.RUNNING, "running"), + (ServerStatus.NOT_RUNNING, "not_running"), +]) +def test_config_server_add_user_remove_user_success( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_config_server_args: Namespace, + server_server_config: Dict[str, Dict[str, str]], + server_status: ServerStatus, + status_name: str, +): + """ + Test the `config_server` function by adding and removing a user. This will be ran with and without + the server status being set to RUNNING. For each scenario we should expect: + - RUNNING -> RedisUsers.write and RedisUsers.apply_to_redis are both called twice + - NOT_RUNNING -> RedisUsers.write is called twice and RedisUsers.apply_to_redis is not called at all + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_config_server_args: An argparse Namespace with args needed by `config_server` + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_status: The server status for this test (either RUNNING or NOT_RUNNING) + :param status_name: The name of the status in string form so we can have unique users for each test + """ + caplog.set_level(logging.INFO) + + # Set up the add_user and remove_user calls to test + user_to_add_and_remove = f"test_config_server_modification_user_{status_name}" + server_config_server_args.add_user = [user_to_add_and_remove, "test_config_server_modification_password"] + server_config_server_args.remove_user = user_to_add_and_remove + + # Create mocks of the necessary calls for this function + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.apply_config_changes") + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + write_mock = mocker.patch("merlin.server.server_util.RedisUsers.write") + apply_to_redis_mock = mocker.patch("merlin.server.server_util.RedisUsers.apply_to_redis") + + # Run the test + expected_apply_calls = 2 if server_status == ServerStatus.RUNNING else 0 + assert config_server(server_config_server_args) is None + assert write_mock.call_count == 2 + assert apply_to_redis_mock.call_count == expected_apply_calls + assert f"Added user {user_to_add_and_remove} to merlin server" in caplog.text + assert f"Removed user {user_to_add_and_remove} to merlin server" in caplog.text + + +def test_config_server_add_user_remove_user_failure( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_config_server_args: Namespace, + server_server_config: Dict[str, Dict[str, str]], + +): + """ + Test the `config_server` function by attempting to add a user that already exists (we do this through mock) + and removing a user that doesn't exist. This should run to completion but never call RedisUsers.write. It + should also log two error messages. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_config_server_args: An argparse Namespace with args needed by `config_server` + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + # Set up the add_user and remove_user calls to test (these users should never actually be added/removed) + user_to_add_and_remove = "test_config_user_not_ever_added" + server_config_server_args.add_user = [user_to_add_and_remove, "test_config_server_modification_password"] + server_config_server_args.remove_user = user_to_add_and_remove + + # Create mocks of the necessary calls for this function + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.apply_config_changes") + mocker.patch("merlin.server.server_util.RedisUsers.add_user", return_value=False) + write_mock = mocker.patch("merlin.server.server_util.RedisUsers.write") + + # Run the test + assert config_server(server_config_server_args) is None + assert write_mock.call_count == 0 + assert f"User '{user_to_add_and_remove}' already exisits within current users" in caplog.text + assert f"User '{user_to_add_and_remove}' doesn't exist within current users." in caplog.text + + +@pytest.mark.parametrize("server_status, expected_log_msgs", [ + (ServerStatus.NOT_INITIALIZED, ["Merlin server has not been initialized.", "Please initalize server by running 'merlin server init'"]), + (ServerStatus.MISSING_CONTAINER, ["Unable to find server image.", "Ensure there is a .sif file in merlin server directory."]), + (ServerStatus.NOT_RUNNING, ["Merlin server is not running."]), + (ServerStatus.RUNNING, ["Merlin server is running."]), +]) +def test_status_server( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_status: ServerStatus, + expected_log_msgs: List[str], +): + """ + Test the `status_server` function to make sure it produces the correct logs for each status. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + :param expected_log_msgs: The logs we're expecting from this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + status_server() + for expected_log_msg in expected_log_msgs: + assert expected_log_msg in caplog.text + + +@pytest.mark.parametrize("server_status, expected_result, expected_log_msg", [ + (ServerStatus.NOT_INITIALIZED, False, "Merlin server has not been intitialized. Please run 'merlin server init' first."), + (ServerStatus.MISSING_CONTAINER, False, "Merlin server has not been intitialized. Please run 'merlin server init' first."), + (ServerStatus.NOT_RUNNING, True, None), + (ServerStatus.RUNNING, False, """Merlin server already running. + Stop current server with 'merlin server stop' before attempting to start a new server."""), +]) +def test_check_for_not_running_server( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_status: ServerStatus, + expected_result: bool, + expected_log_msg: str, +): + """ + Test the `check_for_not_running_server` function with different server statuses. + There should be a logged message for each status and the results we should expect are as + follows: + - NOT_RUNNING status should return True + - any other status should return False + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + :param expected_result: The expected result (T/F) for this test + :param expected_log_msg: The log we're expecting from this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + assert check_for_not_running_server() == expected_result + if expected_log_msg is not None: + assert expected_log_msg in caplog.text + + +def test_start_redis_container_invalid_image_path( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `start_redis_container` function with a nonexistent image path. + This should log an error and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + image_file = "nonexistent.image" + server_server_config["container"]["image"] = image_file + server_server_config["container"]["config"] = "start_redis_container.config" + + # Create the config path so we ensure it exists + config_path = f"{server_testing_dir}/{server_server_config['container']['config']}" + with open(config_path, "w"): + pass + + assert start_redis_container(ServerConfig(server_server_config)) is None + assert f"Unable to find image at {os.path.join(server_testing_dir, image_file)}" in caplog.text + + +def test_start_redis_container_invalid_config_path( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `start_redis_container` function with a nonexistent config path. + This should log an error and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + config_file = "nonexistent.config" + server_server_config["container"]["image"] = "start_redis_container.image" + server_server_config["container"]["config"] = config_file + + # Create the config path so we ensure it exists + image_path = f"{server_testing_dir}/{server_server_config['container']['image']}" + with open(image_path, "w"): + pass + + assert start_redis_container(ServerConfig(server_server_config)) is None + assert f"Unable to find config file at {os.path.join(server_testing_dir, config_file)}" in caplog.text + + +def test_start_redis_container_valid_paths(mocker: "Fixture", server_server_config: Dict[str, Dict[str, str]]): # noqa: F821 + """ + Test the `start_redis_container` function with valid image and config paths. + This should return a subprocess. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + expected_return = "fake subprocess" + mocker.patch("subprocess.Popen", return_value=expected_return) + mocker.patch("os.path.exists", return_value=True) + assert start_redis_container(ServerConfig(server_server_config)) == expected_return + + +def test_server_started_no_redis_start(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `server_started` function with redis not starting. This should log errors and + return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mock_process = mocker.Mock() + mock_process.stdout = mocker.Mock() + + expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(False, expected_redis_out_msg)) + + assert not server_started(mock_process, "unecessary_config") + assert "Redis is unable to start" in caplog.text + assert 'Check to see if there is an unresponsive instance of redis with "ps -e"' in caplog.text + assert expected_redis_out_msg in caplog.text + + +def test_server_started_process_file_dump_fail( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `server_started` function with the dump to the process file failing. + This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mock_process = mocker.Mock() + mock_process.pid = 1234 + mock_process.stdout = mocker.Mock() + + image_pid = 5678 + expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid})) + mocker.patch("merlin.server.server_commands.dump_process_file", return_value=False) + + assert not server_started(mock_process, ServerConfig(server_server_config)) + assert "Unable to create process file for container." in caplog.text + + +@pytest.mark.parametrize("server_status", [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, +]) +def test_server_started_server_not_running( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], + server_status: ServerStatus, +): + """ + Test the `server_started` function with the server status returning a non-running status. + This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_status: The server status for this test + """ + mock_process = mocker.Mock() + mock_process.pid = 1234 + mock_process.stdout = mocker.Mock() + + image_pid = 5678 + expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid})) + mocker.patch("merlin.server.server_commands.dump_process_file", return_value=True) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + + assert not server_started(mock_process, ServerConfig(server_server_config)) + assert "Unable to start merlin server." in caplog.text + + +def test_server_started_no_issues(mocker: "Fixture",server_server_config: Dict[str, Dict[str, str]]): # noqa: F821 + """ + Test the `server_started` function with no issues starting the server. + This should return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mock_process = mocker.Mock() + mock_process.pid = 1234 + mock_process.stdout = mocker.Mock() + + image_pid = 5678 + expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" + mocker.patch( + "merlin.server.server_commands.parse_redis_output", + return_value=(True, {"pid": image_pid, "port": 6379}) + ) + mocker.patch("merlin.server.server_commands.dump_process_file", return_value=True) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + + assert server_started(mock_process, ServerConfig(server_server_config)) + + +def test_start_server_no_running_server(mocker: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with no running server. This should return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=False) + assert not start_server() + + +def test_start_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with no running server. This should return False + and log an error. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=None) + assert not start_server() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_start_server_redis_container_startup_fail(mocker: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with the redis container startup failing. This should return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=True) + mocker.patch("merlin.server.server_commands.start_redis_container", return_value=None) + assert not start_server() + + +def test_start_server_server_did_not_start(mocker: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with the server startup failing. This should return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=True) + mocker.patch("merlin.server.server_commands.start_redis_container", return_value=True) + mocker.patch("merlin.server.server_commands.server_started", return_value=False) + assert not start_server() + + +def test_start_server_successful_start( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], + server_redis_conf_file: str, +): + """ + Test the `start_server` function with a successful start. This should return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.start_redis_container", return_value=True) + mocker.patch("merlin.server.server_commands.server_started", return_value=True) + mocker.patch("merlin.server.server_commands.RedisUsers") + mocker.patch("merlin.server.server_commands.RedisConfig") + mocker.patch("merlin.server.server_commands.AppYaml") + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_path", return_value=server_redis_conf_file) + mocker.patch("os.path.join", return_value=f"{server_testing_dir}/start_server_app.yaml") + + assert start_server() + + +@pytest.mark.parametrize("server_status", [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, +]) +def test_stop_server_server_not_running( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_status: ServerStatus +): + """ + Test the `stop_server` function with a server that's not running. This should log two messages + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + assert not stop_server() + assert "There is no instance of merlin server running." in caplog.text + assert "Start a merlin server first with 'merlin server start'" in caplog.text + + +def test_stop_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): + """ + Test the `stop_server` function with no server config being pulled. This should log a message + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=None) + assert not stop_server() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_stop_server_empty_stdout( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: ServerConfig, +): + """ + Test the `stop_server` function with no server config being pulled. This should log a message + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = b"" + assert not stop_server() + assert "Unable to get the PID for the current merlin server." in caplog.text + + +def test_stop_server_unable_to_stop_server( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: ServerConfig, +): + """ + Test the `stop_server` function with the server status still RUNNING after trying + to kill the server. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = b"some output from status check" + assert not stop_server() + assert "Unable to kill process." in caplog.text + + +def test_stop_server_stop_command_is_not_kill( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: ServerConfig, +): + """ + Test the `stop_server` function with a stop command that's not 'kill'. + This should run through the command successfully and return True. The subprocess + should run the command we provide in this test. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_commands.get_server_status", side_effect=[ServerStatus.RUNNING, ServerStatus.NOT_RUNNING]) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) + custom_stop_command = "not a kill command" + mocker.patch("merlin.server.server_util.ContainerFormatConfig.get_stop_command", return_value=custom_stop_command) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = b"some output from status check" + assert stop_server() + mock_run.assert_called_with(custom_stop_command.split(), stdout=subprocess.PIPE) + + +@pytest.mark.parametrize("server_status", [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, +]) +def test_restart_server_server_not_running( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_status: ServerStatus +): + """ + Test the `restart_server` function with a server that's not running. + This should log two messages and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + assert not restart_server() + assert "Merlin server is not currently running." in caplog.text + assert "Please start a merlin server instance first with 'merlin server start'" in caplog.text + + +def test_restart_server_successful_restart(mocker: "Fixture"): # noqa: F821 + """ + Test the `restart_server` function with a successful restart. This should call + `stop_server` and `start_server`, and return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + stop_server_mock = mocker.patch("merlin.server.server_commands.stop_server") + start_server_mock = mocker.patch("merlin.server.server_commands.start_server") + assert restart_server() + stop_server_mock.assert_called_once() + start_server_mock.assert_called_once() diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 7c70d4c64..092aa6eb7 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -698,8 +698,8 @@ def test_pull_server_image_os_error( @pytest.mark.parametrize("server_config_exists, config_exists, image_exists, pfile_exists, expected_status", [ - (False, True, True, True, ServerStatus.NOT_INITALIZED), # No server config - (True, False, True, True, ServerStatus.NOT_INITALIZED), # Config dir does not exist + (False, True, True, True, ServerStatus.NOT_INITIALIZED), # No server config + (True, False, True, True, ServerStatus.NOT_INITIALIZED), # Config dir does not exist (True, True, False, True, ServerStatus.MISSING_CONTAINER), # Image path does not exist (True, True, True, False, ServerStatus.NOT_RUNNING), # Pfile path does not exist ]) From ff4f649c04365bfc57a27028b19403fadb647bad Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 12:11:18 -0700 Subject: [PATCH 125/201] run fix-style --- merlin/server/server_commands.py | 2 +- tests/unit/server/test_server_commands.py | 145 +++++++++++++--------- tests/unit/server/test_server_config.py | 112 +++++++++-------- 3 files changed, 144 insertions(+), 115 deletions(-) diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 7266007d1..b0015473a 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -199,7 +199,7 @@ def check_for_not_running_server() -> bool: if current_status in status_errors: LOG.info(status_errors[current_status]) return False - + return True diff --git a/tests/unit/server/test_server_commands.py b/tests/unit/server/test_server_commands.py index 2deb64c80..4261a1f8d 100644 --- a/tests/unit/server/test_server_commands.py +++ b/tests/unit/server/test_server_commands.py @@ -1,13 +1,15 @@ """ Tests for the `server_commands.py` module. """ + import logging import os -import pytest import subprocess from argparse import Namespace from typing import Dict, List +import pytest + from merlin.server.server_commands import ( check_for_not_running_server, config_server, @@ -17,7 +19,7 @@ start_redis_container, start_server, status_server, - stop_server + stop_server, ) from merlin.server.server_config import ServerStatus from merlin.server.server_util import ServerConfig @@ -75,10 +77,13 @@ def test_config_server_no_server_config( assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text -@pytest.mark.parametrize("server_status, status_name", [ - (ServerStatus.RUNNING, "running"), - (ServerStatus.NOT_RUNNING, "not_running"), -]) +@pytest.mark.parametrize( + "server_status, status_name", + [ + (ServerStatus.RUNNING, "running"), + (ServerStatus.NOT_RUNNING, "not_running"), + ], +) def test_config_server_add_user_remove_user_success( mocker: "Fixture", # noqa: F821 caplog: "Fixture", # noqa: F821 @@ -93,7 +98,7 @@ def test_config_server_add_user_remove_user_success( the server status being set to RUNNING. For each scenario we should expect: - RUNNING -> RedisUsers.write and RedisUsers.apply_to_redis are both called twice - NOT_RUNNING -> RedisUsers.write is called twice and RedisUsers.apply_to_redis is not called at all - + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object :param caplog: A built-in fixture from the pytest library to capture logs :param server_testing_dir: The path to the the temp output directory for server tests @@ -130,7 +135,6 @@ def test_config_server_add_user_remove_user_failure( caplog: "Fixture", # noqa: F821 server_config_server_args: Namespace, server_server_config: Dict[str, Dict[str, str]], - ): """ Test the `config_server` function by attempting to add a user that already exists (we do this through mock) @@ -160,12 +164,21 @@ def test_config_server_add_user_remove_user_failure( assert f"User '{user_to_add_and_remove}' doesn't exist within current users." in caplog.text -@pytest.mark.parametrize("server_status, expected_log_msgs", [ - (ServerStatus.NOT_INITIALIZED, ["Merlin server has not been initialized.", "Please initalize server by running 'merlin server init'"]), - (ServerStatus.MISSING_CONTAINER, ["Unable to find server image.", "Ensure there is a .sif file in merlin server directory."]), - (ServerStatus.NOT_RUNNING, ["Merlin server is not running."]), - (ServerStatus.RUNNING, ["Merlin server is running."]), -]) +@pytest.mark.parametrize( + "server_status, expected_log_msgs", + [ + ( + ServerStatus.NOT_INITIALIZED, + ["Merlin server has not been initialized.", "Please initalize server by running 'merlin server init'"], + ), + ( + ServerStatus.MISSING_CONTAINER, + ["Unable to find server image.", "Ensure there is a .sif file in merlin server directory."], + ), + (ServerStatus.NOT_RUNNING, ["Merlin server is not running."]), + (ServerStatus.RUNNING, ["Merlin server is running."]), + ], +) def test_status_server( mocker: "Fixture", # noqa: F821 caplog: "Fixture", # noqa: F821 @@ -187,13 +200,28 @@ def test_status_server( assert expected_log_msg in caplog.text -@pytest.mark.parametrize("server_status, expected_result, expected_log_msg", [ - (ServerStatus.NOT_INITIALIZED, False, "Merlin server has not been intitialized. Please run 'merlin server init' first."), - (ServerStatus.MISSING_CONTAINER, False, "Merlin server has not been intitialized. Please run 'merlin server init' first."), - (ServerStatus.NOT_RUNNING, True, None), - (ServerStatus.RUNNING, False, """Merlin server already running. - Stop current server with 'merlin server stop' before attempting to start a new server."""), -]) +@pytest.mark.parametrize( + "server_status, expected_result, expected_log_msg", + [ + ( + ServerStatus.NOT_INITIALIZED, + False, + "Merlin server has not been intitialized. Please run 'merlin server init' first.", + ), + ( + ServerStatus.MISSING_CONTAINER, + False, + "Merlin server has not been intitialized. Please run 'merlin server init' first.", + ), + (ServerStatus.NOT_RUNNING, True, None), + ( + ServerStatus.RUNNING, + False, + """Merlin server already running. + Stop current server with 'merlin server stop' before attempting to start a new server.""", + ), + ], +) def test_check_for_not_running_server( mocker: "Fixture", # noqa: F821 caplog: "Fixture", # noqa: F821 @@ -248,7 +276,7 @@ def test_start_redis_container_invalid_image_path( assert start_redis_container(ServerConfig(server_server_config)) is None assert f"Unable to find image at {os.path.join(server_testing_dir, image_file)}" in caplog.text - + def test_start_redis_container_invalid_config_path( mocker: "Fixture", # noqa: F821 caplog: "Fixture", # noqa: F821 @@ -301,7 +329,7 @@ def test_server_started_no_redis_start(mocker: "Fixture", caplog: "Fixture"): # """ mock_process = mocker.Mock() mock_process.stdout = mocker.Mock() - + expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(False, expected_redis_out_msg)) @@ -327,9 +355,8 @@ def test_server_started_process_file_dump_fail( mock_process = mocker.Mock() mock_process.pid = 1234 mock_process.stdout = mocker.Mock() - + image_pid = 5678 - expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid})) mocker.patch("merlin.server.server_commands.dump_process_file", return_value=False) @@ -337,11 +364,14 @@ def test_server_started_process_file_dump_fail( assert "Unable to create process file for container." in caplog.text -@pytest.mark.parametrize("server_status", [ - ServerStatus.NOT_RUNNING, - ServerStatus.MISSING_CONTAINER, - ServerStatus.NOT_INITIALIZED, -]) +@pytest.mark.parametrize( + "server_status", + [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, + ], +) def test_server_started_server_not_running( mocker: "Fixture", # noqa: F821 caplog: "Fixture", # noqa: F821 @@ -360,9 +390,8 @@ def test_server_started_server_not_running( mock_process = mocker.Mock() mock_process.pid = 1234 mock_process.stdout = mocker.Mock() - + image_pid = 5678 - expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid})) mocker.patch("merlin.server.server_commands.dump_process_file", return_value=True) mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) @@ -371,7 +400,7 @@ def test_server_started_server_not_running( assert "Unable to start merlin server." in caplog.text -def test_server_started_no_issues(mocker: "Fixture",server_server_config: Dict[str, Dict[str, str]]): # noqa: F821 +def test_server_started_no_issues(mocker: "Fixture", server_server_config: Dict[str, Dict[str, str]]): # noqa: F821 """ Test the `server_started` function with no issues starting the server. This should return True. @@ -383,13 +412,9 @@ def test_server_started_no_issues(mocker: "Fixture",server_server_config: Dict[s mock_process = mocker.Mock() mock_process.pid = 1234 mock_process.stdout = mocker.Mock() - + image_pid = 5678 - expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" - mocker.patch( - "merlin.server.server_commands.parse_redis_output", - return_value=(True, {"pid": image_pid, "port": 6379}) - ) + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid, "port": 6379})) mocker.patch("merlin.server.server_commands.dump_process_file", return_value=True) mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) @@ -472,15 +497,16 @@ def test_start_server_successful_start( assert start_server() -@pytest.mark.parametrize("server_status", [ - ServerStatus.NOT_RUNNING, - ServerStatus.MISSING_CONTAINER, - ServerStatus.NOT_INITIALIZED, -]) +@pytest.mark.parametrize( + "server_status", + [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, + ], +) def test_stop_server_server_not_running( - mocker: "Fixture", # noqa: F821 - caplog: "Fixture", # noqa: F821 - server_status: ServerStatus + mocker: "Fixture", caplog: "Fixture", server_status: ServerStatus # noqa: F821 # noqa: F821 ): """ Test the `stop_server` function with a server that's not running. This should log two messages @@ -497,7 +523,7 @@ def test_stop_server_server_not_running( assert "Start a merlin server first with 'merlin server start'" in caplog.text -def test_stop_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): +def test_stop_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 """ Test the `stop_server` function with no server config being pulled. This should log a message and return False. @@ -569,7 +595,9 @@ def test_stop_server_stop_command_is_not_kill( :param caplog: A built-in fixture from the pytest library to capture logs :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ - mocker.patch("merlin.server.server_commands.get_server_status", side_effect=[ServerStatus.RUNNING, ServerStatus.NOT_RUNNING]) + mocker.patch( + "merlin.server.server_commands.get_server_status", side_effect=[ServerStatus.RUNNING, ServerStatus.NOT_RUNNING] + ) mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) custom_stop_command = "not a kill command" @@ -580,16 +608,15 @@ def test_stop_server_stop_command_is_not_kill( mock_run.assert_called_with(custom_stop_command.split(), stdout=subprocess.PIPE) -@pytest.mark.parametrize("server_status", [ - ServerStatus.NOT_RUNNING, - ServerStatus.MISSING_CONTAINER, - ServerStatus.NOT_INITIALIZED, -]) -def test_restart_server_server_not_running( - mocker: "Fixture", # noqa: F821 - caplog: "Fixture", # noqa: F821 - server_status: ServerStatus -): +@pytest.mark.parametrize( + "server_status", + [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, + ], +) +def test_restart_server_server_not_running(mocker: "Fixture", caplog: "Fixture", server_status: ServerStatus): # noqa: F821 """ Test the `restart_server` function with a server that's not running. This should log two messages and return False. diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 092aa6eb7..2528648dc 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -7,13 +7,11 @@ import os import string from typing import Dict, Tuple, Union -import yaml import pytest +import yaml -from merlin.server.server_util import CONTAINER_TYPES, MERLIN_SERVER_SUBDIR, ServerConfig from merlin.server.server_config import ( - LOCAL_APP_YAML, MERLIN_CONFIG_DIR, PASSWORD_LENGTH, ServerStatus, @@ -29,6 +27,8 @@ pull_server_image, write_container_command_files, ) +from merlin.server.server_util import CONTAINER_TYPES, MERLIN_SERVER_SUBDIR, ServerConfig + try: from importlib import resources @@ -136,7 +136,7 @@ def test_write_container_command_files_with_existing_files( :param server_testing_dir: The path to the the temp output directory for server tests """ caplog.set_level(logging.INFO) - mocker.patch('os.path.exists', return_value=True) + mocker.patch("os.path.exists", return_value=True) assert write_container_command_files(server_testing_dir) file_names = [f"{container}.yaml" for container in CONTAINER_TYPES] for file in file_names: @@ -159,7 +159,7 @@ def test_write_container_command_files_with_nonexisting_files( caplog.set_level(logging.INFO) # Mock the os.path.exists function so it returns False - mocker.patch('os.path.exists', return_value=False) + mocker.patch("os.path.exists", return_value=False) # Mock the resources.path context manager mock_path = mocker.patch("merlin.server.server_config.resources.path") @@ -209,7 +209,7 @@ def test_create_server_config_merlin_config_dir_nonexistent( :param server_testing_dir: The path to the the temp output directory for server tests """ nonexistent_dir = f"{server_testing_dir}/merlin_config_dir" - mocker.patch('merlin.server.server_config.MERLIN_CONFIG_DIR', nonexistent_dir) + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", nonexistent_dir) assert not create_server_config() assert f"Unable to find main merlin configuration directory at {nonexistent_dir}" in caplog.text @@ -230,8 +230,8 @@ def test_create_server_config_server_subdir_nonexistent_oserror( # Mock MERLIN_CONFIG_DIR and MERLIN_SERVER_SUBDIR nonexistent_server_subdir = "test_create_server_config_server_subdir_nonexistent" - mocker.patch('merlin.server.server_config.MERLIN_CONFIG_DIR', server_testing_dir) - mocker.patch('merlin.server.server_config.MERLIN_SERVER_SUBDIR', nonexistent_server_subdir) + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mocker.patch("merlin.server.server_config.MERLIN_SERVER_SUBDIR", nonexistent_server_subdir) # Mock os.mkdir so it raises an OSError err_msg = "File not writeable" @@ -257,7 +257,7 @@ def test_create_server_config_no_server_config( # Mock the necessary variables/functions to get us to the pull_server_config call mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) mocker.patch("merlin.server.server_config.write_container_command_files", return_value=True) - mock_open_func = mocker.mock_open(read_data='key: value') + mock_open_func = mocker.mock_open(read_data="key: value") mocker.patch("builtins.open", mock_open_func) # Mock the pull_server_config call (what we're actually testing) and run the test @@ -287,7 +287,7 @@ def test_create_server_config_no_server_dir( # Mock the necessary variables/functions to get us to the get_config_dir call mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) mocker.patch("merlin.server.server_config.write_container_command_files", return_value=True) - mock_open_func = mocker.mock_open(read_data='key: value') + mock_open_func = mocker.mock_open(read_data="key: value") mocker.patch("builtins.open", mock_open_func) mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) @@ -380,7 +380,7 @@ def test_config_merlin_server_pass_user_dont_exist( def setup_pull_server_config_mock( - mocker: "Fixture", + mocker: "Fixture", # noqa: F821 server_testing_dir: str, server_app_yaml_contents: Dict[str, Union[str, int]], server_server_config: Dict[str, Dict[str, str]], @@ -394,7 +394,7 @@ def setup_pull_server_config_mock( :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ mocker.patch("merlin.server.server_util.AppYaml.get_data", return_value=server_app_yaml_contents) - mocker.patch('merlin.server.server_config.MERLIN_CONFIG_DIR', server_testing_dir) + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) mock_data = mocker.mock_open(read_data=str(server_server_config)) mocker.patch("builtins.open", mock_data) @@ -404,8 +404,8 @@ def setup_pull_server_config_mock( [ ("container", 'Unable to find "container" object in {default_app_yaml}'), ("container.format", 'Unable to find "format" in {default_app_yaml}'), - ("process", 'Process config not found in {default_app_yaml}'), - ] + ("process", "Process config not found in {default_app_yaml}"), + ], ) def test_pull_server_config_missing_config_keys( mocker: "Fixture", # noqa: F821 @@ -429,7 +429,7 @@ def test_pull_server_config_missing_config_keys( :param expected_log_message: The expected log message when the key is missing """ # Handle nested key deletion - keys = key_to_delete.split('.') + keys = key_to_delete.split(".") temp_app_yaml = server_app_yaml_contents for key in keys[:-1]: temp_app_yaml = temp_app_yaml[key] @@ -545,14 +545,13 @@ def setup_pull_server_image_mock( image_file: str, create_config_file: bool = False, create_image_file: bool = False, - ): """ Set up the necessary mock calls for the `pull_server_image` function. :param mocker: A built-in fixture from the pytest-mock library to create a Mock object :param server_testing_dir: The path to the the temp output directory for server tests - :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ image_url = "docker://redis" image_path = f"{server_testing_dir}/{image_file}" @@ -567,7 +566,7 @@ def setup_pull_server_image_mock( os.mkdir(config_dir) with open(os.path.join(config_dir, config_file), "w"): pass - + if create_image_file: with open(image_path, "w"): pass @@ -585,20 +584,13 @@ def test_pull_server_image_no_image_path_no_config_path( :param mocker: A built-in fixture from the pytest-mock library to create a Mock object :param server_testing_dir: The path to the the temp output directory for server tests - :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ # Set up mock calls to simulate the setup of this function config_dir = f"{server_testing_dir}/config_dir" config_file = "pull_server_image_no_image_path_no_config_path_config_nonexistent.yaml" image_file = "pull_server_image_no_image_path_no_config_path_image_nonexistent.sif" - setup_pull_server_image_mock( - mocker, - server_testing_dir, - server_server_config, - config_dir, - config_file, - image_file - ) + setup_pull_server_image_mock(mocker, server_testing_dir, server_server_config, config_dir, config_file, image_file) mocked_subprocess = mocker.patch("subprocess.run") # Mock the open function @@ -635,7 +627,7 @@ def test_pull_server_image_both_paths_exist( :param mocker: A built-in fixture from the pytest-mock library to create a Mock object :param caplog: A built-in fixture from the pytest library to capture logs :param server_testing_dir: The path to the the temp output directory for server tests - :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class """ caplog.set_level(logging.INFO) @@ -660,7 +652,7 @@ def test_pull_server_image_both_paths_exist( def test_pull_server_image_os_error( - mocker: "Fixture", + mocker: "Fixture", # noqa: F821 caplog: "Fixture", # noqa: F821 server_testing_dir: str, server_server_config: Dict[str, Dict[str, str]], @@ -697,12 +689,15 @@ def test_pull_server_image_os_error( assert f"Destination location {config_dir} is not writable." in caplog.text -@pytest.mark.parametrize("server_config_exists, config_exists, image_exists, pfile_exists, expected_status", [ - (False, True, True, True, ServerStatus.NOT_INITIALIZED), # No server config - (True, False, True, True, ServerStatus.NOT_INITIALIZED), # Config dir does not exist - (True, True, False, True, ServerStatus.MISSING_CONTAINER), # Image path does not exist - (True, True, True, False, ServerStatus.NOT_RUNNING), # Pfile path does not exist -]) +@pytest.mark.parametrize( + "server_config_exists, config_exists, image_exists, pfile_exists, expected_status", + [ + (False, True, True, True, ServerStatus.NOT_INITIALIZED), # No server config + (True, False, True, True, ServerStatus.NOT_INITIALIZED), # Config dir does not exist + (True, True, False, True, ServerStatus.MISSING_CONTAINER), # Image path does not exist + (True, True, True, False, ServerStatus.NOT_RUNNING), # Pfile path does not exist + ], +) def test_get_server_status_initial_checks( mocker: "Fixture", # noqa: F821 server_server_config: Dict[str, Dict[str, str]], @@ -736,11 +731,12 @@ def test_get_server_status_initial_checks( mocker.patch("merlin.server.server_util.ContainerConfig.get_pfile_path", return_value="pfile_path") # Mock os.path.exists to return the desired values - mocker.patch("os.path.exists", side_effect=lambda path: { - "config_dir": config_exists, - "image_path": image_exists, - "pfile_path": pfile_exists - }.get(path, False)) + mocker.patch( + "os.path.exists", + side_effect=lambda path: {"config_dir": config_exists, "image_path": image_exists, "pfile_path": pfile_exists}.get( + path, False + ), + ) else: mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) @@ -748,10 +744,13 @@ def test_get_server_status_initial_checks( assert get_server_status() == expected_status -@pytest.mark.parametrize("stdout_val, expected_status", [ - (b"", ServerStatus.NOT_RUNNING), # No stdout from subprocess - (b"Successfully started", ServerStatus.RUNNING), # Stdout from subprocess exists -]) +@pytest.mark.parametrize( + "stdout_val, expected_status", + [ + (b"", ServerStatus.NOT_RUNNING), # No stdout from subprocess + (b"Successfully started", ServerStatus.RUNNING), # Stdout from subprocess exists + ], +) def test_get_server_status_subprocess_check( mocker: "Fixture", # noqa: F821 server_server_config: Dict[str, Dict[str, str]], @@ -774,13 +773,16 @@ def test_get_server_status_subprocess_check( assert get_server_status() == expected_status -@pytest.mark.parametrize("data_to_test, expected_result", [ - ({"image_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No parent_pid entry - ({"parent_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No image_pid entry - ({"parent_pid": 123, "image_pid": 456, "hostname": "dummy_server"}, False), # No port entry - ({"parent_pid": 123, "image_pid": 123, "port": 6379}, False), # No hostname entry - ({"parent_pid": 123, "image_pid": 123, "port": 6379, "hostname": "dummy_server"}, True), # All required entries exist -]) +@pytest.mark.parametrize( + "data_to_test, expected_result", + [ + ({"image_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No parent_pid entry + ({"parent_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No image_pid entry + ({"parent_pid": 123, "image_pid": 456, "hostname": "dummy_server"}, False), # No port entry + ({"parent_pid": 123, "image_pid": 123, "port": 6379}, False), # No hostname entry + ({"parent_pid": 123, "image_pid": 123, "port": 6379, "hostname": "dummy_server"}, True), # All required entries exist + ], +) def test_check_process_file_format(data_to_test: Dict[str, Union[int, str]], expected_result: bool): """ Test the `check_process_file_format` function. The first 4 parametrized tests above should all @@ -803,8 +805,8 @@ def test_pull_process_file_valid_file(server_testing_dir: str, server_process_fi """ # Create the valid process file in our temp testing directory process_filepath = f"{server_testing_dir}/valid_process_file.yaml" - with open(process_filepath, 'w') as process_file: - yaml.dump(server_process_file_contents, process_file) + with open(process_filepath, "w") as process_file: + yaml.dump(server_process_file_contents, process_file) # Run the test assert pull_process_file(process_filepath) == server_process_file_contents @@ -824,8 +826,8 @@ def test_pull_process_file_invalid_file(server_testing_dir: str, server_process_ # Create the invalid process file in our temp testing directory process_filepath = f"{server_testing_dir}/invalid_process_file.yaml" - with open(process_filepath, 'w') as process_file: - yaml.dump(server_process_file_contents, process_file) + with open(process_filepath, "w") as process_file: + yaml.dump(server_process_file_contents, process_file) # Run the test assert pull_process_file(process_filepath) is None From 7050822cee81b861a830e420b006e62d90a2382f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 13:15:32 -0700 Subject: [PATCH 126/201] update README for testing directory --- tests/README.md | 116 ++++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/tests/README.md b/tests/README.md index 22efc5470..9b2f7ba1f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,17 +1,21 @@ # Tests This directory utilizes pytest to create and run our test suite. -Here we use pytest fixtures to create a local redis server and a celery app for testing. This directory is organized like so: -- `conftest.py` - The script containing all fixtures for our tests -- `unit/` - The directory containing unit tests - - `test_*.py` - The actual test scripts to run +- `conftest.py` - The script containing common fixtures for our tests +- `context_managers/` - The directory containing context managers used for testing + - `celery_workers_manager.py` - A context manager used to manage celery workers for integration testing + - `server_manager.py` - A context manager used to manage the redis server used for integration testing +- `fixtures/` - The directory containing specific test module fixtures + - `.py` - Fixtures for specific test modules - `integration/` - The directory containing integration tests - `definitions.py` - The test definitions - `run_tests.py` - The script to run the tests defined in `definitions.py` - `conditions.py` - The conditions to test against +- `unit/` - The directory containing unit tests + - `test_*.py` - The actual test scripts to run ## How to Run @@ -44,6 +48,28 @@ To run one unique test: python -m pytest /path/to/test_specific_file.py::TestCertainClass::test_unique_test ``` +## Viewing Results + +Test results will be written to `/tmp/$(whoami)/pytest-of-$(whoami)/pytest-current/python_{major}.{minor}.{micro}_current/`. + +It's good practice to set up a subdirectory in this temporary output folder for each module that you're testing. You can see an example of how this is set up in the files within the module-specific fixture directory. For instance, you can see this in the `examples_testing_dir` fixture from the `tests/fixtures/examples.py` file: + +``` +@pytest.fixture(scope="session") +def examples_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to the examples functionality. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for examples tests + """ + testing_dir = f"{temp_output_dir}/examples_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir +``` + ## Killing the Test Server In case of an issue with the test suite, or if you stop the tests with `ctrl+C`, you may need to stop @@ -58,58 +84,45 @@ not connected> quit ## The Fixture Process Explained -In the world of pytest testing, fixtures are like the building blocks that create a sturdy foundation for your tests. -They ensure that every test starts from the same fresh ground, leading to reliable and consistent results. This section -will dive into the nitty-gritty of these fixtures, showing you how they're architected in this test suite, how to use -them in your tests here, how to combine them for more complex scenarios, how long they stick around during testing, and -what it means to yield a fixture. +In the world of pytest testing, fixtures are like the building blocks that create a sturdy foundation for your tests. They ensure that every test starts from the same fresh ground, leading to reliable and consistent results. This section will dive into the nitty-gritty of these fixtures, showing you how they're architected in this test suite, how to use them in your tests here, how to combine them for more complex scenarios, how long they stick around during testing, and what it means to yield a fixture. ### Fixture Architecture -Fixtures can be defined in two locations: +Fixtures can be defined in two locations within this test suite: -1. `tests/conftest.py`: This file located at the root of the test suite houses common fixtures that are utilized -across various test modules -2. `tests/fixtures/`: This directory contains specific test module fixtures. Each fixture file is named according -to the module(s) that the fixtures defined within are for. +1. `tests/conftest.py`: This file located at the root of the test suite houses common fixtures that are utilized across various test modules +2. `tests/fixtures/`: This directory contains specific test module fixtures. Each fixture file is named according to the module(s) that the fixtures defined within are for. Credit for this setup must be given to [this Medium article](https://medium.com/@nicolaikozel/modularizing-pytest-fixtures-fd40315c5a93). #### Fixture Naming Conventions -For fixtures defined within the `tests/fixtures/` directory, the fixture name should be prefixed by the name of the -fixture file they are defined in. +For fixtures defined within the `tests/fixtures/` directory, the fixture name should be prefixed by the name of the fixture file they are defined in. #### Importing Fixtures as Plugins -Fixtures located in the `tests/fixtures/` directory are technically plugins. Therefore, to use them we must -register them as plugins within the `conftest.py` file (see the top of said file for the implementation). -This allows them to be discovered and used by test modules throughout the suite. +Fixtures located in the `tests/fixtures/` directory are technically plugins. Therefore, to use them we must register them as plugins within the `conftest.py` file (see the top of said file for the implementation). This allows them to be discovered and used by test modules throughout the suite. -**You do not have to register the fixtures you define as plugins in `conftest.py` since the registration there -uses `glob` to grab everything from the `tests/fixtures/` directory automatically.** +**You do not have to register the fixtures you define as plugins in `conftest.py` since the registration there uses `glob` to grab everything from the `tests/fixtures/` directory automatically.** ### How to Integrate Fixtures Into Tests -Probably the most important part of fixtures is understanding how to use them. Luckily, this process is very -simple and can be dumbed down to just a couple steps: +Probably the most important part of fixtures is understanding how to use them. Luckily, this process is very simple and can be dumbed down to just a couple steps: -0. **[Module-specific fixtures only]** If you're creating a module-specific fixture (i.e. a fixture that won't be used throughout the entire test -suite), then create a file in the `tests/fixtures/` directory. +0. **[Module-specific fixtures only]** If you're creating a module-specific fixture (i.e. a fixture that won't be used throughout the entire test suite), then create a file in the `tests/fixtures/` directory. -1. Create a fixture in either the `conftest.py` file or the file you created in the `tests/fixtures/` directory -by using the `@pytest.fixture` decorator. For example: +1. Create a fixture in either the `conftest.py` file or the file you created in the `tests/fixtures/` directory by using the `@pytest.fixture` decorator. For example: ``` @pytest.fixture -def dummy_fixture(): +def dummy_fixture() -> str: return "hello world" ``` 2. Use it as an argument in a test function (you don't even need to import it!): ``` -def test_dummy(dummy_fixture): +def test_dummy(dummy_fixture: str): assert dummy_fixture == "hello world" ``` @@ -117,22 +130,18 @@ For more information, see [Pytest's documentation](https://docs.pytest.org/en/7. ### Fixtureception -One of the coolest and most useful aspects of fixtures that we utilize in this test suite is the ability for -fixtures to be used within other fixtures. For more info on this from pytest, see -[here](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#fixtures-can-request-other-fixtures). +One of the coolest and most useful aspects of fixtures that we utilize in this test suite is the ability for fixtures to be used within other fixtures. For more info on this from pytest, see [here](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#fixtures-can-request-other-fixtures). + +Pytest will handle fixtures within fixtures in a stack-based way. Let's look at how creating the `redis_pass` fixture from our `conftest.py` file works in order to illustrate the process. -Pytest will handle fixtures within fixtures in a stack-based way. Let's look at how creating the `redis_pass` -fixture from our `conftest.py` file works in order to illustrate the process. -1. First, we start by telling pytest that we want to use the `redis_pass` fixture by providing it as an argument -to a test/fixture: +1. First, we start by telling pytest that we want to use the `redis_pass` fixture by providing it as an argument to a test/fixture: ``` def test_example(redis_pass): ... ``` -2. Now pytest will find the `redis_pass` fixture and put it at the top of the stack to be created. However, -it'll see that this fixture requires another fixture `merlin_server_dir` as an argument: +2. Now pytest will find the `redis_pass` fixture and put it at the top of the stack to be created. However, it'll see that this fixture requires another fixture `merlin_server_dir` as an argument: ``` @pytest.fixture(scope="session") @@ -140,8 +149,7 @@ def redis_pass(merlin_server_dir): ... ``` -3. Pytest then puts the `merlin_server_dir` fixture at the top of the stack, but similarly it sees that this fixture -requires yet another fixture `temp_output_dir`: +3. Pytest then puts the `merlin_server_dir` fixture at the top of the stack, but similarly it sees that this fixture requires yet another fixture `temp_output_dir`: ``` @pytest.fixture(scope="session") @@ -149,34 +157,24 @@ def merlin_server_dir(temp_output_dir: str) -> str: ... ``` -4. This process continues until it reaches a fixture that doesn't require any more fixtures. At this point the base -fixture is created and pytest will start working its way back up the stack to the first fixture it looked at (in this -case `redis_pass`). +4. This process continues until it reaches a fixture that doesn't require any more fixtures. At this point the base fixture is created and pytest will start working its way back up the stack to the first fixture it looked at (in this case `redis_pass`). -5. Once all required fixtures are created, execution will be returned to the test which can now access the fixture -that was requested (`redis_pass`). +5. Once all required fixtures are created, execution will be returned to the test which can now access the fixture that was requested (`redis_pass`). -As you can see, if we have to re-do this process for every test it could get pretty time intensive. This is where fixture -scopes come to save the day. +As you can see, if we have to re-do this process for every test it could get pretty time intensive. This is where fixture scopes come to save the day. ### Fixture Scopes -There are several different scopes that you can set for fixtures. The majority of our fixtures in `conftest.py` -use a `session` scope so that we only have to create the fixtures one time (as some of them can take a few seconds -to set up). The goal is to create fixtures with the most general use-case in mind so that we can re-use them for -larger scopes, which helps with efficiency. +There are several different scopes that you can set for fixtures. The majority of our fixtures in `conftest.py` use a `session` scope so that we only have to create the fixtures one time (as some of them can take a few seconds to set up). The goal for fixtures defined in `conftest.py` is to create fixtures with the most general use-case in mind so that we can re-use them for larger scopes, which helps with efficiency. + +For fixtures that need to be reset on each run, we generally try to place these in the module-specific fixture directory `tests/fixtures/`. -For more info on scopes, see -[Pytest's Fixture Scope documentation](https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session). +For more info on scopes, see [Pytest's Fixture Scope documentation](https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session). ### Yielding Fixtures -In several fixtures throughout our test suite, we need to run some sort of teardown for the fixture. For example, -once we no longer need the `redis_server` fixture, we need to shut the server down so it stops using resources. -This is where yielding fixtures becomes extremely useful. +In several fixtures throughout our test suite, we need to run some sort of teardown for the fixture. For example, once we no longer need the `redis_server` fixture, we need to shut the server down so it stops using resources. This is where yielding fixtures becomes extremely useful. -Using the `yield` keyword allows execution to be returned to a test that needs the fixture once the feature has -been set up. After all tests using the fixture have been ran, execution will return to the fixture for us to run -our teardown code. +Using the `yield` keyword allows execution to be returned to a test that needs the fixture once the feature has been set up. After all tests using the fixture have been ran, execution will return to the fixture for us to run our teardown code. For more information on yielding fixtures, see [Pytest's documentation](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#teardown-cleanup-aka-fixture-finalization). \ No newline at end of file From 0c7402102f015ffe6d7032ba7a96b79fd8eef92e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 13:16:21 -0700 Subject: [PATCH 127/201] update the temp_output_directory to include python version --- tests/conftest.py | 5 ++++- tests/context_managers/celery_workers_manager.py | 2 +- tests/context_managers/server_manager.py | 2 +- tests/unit/common/test_sample_index.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2a4b5f169..11bd93134 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,7 @@ This module contains pytest fixtures to be used throughout the entire test suite. """ import os +import sys from copy import copy from glob import glob from time import sleep @@ -110,7 +111,9 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: """ # Log the cwd, then create and move into the temporary one cwd = os.getcwd() - temp_integration_outfile_dir = tmp_path_factory.mktemp("integration_outfiles_") + temp_integration_outfile_dir = tmp_path_factory.mktemp( + f"python_{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}_" + ) os.chdir(temp_integration_outfile_dir) yield temp_integration_outfile_dir diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 0acdee130..118aa21a1 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -135,7 +135,7 @@ def start_worker(self, worker_launch_cmd: List[str]): app.worker_main instead of the normal "celery -A worker" command to launch the workers since our celery app is created in a pytest fixture and is unrecognizable by the celery command. For each worker, the output of it's logs are sent to - /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ under a file with a name + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/python_{major}.{minor}.{micro}_current/ under a file with a name similar to: test_worker_*.log. NOTE: pytest-current/ will have the results of the most recent test run. If you want to see a previous run check under pytest-/. HOWEVER, only the 3 most recent test runs will be saved. diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index c88948772..b99afb2c6 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -56,7 +56,7 @@ def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: T def initialize_server(self): """ Initialize the setup for the local redis server. We'll write the folder to: - /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/python_{major}.{minor}.{micro}_current/ We'll set the password to be 'merlin-test-server' so it'll be easy to shutdown if necessary """ subprocess.run( diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index d857b7ce5..c9cd108ee 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -80,7 +80,7 @@ class TestSampleIndex: NOTE to see output of creating any hierarchy, change `write_all_hierarchies` to True. The results of each hierarchy will be written to: - /tmp/`whoami`/pytest/pytest-of-`whoami`/pytest-current/integration_outfiles_current/test_sample_index/ + /tmp/`whoami`/pytest/pytest-of-`whoami`/pytest-current/python_{major}.{minor}.{micro}_current/test_sample_index/ """ write_all_hierarchies = False From 42c121b307bde69e6b42c907f3faae7fa3ca5e68 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 13:47:02 -0700 Subject: [PATCH 128/201] mock the open.write to try to fix github CI --- tests/unit/server/test_server_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 2528648dc..aa992e521 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -596,6 +596,7 @@ def test_pull_server_image_no_image_path_no_config_path( # Mock the open function read_data = "Mocked file content" mocked_open = mocker.mock_open(read_data=read_data) + mocked_open.write = mocker.Mock() mocker.patch("builtins.open", mocked_open) # Call the function From f5e8671c758b8fb7937695d7b2ea644666bacdd8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 14:10:24 -0700 Subject: [PATCH 129/201] ensure config dir is created --- tests/unit/server/test_server_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index aa992e521..8224214ea 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -555,6 +555,9 @@ def setup_pull_server_image_mock( """ image_url = "docker://redis" image_path = f"{server_testing_dir}/{image_file}" + + os.makedirs(config_dir, exist_ok=True) + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value=config_dir) mocker.patch("merlin.server.server_util.ContainerConfig.get_config_name", return_value=config_file) @@ -562,8 +565,6 @@ def setup_pull_server_image_mock( mocker.patch("merlin.server.server_util.ContainerConfig.get_image_path", return_value=image_path) if create_config_file: - if not os.path.exists(config_dir): - os.mkdir(config_dir) with open(os.path.join(config_dir, config_file), "w"): pass From 3ce140e4298c3cf0b0891038dd0aeeb71ba130f8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 14:11:08 -0700 Subject: [PATCH 130/201] update CHANGELOG --- CHANGELOG.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1011f7fa..e5774a1ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,23 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.12.2b1] +## [Unreleased] ### Added -- Coverage to the test suite. This includes adding tests for: +- Several new unit tests for the following subdirectories: - `merlin/common/` - `merlin/config/` - `merlin/examples/` - - `celeryadapter.py` + - `merlin/server/` - Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures - `RedisServerManager`: context to help with starting/stopping a redis server for tests - `CeleryWorkersManager`: context to help with starting/stopping workers for tests - Ability to copy and print the `Config` object from `merlin/config/__init__.py` + +### Changed +- Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier + +## [1.12.2b1] +### Added - Conflict handler option to the `dict_deep_merge` function in `utils.py` - Ability to add module-specific pytest fixtures - Added fixtures specifically for testing status functionality From 73e4cf5564fe3d3092e02212bf139bef95ca2d8b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 14:42:38 -0700 Subject: [PATCH 131/201] add print of exception to OSError catch in pull_server_image --- merlin/server/server_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 93e5b0baa..7c886abdf 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -328,8 +328,8 @@ def pull_server_image() -> bool: with resources.path("merlin.server", config_file) as file: with open(os.path.join(config_dir, config_file), "w") as outfile, open(file, "r") as infile: outfile.write(infile.read()) - except OSError: - LOG.error(f"Destination location {config_dir} is not writable.") + except OSError as exc: + LOG.error(f"Destination location {config_dir} is not writable. Raised from:\n{exc}") return False else: LOG.info("Redis configuration file already exist.") From 5b36b41497287ee39f91d9d5ec43e7792a898ace Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 15:20:23 -0700 Subject: [PATCH 132/201] change name of config_file in test that's failing --- merlin/server/server_config.py | 2 +- tests/unit/server/test_server_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 7c886abdf..a14263cac 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -297,7 +297,7 @@ def pull_server_image() -> bool: """ Fetch the server image using singularity. - :return:: True if success and False if fail + :return: True if success and False if fail """ server_config = pull_server_config() if not server_config: diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py index 8224214ea..0e6fdc040 100644 --- a/tests/unit/server/test_server_config.py +++ b/tests/unit/server/test_server_config.py @@ -589,7 +589,7 @@ def test_pull_server_image_no_image_path_no_config_path( """ # Set up mock calls to simulate the setup of this function config_dir = f"{server_testing_dir}/config_dir" - config_file = "pull_server_image_no_image_path_no_config_path_config_nonexistent.yaml" + config_file = "redis.conf" image_file = "pull_server_image_no_image_path_no_config_path_image_nonexistent.sif" setup_pull_server_image_mock(mocker, server_testing_dir, server_server_config, config_dir, config_file, image_file) mocked_subprocess = mocker.patch("subprocess.run") From 352e7dfdc667918824f8cb423d765b3b8c58c3ca Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 13 Sep 2024 15:41:27 -0700 Subject: [PATCH 133/201] Added password check and omit if a password doesn't exist --- merlin/managers/redis_connection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index 5b3f4c723..c000cabbc 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -70,7 +70,9 @@ def get_redis_connection(self) -> redis.Redis: try: password = get_backend_password(password_file) except IOError: - password = CONFIG.results_backend.password + password = None + if hasattr(CONFIG.results_backend, "password"): + password = CONFIG.results_backend.password has_ssl = hasattr(CONFIG.results_backend, "cert_reqs") ssl_cert_reqs = "required" From 9782d58f61226d750fd62057bf31682e9a9d7281 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 13 Sep 2024 15:54:04 -0700 Subject: [PATCH 134/201] update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5774a1ed..efa43f947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `RedisServerManager`: context to help with starting/stopping a redis server for tests - `CeleryWorkersManager`: context to help with starting/stopping workers for tests - Ability to copy and print the `Config` object from `merlin/config/__init__.py` +- Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py` ### Changed - Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier +- Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier +- Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot` ## [1.12.2b1] ### Added From 75a9972faa9e2e37de422be3b38087c6ce9a8434 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 16 Sep 2024 11:41:37 -0700 Subject: [PATCH 135/201] change testing log level to debug --- tests/integration/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 59c1fa256..ab02da168 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -101,7 +101,7 @@ def define_tests(): # pylint: disable=R0914,R0915 celery_pbs_regex = rf"(qsub\s+.*)?{celery_regex}" # Shortcuts for Merlin commands - err_lvl = "-lvl error" + err_lvl = "-lvl debug" workers = f"merlin {err_lvl} run-workers" workers_flux = get_worker_by_cmd("flux", workers) workers_pbs = get_worker_by_cmd("qsub", workers) From c27a208ad9fc0fd478f0dbf9ed3d6a7d3b5684b1 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 17 Sep 2024 12:52:49 -0700 Subject: [PATCH 136/201] add debug statement for redis_connection --- merlin/managers/redis_connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index c000cabbc..e72555b08 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -66,6 +66,8 @@ def get_redis_connection(self) -> redis.Redis: from merlin.config.configfile import CONFIG from merlin.config.results_backend import get_backend_password + LOG.debug(f"MANAGER: CONFIG.results_backend: {CONFIG.results_backend}") + password_file = CONFIG.results_backend.password try: password = get_backend_password(password_file) From 97a9cf1fe5fa1be538ed672b1b66124393977920 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 17 Sep 2024 14:26:14 -0700 Subject: [PATCH 137/201] change debug log to info so github ci will display it --- merlin/managers/redis_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index e72555b08..24fa7021f 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -66,7 +66,7 @@ def get_redis_connection(self) -> redis.Redis: from merlin.config.configfile import CONFIG from merlin.config.results_backend import get_backend_password - LOG.debug(f"MANAGER: CONFIG.results_backend: {CONFIG.results_backend}") + LOG.info(f"MANAGER: CONFIG.results_backend: {CONFIG.results_backend}") password_file = CONFIG.results_backend.password try: From ce8bf3786cdbe01843b24b801343c0a0717cd9b0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 17 Sep 2024 15:55:56 -0700 Subject: [PATCH 138/201] attempt to fix password missing from Namespace error --- merlin/managers/redis_connection.py | 20 ++++++++++---------- tests/integration/definitions.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index 24fa7021f..def45d440 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -63,18 +63,18 @@ def get_redis_connection(self) -> redis.Redis: :return: Redis connection object that can be used to access values for the manager. """ - from merlin.config.configfile import CONFIG - from merlin.config.results_backend import get_backend_password + from merlin.config.configfile import CONFIG # pylint: disable=import-outside-toplevel + from merlin.config.results_backend import get_backend_password # pylint: disable=import-outside-toplevel - LOG.info(f"MANAGER: CONFIG.results_backend: {CONFIG.results_backend}") + password_file = CONFIG.results_backend.password if hasattr(CONFIG.results_backend, "password") else None - password_file = CONFIG.results_backend.password - try: - password = get_backend_password(password_file) - except IOError: - password = None - if hasattr(CONFIG.results_backend, "password"): - password = CONFIG.results_backend.password + password = None + if password_file is not None: + try: + password = get_backend_password(password_file) + except IOError: + if hasattr(CONFIG.results_backend, "password"): + password = CONFIG.results_backend.password has_ssl = hasattr(CONFIG.results_backend, "cert_reqs") ssl_cert_reqs = "required" diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index ab02da168..59c1fa256 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -101,7 +101,7 @@ def define_tests(): # pylint: disable=R0914,R0915 celery_pbs_regex = rf"(qsub\s+.*)?{celery_regex}" # Shortcuts for Merlin commands - err_lvl = "-lvl debug" + err_lvl = "-lvl error" workers = f"merlin {err_lvl} run-workers" workers_flux = get_worker_by_cmd("flux", workers) workers_pbs = get_worker_by_cmd("qsub", workers) From 5851d9dfc4b1e5240c087a9a905a0d526a4cffb7 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 17 Sep 2024 16:32:56 -0700 Subject: [PATCH 139/201] run checks for all necessary configurations --- merlin/managers/redis_connection.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index def45d440..31a672c75 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -67,6 +67,12 @@ def get_redis_connection(self) -> redis.Redis: from merlin.config.results_backend import get_backend_password # pylint: disable=import-outside-toplevel password_file = CONFIG.results_backend.password if hasattr(CONFIG.results_backend, "password") else None + server = CONFIG.results_backend.server if hasattr(CONFIG.results_backend, "server") else None + port = CONFIG.results_backend.port if hasattr(CONFIG.results_backend, "port") else None + results_db_num = CONFIG.results_backend.db_num if hasattr(CONFIG.results_backend, "db_num") else None + username = CONFIG.results_backend.username if hasattr(CONFIG.results_backend, "username") else None + has_ssl = hasattr(CONFIG.results_backend, "cert_reqs") + ssl_cert_reqs = CONFIG.results_backend.cert_reqs if has_ssl else "required" password = None if password_file is not None: @@ -76,16 +82,11 @@ def get_redis_connection(self) -> redis.Redis: if hasattr(CONFIG.results_backend, "password"): password = CONFIG.results_backend.password - has_ssl = hasattr(CONFIG.results_backend, "cert_reqs") - ssl_cert_reqs = "required" - if has_ssl: - ssl_cert_reqs = CONFIG.results_backend.cert_reqs - return redis.Redis( - host=CONFIG.results_backend.server, - port=CONFIG.results_backend.port, - db=CONFIG.results_backend.db_num + self.db_num, # Increment db_num to avoid conflicts - username=CONFIG.results_backend.username, + host=server, + port=port, + db=results_db_num + self.db_num, # Increment db_num to avoid conflicts + username=username, password=password, decode_responses=True, ssl=has_ssl, From 97d075efbb6ffb6ee5c963fc3a4cd303600bf534 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 19 Sep 2024 17:12:22 -0700 Subject: [PATCH 140/201] convert stop-workers tests to pytest format --- merlin/celery.py | 4 +- .../dev_workflows/multiple_workers.yaml | 6 +- tests/conftest.py | 34 +- .../celery_workers_manager.py | 4 +- tests/context_managers/server_manager.py | 2 +- tests/integration/commands/__init__.py | 0 .../integration/commands/test_stop_workers.py | 442 ++++++++++++++++++ .../test_specs/multiple_workers.yaml | 56 +++ 8 files changed, 535 insertions(+), 13 deletions(-) create mode 100644 tests/integration/commands/__init__.py create mode 100644 tests/integration/commands/test_stop_workers.py create mode 100644 tests/integration/test_specs/multiple_workers.yaml diff --git a/merlin/celery.py b/merlin/celery.py index eb10f1a12..37b7f07e5 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -114,9 +114,11 @@ def route_for_task(name, args, kwargs, options, task=None, **kw): # pylint: dis BROKER_URI = None RESULTS_BACKEND_URI = None +app_name = "merlin_test_app" if os.getenv("CELERY_ENV") == "test" else "merlin" + # initialize app with essential properties app: Celery = patch_celery().Celery( - "merlin", + app_name, broker=BROKER_URI, backend=RESULTS_BACKEND_URI, broker_use_ssl=BROKER_SSL, diff --git a/merlin/examples/dev_workflows/multiple_workers.yaml b/merlin/examples/dev_workflows/multiple_workers.yaml index 8785d9e9a..967582a53 100644 --- a/merlin/examples/dev_workflows/multiple_workers.yaml +++ b/merlin/examples/dev_workflows/multiple_workers.yaml @@ -46,11 +46,11 @@ merlin: resources: workers: step_1_merlin_test_worker: - args: -l INFO + args: -l INFO --concurrency 1 steps: [step_1] step_2_merlin_test_worker: - args: -l INFO + args: -l INFO --concurrency 1 steps: [step_2] other_merlin_test_worker: - args: -l INFO + args: -l INFO --concurrency 1 steps: [step_3, step_4] diff --git a/tests/conftest.py b/tests/conftest.py index 11bd93134..1480a272d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,6 +99,22 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi ####################################### +@pytest.fixture(scope="session") +def path_to_test_specs() -> str: + """ + Fixture to provide the path to the directory containing test specifications. + + This fixture returns the absolute path to the 'test_specs' directory + within the 'integration' folder of the test directory. It expands + environment variables and user home directory as necessary. + + Returns: + The absolute path to the 'test_specs' directory. + """ + path_to_test_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))) + return os.path.join(path_to_test_dir, os.path.join("integration", "test_specs")) + + @pytest.fixture(scope="session") def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: """ @@ -130,7 +146,7 @@ def merlin_server_dir(temp_output_dir: str) -> str: :param temp_output_dir: The path to the temporary output directory we'll be using for this test run :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture """ - server_dir = f"{temp_output_dir}/merlin_server" + server_dir = os.path.join(temp_output_dir, "merlin_server") if not os.path.exists(server_dir): os.mkdir(server_dir) return server_dir @@ -146,11 +162,14 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ + os.environ["CELERY_ENV"] = "test" with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: redis_server_manager.initialize_server() redis_server_manager.start_server() create_encryption_file( - f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml" + os.path.join(merlin_server_dir, "encrypt_data_key"), + test_encryption_key, + app_yaml_filepath=os.path.join(merlin_server_dir, "app.yaml"), ) # Yield the redis_server uri to any fixtures/tests that may need it yield redis_server_manager.redis_server_uri @@ -165,6 +184,7 @@ def celery_app(redis_server: str) -> Celery: :param redis_server: The redis server uri we'll use to connect to redis :returns: The celery app object we'll use for testing """ + os.environ["CELERY_ENV"] = "test" return Celery("merlin_test_app", broker=redis_server, backend=redis_server) @@ -258,7 +278,7 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): orig_config = copy(CONFIG) # Create an encryption key file (if it doesn't already exist) - key_file = f"{merlin_server_dir}/encrypt_data_key" + key_file = os.path.join(merlin_server_dir, "encrypt_data_key") create_encryption_file(key_file, test_encryption_key) # Set the broker configuration for testing @@ -305,7 +325,7 @@ def redis_broker_config( :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - pass_file = f"{merlin_server_dir}/redis.pass" + pass_file = os.path.join(merlin_server_dir, "redis.pass") create_pass_file(pass_file) CONFIG.broker.password = pass_file @@ -326,7 +346,7 @@ def redis_results_backend_config( :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - pass_file = f"{merlin_server_dir}/redis.pass" + pass_file = os.path.join(merlin_server_dir, "redis.pass") create_pass_file(pass_file) CONFIG.results_backend.password = pass_file @@ -347,7 +367,7 @@ def rabbit_broker_config( :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - pass_file = f"{merlin_server_dir}/rabbit.pass" + pass_file = os.path.join(merlin_server_dir, "rabbit.pass") create_pass_file(pass_file) CONFIG.broker.password = pass_file @@ -368,7 +388,7 @@ def mysql_results_backend_config( :param merlin_server_dir: The directory to the merlin test server configuration :param config: The fixture that sets up most of the CONFIG object for testing """ - pass_file = f"{merlin_server_dir}/mysql.pass" + pass_file = os.path.join(merlin_server_dir, "mysql.pass") create_pass_file(pass_file) create_cert_files(merlin_server_dir, CERT_FILES) diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 118aa21a1..f58c38872 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -158,6 +158,8 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = self.stop_all_workers() raise ValueError(f"The worker {worker_name} is already running. Choose a different name.") + queues = [f"[merlin]_{queue}" for queue in queues] + # Create the launch command for this worker worker_launch_cmd = [ "worker", @@ -174,7 +176,7 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = # Create an echo command to simulate a running celery worker since our celery worker will be spun up in # a different process and we won't be able to see it with 'ps ux' like we normally would echo_process = subprocess.Popen( # pylint: disable=consider-using-with - f"echo 'celery merlin_test_app {' '.join(worker_launch_cmd)}'; sleep inf", + f"echo 'celery -A merlin_test_app {' '.join(worker_launch_cmd)}'; sleep inf", shell=True, preexec_fn=os.setpgrp, # Make this the parent of the group so we can kill the 'sleep inf' that's spun up ) diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py index b99afb2c6..bb5d86036 100644 --- a/tests/context_managers/server_manager.py +++ b/tests/context_managers/server_manager.py @@ -91,7 +91,7 @@ def stop_server(self): if "Merlin server terminated." not in kill_process.stderr: # If it wasn't, try to kill the process by using the pid stored in a file created by `merlin server` try: - with open(f"{self.server_dir}/merlin_server.pf", "r") as process_file: + with open(os.path.join(self.server_dir, "merlin_server.pf"), "r") as process_file: server_process_info = yaml.load(process_file, yaml.Loader) os.kill(int(server_process_info["image_pid"]), signal.SIGKILL) # If the file can't be found then let's make sure there's even a redis-server process running diff --git a/tests/integration/commands/__init__.py b/tests/integration/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py new file mode 100644 index 000000000..ceb1639e8 --- /dev/null +++ b/tests/integration/commands/test_stop_workers.py @@ -0,0 +1,442 @@ +""" +Tests for the `merlin stop-workers` command. +""" + +import os +import re +import shutil +import subprocess +import yaml +from enum import Enum +from typing import List + +from celery import Celery + +from tests.integration.conditions import HasRegex +from tests.context_managers.celery_workers_manager import CeleryWorkersManager + + +class WorkerMessages(Enum): + """ + Enumerated strings to help keep track of the messages + that we're expecting (or not expecting) to see from the + tests in this module. + """ + NO_WORKERS_MSG = "No workers found to stop" + STEP_1_WORKER = "step_1_merlin_test_worker" + STEP_2_WORKER = "step_2_merlin_test_worker" + OTHER_WORKER = "other_merlin_test_worker" + + +class TestStopWorkers: + """ + Tests for the `merlin stop-workers` command. Most of these tests will: + 1. Start workers from a spec file used for testing + - Use CeleryWorkerManager for this to ensure safe stoppage of workers + if something goes wrong + 2. Run the `merlin stop-workers` command from a subprocess + """ + + def load_workers_from_spec(self, spec_filepath: str) -> dict: + """ + Load worker specifications from a YAML file. + + This function reads a YAML file containing study specifications and + extracts the worker information under the "merlin" section. It + constructs a dictionary in the form that CeleryWorkersManager.launch_workers + requires. + + Parameters: + spec_filepath: The file path to the YAML specification file. + + Returns: + A dictionary containing the worker specifications from the + "merlin" section of the YAML file. + """ + # Read in the contents of the spec file + with open(spec_filepath, "r") as spec_file: + spec_contents = yaml.load(spec_file, yaml.Loader) + + # Initialize an empty dictionary to hold worker_info + worker_info = {} + + # Access workers and steps from spec_contents + workers = spec_contents["merlin"]["resources"]["workers"] + study_steps = {step['name']: step['run']['task_queue'] for step in spec_contents['study']} + + # Grab the concurrency and queues from each worker and add it to the worker_info dict + for worker_name, worker_settings in workers.items(): + match = re.search(r'--concurrency\s+(\d+)', worker_settings["args"]) + concurrency = int(match.group(1)) if match else 1 + queues = [study_steps[step] for step in worker_settings["steps"]] + worker_info[worker_name] = {"concurrency": concurrency, "queues": queues} + + return worker_info + + def copy_app_yaml_to_cwd(self, merlin_server_dir: str): + """ + Copy the app.yaml file from the directory provided to the current working + directory. + + Grab the app.yaml file from `merlin_server_dir` and copy it to the current + working directory so that Merlin will read this in as the server configuration + for whatever test is calling this. + + Parameters: + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + copied_app_yaml = os.path.join(os.getcwd(), "app.yaml") + if not os.path.exists(copied_app_yaml): + server_app_yaml = os.path.join(merlin_server_dir, "app.yaml") + shutil.copy(server_app_yaml, copied_app_yaml) + + def run_test_with_workers( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + conditions: List, + flag: str = None, + ): + """ + Helper function to run common testing logic for tests with workers started. + + This function will: + 0. Read in the necessary fixtures as parameters. The purpose of these fixtures + include: + - Starting a containerized redis server + - Updating the CONFIG object to point to the containerized redis server + - Grabbing paths to our test specs and the merlin server directory created + from starting the containerized redis server + 1. Load in the worker specifications from the `multiple_workers.yaml` file. + 2. Use a context manager to start up the workers on the celery app connected to + the containerized redis server + 3. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 4. Run the test command that's provided and check that the conditions given are + passing. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + conditions: + A list of `Condition` instances that need to pass in order for this test to + be successful. + flag: + An optional flag to add to the `merlin stop-workers` command so we can test + different functionality for the command. + """ + from merlin.celery import app as celery_app + + # Grab worker configurations from the spec file + multiple_worker_spec = os.path.join(path_to_test_specs, "multiple_workers.yaml") + workers_from_spec = self.load_workers_from_spec(multiple_worker_spec) + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as workers_manager: + workers_manager.launch_workers(workers_from_spec) + + # Copy the app.yaml to the cwd so merlin will connect to the testing server + self.copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + stop_workers_cmd = "merlin stop-workers" + cmd_to_test = f"{stop_workers_cmd} {flag}" if flag is not None else stop_workers_cmd + result = subprocess.run(cmd_to_test, capture_output=True, text=True, shell=True) + + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + for condition in conditions: + condition.ingest_info(info) + assert condition.passes + + def test_no_workers( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + merlin_server_dir: str, + ): + """ + Test the `merlin stop-workers` command with no workers started in the first place. + + This test will: + 0. Set up the appropriate fixtures. This includes: + - Starting a containerized redis server + - Updating the CONFIG object to point to the containerized redis server + - Grabbing the path to the merlin server directory created from starting the + containerized redis server + 1. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 2. Run the `merlin stop-workers` command and ensure that no workers are stopped (this + should be the case since no workers were started in the first place) + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + # Copy the app.yaml to the cwd so merlin will connect to the testing server + self.copy_app_yaml_to_cwd(merlin_server_dir) + + # Define our test conditions + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value), # No workers should be launched so we should see this + HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), # None of these workers should be started + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # None of these workers should be started + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # None of these workers should be started + ] + + # Run the test + result = subprocess.run("merlin stop-workers", capture_output=True, text=True, shell=True) + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + for condition in conditions: + condition.ingest_info(info) + assert condition.passes + + def test_no_flags( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin stop-workers` command with no flags. + + Run the `merlin stop-workers` command and ensure that all workers are stopped. + To see more information on exactly what this test is doing, see the + `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped + ] + self.run_test_with_workers( + redis_server, + redis_results_backend_config, + redis_broker_config, + path_to_test_specs, + merlin_server_dir, + conditions, + ) + + def test_spec_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin stop-workers` command with the `--spec` flag. + + Run the `merlin stop-workers` command with the `--spec` flag and ensure that all + workers are stopped. To see more information on exactly what this test is doing, + see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped + ] + self.run_test_with_workers( + redis_server, + redis_results_backend_config, + redis_broker_config, + path_to_test_specs, + merlin_server_dir, + conditions, + flag=f"--spec {os.path.join(path_to_test_specs, "multiple_workers.yaml")}" + ) + + def test_workers_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin stop-workers` command with the `--workers` flag. + + Run the `merlin stop-workers` command with the `--workers` flag and ensure that + only the workers given with this flag are stopped. To see more information on + exactly what this test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped + ] + self.run_test_with_workers( + redis_server, + redis_results_backend_config, + redis_broker_config, + path_to_test_specs, + merlin_server_dir, + conditions, + flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}" + ) + + def test_queues_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin stop-workers` command with the `--queues` flag. + + Run the `merlin stop-workers` command with the `--queues` flag and ensure that + only the workers attached to the given queues are stopped. To see more information + on exactly what this test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # One workers should be stopped + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # This worker should NOT be stopped + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped + ] + self.run_test_with_workers( + redis_server, + redis_results_backend_config, + redis_broker_config, + path_to_test_specs, + merlin_server_dir, + conditions, + flag=f"--queues hello_queue" + ) diff --git a/tests/integration/test_specs/multiple_workers.yaml b/tests/integration/test_specs/multiple_workers.yaml new file mode 100644 index 000000000..967582a53 --- /dev/null +++ b/tests/integration/test_specs/multiple_workers.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: other_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO --concurrency 1 + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO --concurrency 1 + steps: [step_2] + other_merlin_test_worker: + args: -l INFO --concurrency 1 + steps: [step_3, step_4] From 04e9122bf096bb9725a7289040de12b24d288e17 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 19 Sep 2024 17:25:10 -0700 Subject: [PATCH 141/201] update github wf and comment out stop-workers tests in definitions.py --- .github/workflows/push-pr_workflow.yml | 78 ++++++++++++- tests/integration/definitions.py | 152 ++++++++++++------------- 2 files changed, 151 insertions(+), 79 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 1d3c9d958..b90b7fa6f 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -82,7 +82,7 @@ jobs: python3 -m pylint merlin --rcfile=setup.cfg --exit-zero python3 -m pylint tests --rcfile=setup.cfg --exit-zero - Local-test-suite: + common-setup: runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 @@ -112,8 +112,7 @@ jobs: python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - pip freeze - + - name: Install singularity run: | sudo apt-get update && sudo apt-get install -y \ @@ -135,6 +134,61 @@ jobs: make -C ./builddir && \ sudo make -C ./builddir install + # TODO make this setup inheritable and then create unit/local/integration tests off of it? + Local-test-suite: + needs: common-setup + # runs-on: ubuntu-latest + # env: + # GO_VERSION: 1.18.1 + # SINGULARITY_VERSION: 3.9.9 + # OS: linux + # ARCH: amd64 + + # strategy: + # matrix: + # python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + + # steps: + # - uses: actions/checkout@v2 + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python-version }} + + # - name: Check cache + # uses: actions/cache@v2 + # with: + # path: ${{ env.pythonLocation }} + # key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade pip + # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # pip3 install -r requirements/dev.txt + # pip freeze + + # - name: Install singularity + # run: | + # sudo apt-get update && sudo apt-get install -y \ + # build-essential \ + # libssl-dev \ + # uuid-dev \ + # libgpgme11-dev \ + # squashfs-tools \ + # libseccomp-dev \ + # pkg-config + # wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + # sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + # rm go$GO_VERSION.$OS-$ARCH.tar.gz + # export PATH=$PATH:/usr/local/go/bin + # wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + # tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + # cd singularity-ce-$SINGULARITY_VERSION + # ./mconfig && \ + # make -C ./builddir && \ + # sudo make -C ./builddir install + steps: - name: Install merlin to run unit tests run: | pip3 install -e . @@ -153,6 +207,24 @@ jobs: run: | python3 tests/integration/run_tests.py --verbose --local + Integration-tests: + needs: common-setup + steps: + - name: Install merlin and setup redis as the broker + run: | + pip3 install -e . + merlin config --broker redis + + - name: Install CLI task dependencies generated from the 'feature demo' workflow + run: | + merlin example feature_demo + pip3 install -r feature_demo/requirements.txt + + # TODO remove the --ignore statement once those tests are fixed + - name: Run integration test suite for distributed tests + run: | + pytest --ignore tests/integration/test_celeryadapter.py tests/integration/ + Distributed-test-suite: runs-on: ubuntu-latest services: diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 59c1fa256..8f393bb58 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -650,82 +650,82 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } - stop_workers_tests = { - "stop workers no workers": { - "cmds": f"{stop}", - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop"), - HasRegex("step_1_merlin_test_worker", negate=True), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - }, - "stop workers no flags": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "stop workers with spec flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop} --spec {mul_workers_demo}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "stop workers with workers flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop} --workers step_1_merlin_test_worker step_2_merlin_test_worker", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "stop workers with queues flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{stop} --queues hello_queue", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found to stop", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, + # stop_workers_tests = { + # "stop workers no workers": { + # "cmds": f"{stop}", + # "conditions": [ + # HasReturnCode(), + # HasRegex("No workers found to stop"), + # HasRegex("step_1_merlin_test_worker", negate=True), + # HasRegex("step_2_merlin_test_worker", negate=True), + # HasRegex("other_merlin_test_worker", negate=True), + # ], + # "run type": "distributed", + # }, + # "stop workers no flags": { + # "cmds": [ + # f"{workers} {mul_workers_demo}", + # f"{stop}", + # ], + # "conditions": [ + # HasReturnCode(), + # HasRegex("No workers found to stop", negate=True), + # HasRegex("step_1_merlin_test_worker"), + # HasRegex("step_2_merlin_test_worker"), + # HasRegex("other_merlin_test_worker"), + # ], + # "run type": "distributed", + # "cleanup": KILL_WORKERS, + # "num procs": 2, + # }, + # "stop workers with spec flag": { + # "cmds": [ + # f"{workers} {mul_workers_demo}", + # f"{stop} --spec {mul_workers_demo}", + # ], + # "conditions": [ + # HasReturnCode(), + # HasRegex("No workers found to stop", negate=True), + # HasRegex("step_1_merlin_test_worker"), + # HasRegex("step_2_merlin_test_worker"), + # HasRegex("other_merlin_test_worker"), + # ], + # "run type": "distributed", + # "cleanup": KILL_WORKERS, + # "num procs": 2, + # }, + # "stop workers with workers flag": { + # "cmds": [ + # f"{workers} {mul_workers_demo}", + # f"{stop} --workers step_1_merlin_test_worker step_2_merlin_test_worker", + # ], + # "conditions": [ + # HasReturnCode(), + # HasRegex("No workers found to stop", negate=True), + # HasRegex("step_1_merlin_test_worker"), + # HasRegex("step_2_merlin_test_worker"), + # HasRegex("other_merlin_test_worker", negate=True), + # ], + # "run type": "distributed", + # "cleanup": KILL_WORKERS, + # "num procs": 2, + # }, + # "stop workers with queues flag": { + # "cmds": [ + # f"{workers} {mul_workers_demo}", + # f"{stop} --queues hello_queue", + # ], + # "conditions": [ + # HasReturnCode(), + # HasRegex("No workers found to stop", negate=True), + # HasRegex("step_1_merlin_test_worker"), + # HasRegex("step_2_merlin_test_worker", negate=True), + # HasRegex("other_merlin_test_worker", negate=True), + # ], + # "run type": "distributed", + # "cleanup": KILL_WORKERS, + # "num procs": 2, + # }, } query_workers_tests = { "query workers no workers": { From f93c7f627f5f875d5d78b3f8ec78b13be0a04a6d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 19 Sep 2024 17:28:11 -0700 Subject: [PATCH 142/201] add missing key to GH wf file --- .github/workflows/push-pr_workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index b90b7fa6f..33c39b52d 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -137,7 +137,7 @@ jobs: # TODO make this setup inheritable and then create unit/local/integration tests off of it? Local-test-suite: needs: common-setup - # runs-on: ubuntu-latest + runs-on: ubuntu-latest # env: # GO_VERSION: 1.18.1 # SINGULARITY_VERSION: 3.9.9 @@ -209,6 +209,7 @@ jobs: Integration-tests: needs: common-setup + runs-on: ubuntu-latest steps: - name: Install merlin and setup redis as the broker run: | From 835399c2468992c52d28383f785d67172197d8c5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 19 Sep 2024 17:31:49 -0700 Subject: [PATCH 143/201] fix invalid syntax in definitions.py --- tests/integration/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 8f393bb58..29c248158 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -726,7 +726,7 @@ def define_tests(): # pylint: disable=R0914,R0915 # "cleanup": KILL_WORKERS, # "num procs": 2, # }, - } + # } query_workers_tests = { "query workers no workers": { "cmds": f"{query}", From 176ff4daf1ba9cd442f4304c585d25888d31635d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 23 Sep 2024 18:05:02 -0700 Subject: [PATCH 144/201] comment out stop_workers tests --- tests/integration/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 29c248158..d4186f565 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -876,7 +876,7 @@ def define_tests(): # pylint: disable=R0914,R0915 # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, - stop_workers_tests, + # stop_workers_tests, query_workers_tests, distributed_tests, distributed_error_checks, From e38cc937f63762dd28b4a0e95b1addd7518f4112 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 23 Sep 2024 18:06:04 -0700 Subject: [PATCH 145/201] playing with new caches for workflow CI --- .github/workflows/push-pr_workflow.yml | 136 ++++++++++++++----------- 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 33c39b52d..7bdfdc621 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -82,7 +82,7 @@ jobs: python3 -m pylint merlin --rcfile=setup.cfg --exit-zero python3 -m pylint tests --rcfile=setup.cfg --exit-zero - common-setup: + Local-test-suite: runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 @@ -101,11 +101,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Check cache + - name: Cache Python dependencies uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + path: ~/.cache/pip + key: ${{ os.runner }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | @@ -113,6 +113,14 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt + - name: Cache Singularity build artifacts + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/go + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + - name: Install singularity run: | sudo apt-get update && sudo apt-get install -y \ @@ -134,61 +142,6 @@ jobs: make -C ./builddir && \ sudo make -C ./builddir install - # TODO make this setup inheritable and then create unit/local/integration tests off of it? - Local-test-suite: - needs: common-setup - runs-on: ubuntu-latest - # env: - # GO_VERSION: 1.18.1 - # SINGULARITY_VERSION: 3.9.9 - # OS: linux - # ARCH: amd64 - - # strategy: - # matrix: - # python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] - - # steps: - # - uses: actions/checkout@v2 - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v2 - # with: - # python-version: ${{ matrix.python-version }} - - # - name: Check cache - # uses: actions/cache@v2 - # with: - # path: ${{ env.pythonLocation }} - # key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - # - name: Install dependencies - # run: | - # python3 -m pip install --upgrade pip - # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - # pip3 install -r requirements/dev.txt - # pip freeze - - # - name: Install singularity - # run: | - # sudo apt-get update && sudo apt-get install -y \ - # build-essential \ - # libssl-dev \ - # uuid-dev \ - # libgpgme11-dev \ - # squashfs-tools \ - # libseccomp-dev \ - # pkg-config - # wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz - # sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz - # rm go$GO_VERSION.$OS-$ARCH.tar.gz - # export PATH=$PATH:/usr/local/go/bin - # wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz - # tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz - # cd singularity-ce-$SINGULARITY_VERSION - # ./mconfig && \ - # make -C ./builddir && \ - # sudo make -C ./builddir install - steps: - name: Install merlin to run unit tests run: | pip3 install -e . @@ -207,14 +160,73 @@ jobs: run: | python3 tests/integration/run_tests.py --verbose --local + # TODO get this and local-test-suite working + # - probably need to move the python 3.x call to this below + # - not sure if we need --broker redis in the Install step below Integration-tests: - needs: common-setup runs-on: ubuntu-latest + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 + + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + steps: - - name: Install merlin and setup redis as the broker + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache Python dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ os.runner }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip3 install -r requirements/dev.txt + + - name: Install merlin run: | pip3 install -e . - merlin config --broker redis + merlin config + + - name: Cache Singularity build artifacts + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/go + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install - name: Install CLI task dependencies generated from the 'feature demo' workflow run: | From c136058af61be6f294b582871620ffd87c7c95e1 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 23 Sep 2024 18:07:54 -0700 Subject: [PATCH 146/201] fix yaml syntax error --- .github/workflows/push-pr_workflow.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 7bdfdc621..9baaa38fa 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -114,12 +114,12 @@ jobs: pip3 install -r requirements/dev.txt - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/go - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/go + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity run: | @@ -200,12 +200,12 @@ jobs: merlin config - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/go - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/go + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity run: | From 56a6a057d4cf958029951f77cb0119fbc0795f5c Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 23 Sep 2024 18:10:29 -0700 Subject: [PATCH 147/201] fix typo for getting runner os --- .github/workflows/push-pr_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 9baaa38fa..40686f4eb 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -105,7 +105,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ os.runner }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | @@ -186,7 +186,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ os.runner }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | From f45a7984df6a77d0933a6408ffce0e2096a5c3ce Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 23 Sep 2024 18:26:38 -0700 Subject: [PATCH 148/201] fix test and add python version to CI cache --- .github/workflows/push-pr_workflow.yml | 13 ++++++++----- tests/integration/commands/test_stop_workers.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 40686f4eb..aa9e51f5f 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -105,7 +105,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | @@ -160,9 +160,12 @@ jobs: run: | python3 tests/integration/run_tests.py --verbose --local - # TODO get this and local-test-suite working - # - probably need to move the python 3.x call to this below - # - not sure if we need --broker redis in the Install step below + # TODO: + # - check if `merlin config` works just fine or if --broker redis is necessary + # - create common-setup job again but with correct caches + # - make local/integration tests depend on common-setup and check caches (see if this works) + # - see if local tests even need singularity + # - likely will for now but won't once server tests are ported to pytest Integration-tests: runs-on: ubuntu-latest env: @@ -186,7 +189,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py index ceb1639e8..c0100f50e 100644 --- a/tests/integration/commands/test_stop_workers.py +++ b/tests/integration/commands/test_stop_workers.py @@ -336,7 +336,7 @@ def test_spec_flag( path_to_test_specs, merlin_server_dir, conditions, - flag=f"--spec {os.path.join(path_to_test_specs, "multiple_workers.yaml")}" + flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}" ) def test_workers_flag( From 290d35078279bee2c8953d53b6e63f388048cfdf Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 24 Sep 2024 08:17:18 -0700 Subject: [PATCH 149/201] add in common-setup step again with caches this time --- .github/workflows/push-pr_workflow.yml | 73 +++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index aa9e51f5f..58a8095e1 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -82,7 +82,78 @@ jobs: python3 -m pylint merlin --rcfile=setup.cfg --exit-zero python3 -m pylint tests --rcfile=setup.cfg --exit-zero + Common-setup: + runs-on: ubuntu-latest + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 + + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache Python dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip3 install -r requirements/dev.txt + + - name: Cache Singularity build artifacts + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/go + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install + + - name: Install merlin to run unit tests + run: | + pip3 install -e . + merlin config + + - name: Install CLI task dependencies generated from the 'feature demo' workflow + run: | + merlin example feature_demo + pip3 install -r feature_demo/requirements.txt + Local-test-suite: + needs: Common-setup runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 @@ -161,12 +232,12 @@ jobs: python3 tests/integration/run_tests.py --verbose --local # TODO: - # - check if `merlin config` works just fine or if --broker redis is necessary # - create common-setup job again but with correct caches # - make local/integration tests depend on common-setup and check caches (see if this works) # - see if local tests even need singularity # - likely will for now but won't once server tests are ported to pytest Integration-tests: + needs: Common-setup runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 From 8a1bc14b37f69e2aa3a345973b1b28336f0410b2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 24 Sep 2024 08:17:44 -0700 Subject: [PATCH 150/201] run fix-style --- tests/conftest.py | 4 ++-- .../integration/commands/test_stop_workers.py | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1480a272d..374b8d836 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -104,8 +104,8 @@ def path_to_test_specs() -> str: """ Fixture to provide the path to the directory containing test specifications. - This fixture returns the absolute path to the 'test_specs' directory - within the 'integration' folder of the test directory. It expands + This fixture returns the absolute path to the 'test_specs' directory + within the 'integration' folder of the test directory. It expands environment variables and user home directory as necessary. Returns: diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py index c0100f50e..9f385fa64 100644 --- a/tests/integration/commands/test_stop_workers.py +++ b/tests/integration/commands/test_stop_workers.py @@ -6,14 +6,14 @@ import re import shutil import subprocess -import yaml from enum import Enum from typing import List +import yaml from celery import Celery -from tests.integration.conditions import HasRegex from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.integration.conditions import HasRegex class WorkerMessages(Enum): @@ -22,6 +22,7 @@ class WorkerMessages(Enum): that we're expecting (or not expecting) to see from the tests in this module. """ + NO_WORKERS_MSG = "No workers found to stop" STEP_1_WORKER = "step_1_merlin_test_worker" STEP_2_WORKER = "step_2_merlin_test_worker" @@ -59,14 +60,14 @@ def load_workers_from_spec(self, spec_filepath: str) -> dict: # Initialize an empty dictionary to hold worker_info worker_info = {} - + # Access workers and steps from spec_contents workers = spec_contents["merlin"]["resources"]["workers"] - study_steps = {step['name']: step['run']['task_queue'] for step in spec_contents['study']} + study_steps = {step["name"]: step["run"]["task_queue"] for step in spec_contents["study"]} # Grab the concurrency and queues from each worker and add it to the worker_info dict for worker_name, worker_settings in workers.items(): - match = re.search(r'--concurrency\s+(\d+)', worker_settings["args"]) + match = re.search(r"--concurrency\s+(\d+)", worker_settings["args"]) concurrency = int(match.group(1)) if match else 1 queues = [study_steps[step] for step in worker_settings["steps"]] worker_info[worker_name] = {"concurrency": concurrency, "queues": queues} @@ -237,7 +238,7 @@ def test_no_workers( for condition in conditions: condition.ingest_info(info) assert condition.passes - + def test_no_flags( self, redis_server: str, @@ -336,7 +337,7 @@ def test_spec_flag( path_to_test_specs, merlin_server_dir, conditions, - flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}" + flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", ) def test_workers_flag( @@ -387,7 +388,7 @@ def test_workers_flag( path_to_test_specs, merlin_server_dir, conditions, - flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}" + flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", ) def test_queues_flag( @@ -438,5 +439,5 @@ def test_queues_flag( path_to_test_specs, merlin_server_dir, conditions, - flag=f"--queues hello_queue" + flag=f"--queues hello_queue", ) From 58622bab94d8b4e241a276d4ea4c39f7aa23b0c4 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 24 Sep 2024 08:24:34 -0700 Subject: [PATCH 151/201] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index efa43f947..b84521d9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier - Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier - Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot` +- Moved stop-workers to a pytest test ## [1.12.2b1] ### Added From 917f8d73f21e12678c24213796f94cbdab071409 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 24 Sep 2024 08:31:26 -0700 Subject: [PATCH 152/201] fix remaining style issues --- tests/integration/commands/test_stop_workers.py | 3 +-- tests/integration/definitions.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py index 9f385fa64..ca3532828 100644 --- a/tests/integration/commands/test_stop_workers.py +++ b/tests/integration/commands/test_stop_workers.py @@ -10,7 +10,6 @@ from typing import List import yaml -from celery import Celery from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.integration.conditions import HasRegex @@ -439,5 +438,5 @@ def test_queues_flag( path_to_test_specs, merlin_server_dir, conditions, - flag=f"--queues hello_queue", + flag="--queues hello_queue", ) diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index d4186f565..9c9382f85 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -109,7 +109,7 @@ def define_tests(): # pylint: disable=R0914,R0915 run = f"merlin {err_lvl} run" restart = f"merlin {err_lvl} restart" purge = "merlin purge" - stop = "merlin stop-workers" + # stop = "merlin stop-workers" query = "merlin query-workers" # Shortcuts for example workflow paths From 91c75055f083c7844d6532cf3115cc89f6572712 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 24 Sep 2024 08:50:30 -0700 Subject: [PATCH 153/201] run without caches to compare execution time of test suite --- .github/workflows/push-pr_workflow.yml | 72 +++++++++++++------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 58a8095e1..59404df27 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -101,11 +101,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Cache Python dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Python dependencies + # uses: actions/cache@v2 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | @@ -113,13 +113,13 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/go - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Singularity build artifacts + # uses: actions/cache@v2 + # with: + # path: | + # ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + # /usr/local/go + # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity run: | @@ -172,11 +172,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Cache Python dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Python dependencies + # uses: actions/cache@v2 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | @@ -184,13 +184,13 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/go - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Singularity build artifacts + # uses: actions/cache@v2 + # with: + # path: | + # ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + # /usr/local/go + # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity run: | @@ -256,11 +256,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Cache Python dependencies - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Python dependencies + # uses: actions/cache@v2 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install dependencies run: | @@ -273,13 +273,13 @@ jobs: pip3 install -e . merlin config - - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/go - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Singularity build artifacts + # uses: actions/cache@v2 + # with: + # path: | + # ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + # /usr/local/go + # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity run: | From 608e00e5b8c989733500c6560a21bf80d4935a15 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 25 Sep 2024 11:47:31 -0700 Subject: [PATCH 154/201] allow redis config to not use ssl --- merlin/managers/redis_connection.py | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index 31a672c75..37ee149a4 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -71,8 +71,6 @@ def get_redis_connection(self) -> redis.Redis: port = CONFIG.results_backend.port if hasattr(CONFIG.results_backend, "port") else None results_db_num = CONFIG.results_backend.db_num if hasattr(CONFIG.results_backend, "db_num") else None username = CONFIG.results_backend.username if hasattr(CONFIG.results_backend, "username") else None - has_ssl = hasattr(CONFIG.results_backend, "cert_reqs") - ssl_cert_reqs = CONFIG.results_backend.cert_reqs if has_ssl else "required" password = None if password_file is not None: @@ -82,13 +80,21 @@ def get_redis_connection(self) -> redis.Redis: if hasattr(CONFIG.results_backend, "password"): password = CONFIG.results_backend.password - return redis.Redis( - host=server, - port=port, - db=results_db_num + self.db_num, # Increment db_num to avoid conflicts - username=username, - password=password, - decode_responses=True, - ssl=has_ssl, - ssl_cert_reqs=ssl_cert_reqs, - ) + # Base configuration for Redis connection (this does not have ssl) + redis_config = { + "host": server, + "port": port, + "db": results_db_num + self.db_num, # Increment db_num to avoid conflicts + "username": username, + "password": password, + "decode_responses": True, + } + + # Add ssl settings if necessary + if CONFIG.results_backend.name == "rediss": + redis_config.update({ + "ssl": True, + "ssl_cert_reqs": getattr(CONFIG.results_backend, "cert_reqs", "required"), + }) + + return redis.Redis(**redis_config) From bf41a2de16d12ddabc19e9ef7582fd9d22055761 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 26 Sep 2024 11:06:24 -0700 Subject: [PATCH 155/201] remove stop-workers and query-workers tests from definitions.py --- tests/integration/definitions.py | 158 ------------------------------- 1 file changed, 158 deletions(-) diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 9c9382f85..6725b8d56 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -109,8 +109,6 @@ def define_tests(): # pylint: disable=R0914,R0915 run = f"merlin {err_lvl} run" restart = f"merlin {err_lvl} restart" purge = "merlin purge" - # stop = "merlin stop-workers" - query = "merlin query-workers" # Shortcuts for example workflow paths examples = "merlin/examples/workflows" @@ -650,160 +648,6 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } - # stop_workers_tests = { - # "stop workers no workers": { - # "cmds": f"{stop}", - # "conditions": [ - # HasReturnCode(), - # HasRegex("No workers found to stop"), - # HasRegex("step_1_merlin_test_worker", negate=True), - # HasRegex("step_2_merlin_test_worker", negate=True), - # HasRegex("other_merlin_test_worker", negate=True), - # ], - # "run type": "distributed", - # }, - # "stop workers no flags": { - # "cmds": [ - # f"{workers} {mul_workers_demo}", - # f"{stop}", - # ], - # "conditions": [ - # HasReturnCode(), - # HasRegex("No workers found to stop", negate=True), - # HasRegex("step_1_merlin_test_worker"), - # HasRegex("step_2_merlin_test_worker"), - # HasRegex("other_merlin_test_worker"), - # ], - # "run type": "distributed", - # "cleanup": KILL_WORKERS, - # "num procs": 2, - # }, - # "stop workers with spec flag": { - # "cmds": [ - # f"{workers} {mul_workers_demo}", - # f"{stop} --spec {mul_workers_demo}", - # ], - # "conditions": [ - # HasReturnCode(), - # HasRegex("No workers found to stop", negate=True), - # HasRegex("step_1_merlin_test_worker"), - # HasRegex("step_2_merlin_test_worker"), - # HasRegex("other_merlin_test_worker"), - # ], - # "run type": "distributed", - # "cleanup": KILL_WORKERS, - # "num procs": 2, - # }, - # "stop workers with workers flag": { - # "cmds": [ - # f"{workers} {mul_workers_demo}", - # f"{stop} --workers step_1_merlin_test_worker step_2_merlin_test_worker", - # ], - # "conditions": [ - # HasReturnCode(), - # HasRegex("No workers found to stop", negate=True), - # HasRegex("step_1_merlin_test_worker"), - # HasRegex("step_2_merlin_test_worker"), - # HasRegex("other_merlin_test_worker", negate=True), - # ], - # "run type": "distributed", - # "cleanup": KILL_WORKERS, - # "num procs": 2, - # }, - # "stop workers with queues flag": { - # "cmds": [ - # f"{workers} {mul_workers_demo}", - # f"{stop} --queues hello_queue", - # ], - # "conditions": [ - # HasReturnCode(), - # HasRegex("No workers found to stop", negate=True), - # HasRegex("step_1_merlin_test_worker"), - # HasRegex("step_2_merlin_test_worker", negate=True), - # HasRegex("other_merlin_test_worker", negate=True), - # ], - # "run type": "distributed", - # "cleanup": KILL_WORKERS, - # "num procs": 2, - # }, - # } - query_workers_tests = { - "query workers no workers": { - "cmds": f"{query}", - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!"), - HasRegex("step_1_merlin_test_worker", negate=True), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - }, - "query workers no flags": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "query workers with spec flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query} --spec {mul_workers_demo}", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker"), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "query workers with workers flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query} --workers step_1_merlin_test_worker step_2_merlin_test_worker", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker"), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - "query workers with queues flag": { - "cmds": [ - f"{workers} {mul_workers_demo}", - f"{query} --queues hello_queue", - ], - "conditions": [ - HasReturnCode(), - HasRegex("No workers found!", negate=True), - HasRegex("step_1_merlin_test_worker"), - HasRegex("step_2_merlin_test_worker", negate=True), - HasRegex("other_merlin_test_worker", negate=True), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - }, - } distributed_tests = { # noqa: F841 "run and purge feature_demo": { "cmds": f"{run} {demo}; {purge} {demo} -f", @@ -876,8 +720,6 @@ def define_tests(): # pylint: disable=R0914,R0915 # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, - # stop_workers_tests, - query_workers_tests, distributed_tests, distributed_error_checks, ]: From 630c9c9f130e817b55776a27f2d2e1ee246a7785 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 26 Sep 2024 11:07:38 -0700 Subject: [PATCH 156/201] create helper_funcs file with common testing functions --- tests/integration/helper_funcs.py | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/integration/helper_funcs.py diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py new file mode 100644 index 000000000..3146b7cb7 --- /dev/null +++ b/tests/integration/helper_funcs.py @@ -0,0 +1,111 @@ +""" +This module contains helper functions for the integration +test suite. +""" + +import os +import shutil +import re +from typing import Dict, List + +import yaml + +from tests.integration.conditions import Condition + +def load_workers_from_spec(spec_filepath: str) -> dict: + """ + Load worker specifications from a YAML file. + + This function reads a YAML file containing study specifications and + extracts the worker information under the "merlin" section. It + constructs a dictionary in the form that CeleryWorkersManager.launch_workers + requires. + + Parameters: + spec_filepath: The file path to the YAML specification file. + + Returns: + A dictionary containing the worker specifications from the + "merlin" section of the YAML file. + """ + # Read in the contents of the spec file + with open(spec_filepath, "r") as spec_file: + spec_contents = yaml.load(spec_file, yaml.Loader) + + # Initialize an empty dictionary to hold worker_info + worker_info = {} + + # Access workers and steps from spec_contents + workers = spec_contents["merlin"]["resources"]["workers"] + study_steps = {step["name"]: step["run"]["task_queue"] for step in spec_contents["study"]} + + # Grab the concurrency and queues from each worker and add it to the worker_info dict + for worker_name, worker_settings in workers.items(): + match = re.search(r"--concurrency\s+(\d+)", worker_settings["args"]) + concurrency = int(match.group(1)) if match else 1 + queues = [study_steps[step] for step in worker_settings["steps"]] + worker_info[worker_name] = {"concurrency": concurrency, "queues": queues} + + return worker_info + + +def copy_app_yaml_to_cwd(merlin_server_dir: str): + """ + Copy the app.yaml file from the directory provided to the current working + directory. + + Grab the app.yaml file from `merlin_server_dir` and copy it to the current + working directory so that Merlin will read this in as the server configuration + for whatever test is calling this. + + Parameters: + merlin_server_dir: + The path to the `merlin_server` directory that should be created by the + `redis_server` fixture. + """ + copied_app_yaml = os.path.join(os.getcwd(), "app.yaml") + if not os.path.exists(copied_app_yaml): + server_app_yaml = os.path.join(merlin_server_dir, "app.yaml") + shutil.copy(server_app_yaml, copied_app_yaml) + + +def check_test_conditions(conditions: List[Condition], info: Dict[str, str]): + """ + Ensure all specified test conditions are satisfied based on the output + from a subprocess. + + This function iterates through a list of `Condition` instances, ingests + the provided information (stdout, stderr, and return code) for each + condition, and checks if each condition passes. If any condition fails, + an AssertionError is raised with a detailed message that includes the + condition that failed, along with the captured output and return code. + + Parameters: + conditions: + A list of Condition instances that define the expectations for the test. + info: + A dictionary containing the output from the subprocess, which should + include the following keys: + - 'stdout': The standard output captured from the subprocess. + - 'stderr': The standard error output captured from the subprocess. + - 'return_code': The return code of the subprocess, indicating success + or failure of the command executed. + + Raises: + AssertionError + If any of the conditions do not pass, an AssertionError is raised with + a detailed message including the failed condition and the subprocess + output. + """ + for condition in conditions: + condition.ingest_info(info) + try: + assert condition.passes + except AssertionError: + error_message = ( + f"Condition failed: {condition}\n" + f"Captured stdout: {info['stdout']}\n" + f"Captured stderr: {info['stderr']}\n" + f"Return code: {info['return_code']}\n" + ) + raise AssertionError(error_message) \ No newline at end of file From 17889fdee2c1a780bcaf100cefc09fd3d6bfac8d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 26 Sep 2024 11:08:30 -0700 Subject: [PATCH 157/201] move query-workers to pytest and add base class w/ stop-workers tests --- tests/integration/commands/base_classes.py | 120 ++++++++ .../commands/test_query_workers.py | 264 ++++++++++++++++++ .../integration/commands/test_stop_workers.py | 210 ++------------ 3 files changed, 401 insertions(+), 193 deletions(-) create mode 100644 tests/integration/commands/base_classes.py create mode 100644 tests/integration/commands/test_query_workers.py diff --git a/tests/integration/commands/base_classes.py b/tests/integration/commands/base_classes.py new file mode 100644 index 000000000..802da1fd0 --- /dev/null +++ b/tests/integration/commands/base_classes.py @@ -0,0 +1,120 @@ +""" +This module will contain the base classes used for +the integration tests in this command directory. +""" + +import os +import subprocess +from typing import List + +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.integration.conditions import Condition +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec + + +class BaseWorkerInteractionTests: + """ + Base class for tests that interact with the worker in some way. + Contains necessary methods for executing the tests. + """ + + def run_test_with_workers( + self, + path_to_test_specs: str, + merlin_server_dir: str, + conditions: List[Condition], + command: str, + flag: str = None, + ): + """ + Helper function to run common testing logic for tests with workers started. + + This function will: + 0. Read in the necessary fixtures as parameters. These fixtures grab paths to + our test specs and the merlin server directory created from starting the + containerized redis server. + 1. Load in the worker specifications from the `multiple_workers.yaml` file. + 2. Use a context manager to start up the workers on the celery app connected to + the containerized redis server + 3. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 4. Run the test command that's provided and check that the conditions given are + passing. + + Parameters: + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + conditions: + A list of `Condition` instances that need to pass in order for this test to + be successful. + command: + The command that we're testing. E.g. "merlin stop-workers" + flag: + An optional flag to add to the command that we're testing so we can test + different functionality for the command. + """ + from merlin.celery import app as celery_app + + # Grab worker configurations from the spec file + multiple_worker_spec = os.path.join(path_to_test_specs, "multiple_workers.yaml") + workers_from_spec = load_workers_from_spec(multiple_worker_spec) + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as workers_manager: + workers_manager.launch_workers(workers_from_spec) + + # Copy the app.yaml to the cwd so merlin will connect to the testing server + copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + cmd_to_test = f"{command} {flag}" if flag else command + result = subprocess.run(cmd_to_test, capture_output=True, text=True, shell=True) + + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + check_test_conditions(conditions, info) + + def run_test_without_workers(self, merlin_server_dir: str, conditions: List[Condition], command: str): + """ + Helper function to run common testing logic for tests with no workers started. + + This test will: + 0. Read in the `merlin_server_dir` fixture as a parameter. This fixture is a + path to the merlin server directory created from starting the containerized + redis server. + 1. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 2. Run the test command that's provided and check that the conditions given are + passing. + + Parameters: + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + conditions: + A list of `Condition` instances that need to pass in order for this test to + be successful. + command: + The command that we're testing. E.g. "merlin stop-workers" + """ + # Copy the app.yaml to the cwd so merlin will connect to the testing server + copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + result = subprocess.run(self.command_to_test, capture_output=True, text=True, shell=True) + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + check_test_conditions(conditions, info) \ No newline at end of file diff --git a/tests/integration/commands/test_query_workers.py b/tests/integration/commands/test_query_workers.py new file mode 100644 index 000000000..abd3ba0a9 --- /dev/null +++ b/tests/integration/commands/test_query_workers.py @@ -0,0 +1,264 @@ +""" +Tests for the `merlin query-workers` command. +""" + +import os +from enum import Enum + +from tests.integration.commands.base_classes import BaseWorkerInteractionTests +from tests.integration.conditions import HasRegex + + +class WorkerMessages(Enum): + """ + Enumerated strings to help keep track of the messages + that we're expecting (or not expecting) to see from the + tests in this module. + """ + + NO_WORKERS_MSG = "No workers found!" + STEP_1_WORKER = "step_1_merlin_test_worker" + STEP_2_WORKER = "step_2_merlin_test_worker" + OTHER_WORKER = "other_merlin_test_worker" + + +class TestQueryWorkers(BaseWorkerInteractionTests): + """ + Tests for the `merlin query-workers` command. Most of these tests will: + 1. Start workers from a spec file used for testing + - Use CeleryWorkerManager for this to ensure safe stoppage of workers + if something goes wrong + 2. Run the `merlin query-workers` command from a subprocess + """ + + command_to_test = "merlin query-workers" + + def test_no_workers( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + merlin_server_dir: str, + ): + """ + Test the `merlin query-workers` command with no workers started in the first place. + + Run the `merlin query-workers` command and ensure that a "no workers found" message + is written to the output. To see more information on exactly what this test is doing, + see the `run_test_without_workers()` method of the base class. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value), # No workers should be launched so we should see this + HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), # None of these workers should be started + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # None of these workers should be started + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # None of these workers should be started + ] + self.run_test_without_workers(merlin_server_dir, conditions, self.command_to_test) + + def test_no_flags( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin query-workers` command with no flags. + + Run the `merlin query-workers` command and ensure that all workers are queried. + To see more information on exactly what this test is doing, see the + `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be found + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be queried + ] + self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test) + + def test_spec_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin query-workers` command with the `--spec` flag. + + Run the `merlin query-workers` command with the `--spec` flag and ensure that all + workers are queried. To see more information on exactly what this test is doing, + see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be queried + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be queried + ] + self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + self.command_to_test, + flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", + ) + + def test_workers_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin query-workers` command with the `--workers` flag. + + Run the `merlin query-workers` command with the `--workers` flag and ensure that + only the workers given with this flag are queried. To see more information on + exactly what this test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be queried + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be queried + ] + self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + self.command_to_test, + flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", + ) + + def test_queues_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + ): + """ + Test the `merlin query-workers` command with the `--queues` flag. + + Run the `merlin query-workers` command with the `--queues` flag and ensure that + only the workers attached to the given queues are queried. To see more information + on exactly what this test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + conditions = [ + HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # One worker should be queried + HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # This worker should NOT be queried + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be queried + ] + self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + self.command_to_test, + flag="--queues hello_queue", + ) diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py index ca3532828..6922157ab 100644 --- a/tests/integration/commands/test_stop_workers.py +++ b/tests/integration/commands/test_stop_workers.py @@ -3,15 +3,9 @@ """ import os -import re -import shutil -import subprocess from enum import Enum -from typing import List -import yaml - -from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.integration.commands.base_classes import BaseWorkerInteractionTests from tests.integration.conditions import HasRegex @@ -28,7 +22,7 @@ class WorkerMessages(Enum): OTHER_WORKER = "other_merlin_test_worker" -class TestStopWorkers: +class TestStopWorkers(BaseWorkerInteractionTests): """ Tests for the `merlin stop-workers` command. Most of these tests will: 1. Start workers from a spec file used for testing @@ -37,143 +31,7 @@ class TestStopWorkers: 2. Run the `merlin stop-workers` command from a subprocess """ - def load_workers_from_spec(self, spec_filepath: str) -> dict: - """ - Load worker specifications from a YAML file. - - This function reads a YAML file containing study specifications and - extracts the worker information under the "merlin" section. It - constructs a dictionary in the form that CeleryWorkersManager.launch_workers - requires. - - Parameters: - spec_filepath: The file path to the YAML specification file. - - Returns: - A dictionary containing the worker specifications from the - "merlin" section of the YAML file. - """ - # Read in the contents of the spec file - with open(spec_filepath, "r") as spec_file: - spec_contents = yaml.load(spec_file, yaml.Loader) - - # Initialize an empty dictionary to hold worker_info - worker_info = {} - - # Access workers and steps from spec_contents - workers = spec_contents["merlin"]["resources"]["workers"] - study_steps = {step["name"]: step["run"]["task_queue"] for step in spec_contents["study"]} - - # Grab the concurrency and queues from each worker and add it to the worker_info dict - for worker_name, worker_settings in workers.items(): - match = re.search(r"--concurrency\s+(\d+)", worker_settings["args"]) - concurrency = int(match.group(1)) if match else 1 - queues = [study_steps[step] for step in worker_settings["steps"]] - worker_info[worker_name] = {"concurrency": concurrency, "queues": queues} - - return worker_info - - def copy_app_yaml_to_cwd(self, merlin_server_dir: str): - """ - Copy the app.yaml file from the directory provided to the current working - directory. - - Grab the app.yaml file from `merlin_server_dir` and copy it to the current - working directory so that Merlin will read this in as the server configuration - for whatever test is calling this. - - Parameters: - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - copied_app_yaml = os.path.join(os.getcwd(), "app.yaml") - if not os.path.exists(copied_app_yaml): - server_app_yaml = os.path.join(merlin_server_dir, "app.yaml") - shutil.copy(server_app_yaml, copied_app_yaml) - - def run_test_with_workers( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - conditions: List, - flag: str = None, - ): - """ - Helper function to run common testing logic for tests with workers started. - - This function will: - 0. Read in the necessary fixtures as parameters. The purpose of these fixtures - include: - - Starting a containerized redis server - - Updating the CONFIG object to point to the containerized redis server - - Grabbing paths to our test specs and the merlin server directory created - from starting the containerized redis server - 1. Load in the worker specifications from the `multiple_workers.yaml` file. - 2. Use a context manager to start up the workers on the celery app connected to - the containerized redis server - 3. Copy the app.yaml file for the containerized redis server to the current working - directory so that merlin will connect to it when we run our test - 4. Run the test command that's provided and check that the conditions given are - passing. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - conditions: - A list of `Condition` instances that need to pass in order for this test to - be successful. - flag: - An optional flag to add to the `merlin stop-workers` command so we can test - different functionality for the command. - """ - from merlin.celery import app as celery_app - - # Grab worker configurations from the spec file - multiple_worker_spec = os.path.join(path_to_test_specs, "multiple_workers.yaml") - workers_from_spec = self.load_workers_from_spec(multiple_worker_spec) - - # We use a context manager to start workers so that they'll safely stop even if this test fails - with CeleryWorkersManager(celery_app) as workers_manager: - workers_manager.launch_workers(workers_from_spec) - - # Copy the app.yaml to the cwd so merlin will connect to the testing server - self.copy_app_yaml_to_cwd(merlin_server_dir) - - # Run the test - stop_workers_cmd = "merlin stop-workers" - cmd_to_test = f"{stop_workers_cmd} {flag}" if flag is not None else stop_workers_cmd - result = subprocess.run(cmd_to_test, capture_output=True, text=True, shell=True) - - info = { - "stdout": result.stdout, - "stderr": result.stderr, - "return_code": result.returncode, - } - - # Ensure all test conditions are satisfied - for condition in conditions: - condition.ingest_info(info) - assert condition.passes + command_to_test = "merlin stop-workers" def test_no_workers( self, @@ -185,16 +43,9 @@ def test_no_workers( """ Test the `merlin stop-workers` command with no workers started in the first place. - This test will: - 0. Set up the appropriate fixtures. This includes: - - Starting a containerized redis server - - Updating the CONFIG object to point to the containerized redis server - - Grabbing the path to the merlin server directory created from starting the - containerized redis server - 1. Copy the app.yaml file for the containerized redis server to the current working - directory so that merlin will connect to it when we run our test - 2. Run the `merlin stop-workers` command and ensure that no workers are stopped (this - should be the case since no workers were started in the first place) + Run the `merlin stop-workers` command and ensure that a "no workers found" message + is written to the output. To see more information on exactly what this test is doing, + see the `run_test_without_workers()` method of the base class. Parameters: redis_server: @@ -214,29 +65,13 @@ def test_no_workers( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ - # Copy the app.yaml to the cwd so merlin will connect to the testing server - self.copy_app_yaml_to_cwd(merlin_server_dir) - - # Define our test conditions conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value), # No workers should be launched so we should see this HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), # None of these workers should be started HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # None of these workers should be started HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # None of these workers should be started ] - - # Run the test - result = subprocess.run("merlin stop-workers", capture_output=True, text=True, shell=True) - info = { - "stdout": result.stdout, - "stderr": result.stderr, - "return_code": result.returncode, - } - - # Ensure all test conditions are satisfied - for condition in conditions: - condition.ingest_info(info) - assert condition.passes + self.run_test_without_workers(merlin_server_dir, conditions, self.command_to_test) def test_no_flags( self, @@ -251,7 +86,7 @@ def test_no_flags( Run the `merlin stop-workers` command and ensure that all workers are stopped. To see more information on exactly what this test is doing, see the - `run_test_with_workers()` method. + `run_test_with_workers()` method of the base class. Parameters: redis_server: @@ -279,14 +114,7 @@ def test_no_flags( HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped ] - self.run_test_with_workers( - redis_server, - redis_results_backend_config, - redis_broker_config, - path_to_test_specs, - merlin_server_dir, - conditions, - ) + self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test) def test_spec_flag( self, @@ -301,7 +129,7 @@ def test_spec_flag( Run the `merlin stop-workers` command with the `--spec` flag and ensure that all workers are stopped. To see more information on exactly what this test is doing, - see the `run_test_with_workers()` method. + see the `run_test_with_workers()` method of the base class. Parameters: redis_server: @@ -330,12 +158,10 @@ def test_spec_flag( HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped ] self.run_test_with_workers( - redis_server, - redis_results_backend_config, - redis_broker_config, path_to_test_specs, merlin_server_dir, conditions, + self.command_to_test, flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", ) @@ -352,7 +178,8 @@ def test_workers_flag( Run the `merlin stop-workers` command with the `--workers` flag and ensure that only the workers given with this flag are stopped. To see more information on - exactly what this test is doing, see the `run_test_with_workers()` method. + exactly what this test is doing, see the `run_test_with_workers()` method of the + base class. Parameters: redis_server: @@ -381,12 +208,10 @@ def test_workers_flag( HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped ] self.run_test_with_workers( - redis_server, - redis_results_backend_config, - redis_broker_config, path_to_test_specs, merlin_server_dir, conditions, + self.command_to_test, flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", ) @@ -403,7 +228,8 @@ def test_queues_flag( Run the `merlin stop-workers` command with the `--queues` flag and ensure that only the workers attached to the given queues are stopped. To see more information - on exactly what this test is doing, see the `run_test_with_workers()` method. + on exactly what this test is doing, see the `run_test_with_workers()` method of the + base class. Parameters: redis_server: @@ -432,11 +258,9 @@ def test_queues_flag( HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped ] self.run_test_with_workers( - redis_server, - redis_results_backend_config, - redis_broker_config, path_to_test_specs, merlin_server_dir, conditions, + self.command_to_test, flag="--queues hello_queue", ) From 5c28b49b2822b8f351a98a100b4ca74eb8875896 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 26 Sep 2024 11:08:48 -0700 Subject: [PATCH 158/201] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b84521d9f..b74389865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier - Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier - Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot` -- Moved stop-workers to a pytest test +- Moved stop-workers and query-workers integration tests to pytest tests ## [1.12.2b1] ### Added From 643b4d1b7580900bf4495697407d915829e98cc6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 27 Sep 2024 12:19:49 -0700 Subject: [PATCH 159/201] final changes for the stop-workers & query-workers tests --- .../celery_workers_manager.py | 5 ++- tests/integration/commands/base_classes.py | 24 +++++++---- .../commands/test_query_workers.py | 22 ++++++---- .../integration/commands/test_stop_workers.py | 41 +++++++++++++++---- tests/integration/definitions.py | 1 - tests/integration/helper_funcs.py | 5 ++- 6 files changed, 68 insertions(+), 30 deletions(-) diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index f58c38872..2daecb612 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -80,8 +80,9 @@ def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: T try: if str(pid) in ps_proc.stdout: os.kill(pid, signal.SIGKILL) - except ProcessLookupError as exc: - raise ProcessLookupError(f"PID {pid} not found. Output of 'ps ux':\n{ps_proc.stdout}") from exc + # If the process can't be found then it doesn't exist anymore + except ProcessLookupError: + pass def _is_worker_ready(self, worker_name: str, verbose: bool = False) -> bool: """ diff --git a/tests/integration/commands/base_classes.py b/tests/integration/commands/base_classes.py index 802da1fd0..917e8db6a 100644 --- a/tests/integration/commands/base_classes.py +++ b/tests/integration/commands/base_classes.py @@ -5,6 +5,7 @@ import os import subprocess +from contextlib import contextmanager from typing import List from tests.context_managers.celery_workers_manager import CeleryWorkersManager @@ -12,12 +13,13 @@ from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec -class BaseWorkerInteractionTests: +class BaseStopWorkersAndQueryWorkersTest: """ - Base class for tests that interact with the worker in some way. + Base class for `stop-workers` and `query-workers` tests. Contains necessary methods for executing the tests. """ + @contextmanager def run_test_with_workers( self, path_to_test_specs: str, @@ -27,9 +29,12 @@ def run_test_with_workers( flag: str = None, ): """ - Helper function to run common testing logic for tests with workers started. + Helper method to run common testing logic for tests with workers started. + This method must also be a context manager so we can check the status of the + workers prior to the CeleryWorkersManager running it's exit code that shuts down + all active workers. - This function will: + This method will: 0. Read in the necessary fixtures as parameters. These fixtures grab paths to our test specs and the merlin server directory created from starting the containerized redis server. @@ -40,6 +45,9 @@ def run_test_with_workers( directory so that merlin will connect to it when we run our test 4. Run the test command that's provided and check that the conditions given are passing. + 5. Yield control back to the calling method. + 6. Safely terminate workers that may have not been stopped once the calling method + completes. Parameters: path_to_test_specs: @@ -81,7 +89,9 @@ def run_test_with_workers( # Ensure all test conditions are satisfied check_test_conditions(conditions, info) - + + yield + def run_test_without_workers(self, merlin_server_dir: str, conditions: List[Condition], command: str): """ Helper function to run common testing logic for tests with no workers started. @@ -109,7 +119,7 @@ def run_test_without_workers(self, merlin_server_dir: str, conditions: List[Cond copy_app_yaml_to_cwd(merlin_server_dir) # Run the test - result = subprocess.run(self.command_to_test, capture_output=True, text=True, shell=True) + result = subprocess.run(command, capture_output=True, text=True, shell=True) info = { "stdout": result.stdout, "stderr": result.stderr, @@ -117,4 +127,4 @@ def run_test_without_workers(self, merlin_server_dir: str, conditions: List[Cond } # Ensure all test conditions are satisfied - check_test_conditions(conditions, info) \ No newline at end of file + check_test_conditions(conditions, info) diff --git a/tests/integration/commands/test_query_workers.py b/tests/integration/commands/test_query_workers.py index abd3ba0a9..d1b4cf9de 100644 --- a/tests/integration/commands/test_query_workers.py +++ b/tests/integration/commands/test_query_workers.py @@ -5,7 +5,7 @@ import os from enum import Enum -from tests.integration.commands.base_classes import BaseWorkerInteractionTests +from tests.integration.commands.base_classes import BaseStopWorkersAndQueryWorkersTest from tests.integration.conditions import HasRegex @@ -22,7 +22,7 @@ class WorkerMessages(Enum): OTHER_WORKER = "other_merlin_test_worker" -class TestQueryWorkers(BaseWorkerInteractionTests): +class TestQueryWorkers(BaseStopWorkersAndQueryWorkersTest): """ Tests for the `merlin query-workers` command. Most of these tests will: 1. Start workers from a spec file used for testing @@ -114,7 +114,8 @@ def test_no_flags( HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be queried ] - self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test) + with self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test): + pass def test_spec_flag( self, @@ -157,13 +158,14 @@ def test_spec_flag( HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be queried ] - self.run_test_with_workers( + with self.run_test_with_workers( path_to_test_specs, merlin_server_dir, conditions, self.command_to_test, flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", - ) + ): + pass def test_workers_flag( self, @@ -206,13 +208,14 @@ def test_workers_flag( HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be queried ] - self.run_test_with_workers( + with self.run_test_with_workers( path_to_test_specs, merlin_server_dir, conditions, self.command_to_test, flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", - ) + ): + pass def test_queues_flag( self, @@ -255,10 +258,11 @@ def test_queues_flag( HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # This worker should NOT be queried HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be queried ] - self.run_test_with_workers( + with self.run_test_with_workers( path_to_test_specs, merlin_server_dir, conditions, self.command_to_test, flag="--queues hello_queue", - ) + ): + pass diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py index 6922157ab..ed2b360bc 100644 --- a/tests/integration/commands/test_stop_workers.py +++ b/tests/integration/commands/test_stop_workers.py @@ -5,7 +5,7 @@ import os from enum import Enum -from tests.integration.commands.base_classes import BaseWorkerInteractionTests +from tests.integration.commands.base_classes import BaseStopWorkersAndQueryWorkersTest from tests.integration.conditions import HasRegex @@ -22,7 +22,7 @@ class WorkerMessages(Enum): OTHER_WORKER = "other_merlin_test_worker" -class TestStopWorkers(BaseWorkerInteractionTests): +class TestStopWorkers(BaseStopWorkersAndQueryWorkersTest): """ Tests for the `merlin stop-workers` command. Most of these tests will: 1. Start workers from a spec file used for testing @@ -108,13 +108,21 @@ def test_no_flags( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ + from merlin.celery import app as celery_app + + # Define test conditions conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped ] - self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test) + + # Run the test + with self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test): + # After the test runs and before the CeleryWorkersManager exits, ensure there are no workers on the app + active_queues = celery_app.control.inspect().active_queues() + assert active_queues is None def test_spec_flag( self, @@ -151,19 +159,22 @@ def test_spec_flag( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ + from merlin.celery import app as celery_app conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped ] - self.run_test_with_workers( + with self.run_test_with_workers( path_to_test_specs, merlin_server_dir, conditions, self.command_to_test, flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", - ) + ): + active_queues = celery_app.control.inspect().active_queues() + assert active_queues is None def test_workers_flag( self, @@ -201,19 +212,24 @@ def test_workers_flag( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ + from merlin.celery import app as celery_app conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped ] - self.run_test_with_workers( + with self.run_test_with_workers( path_to_test_specs, merlin_server_dir, conditions, self.command_to_test, flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", - ) + ): + active_queues = celery_app.control.inspect().active_queues() + worker_name = f"celery@{WorkerMessages.OTHER_WORKER.value}" + assert worker_name in active_queues + def test_queues_flag( self, @@ -251,16 +267,23 @@ def test_queues_flag( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ + from merlin.celery import app as celery_app conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # One workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # This worker should NOT be stopped HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped ] - self.run_test_with_workers( + with self.run_test_with_workers( path_to_test_specs, merlin_server_dir, conditions, self.command_to_test, flag="--queues hello_queue", - ) + ): + active_queues = celery_app.control.inspect().active_queues() + workers_that_should_be_alive = [ + f"celery@{WorkerMessages.OTHER_WORKER.value}", f"celery@{WorkerMessages.STEP_2_WORKER.value}" + ] + for worker_name in workers_that_should_be_alive: + assert worker_name in active_queues diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 6725b8d56..eeb23fc71 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -124,7 +124,6 @@ def define_tests(): # pylint: disable=R0914,R0915 flux_restart = f"{examples}/flux/flux_par_restart.yaml" flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" - mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" cli_substitution_wf = f"{test_specs}/cli_substitution_test.yaml" chord_err_wf = f"{test_specs}/chord_err.yaml" diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py index 3146b7cb7..ac3d5d7de 100644 --- a/tests/integration/helper_funcs.py +++ b/tests/integration/helper_funcs.py @@ -4,14 +4,15 @@ """ import os -import shutil import re +import shutil from typing import Dict, List import yaml from tests.integration.conditions import Condition + def load_workers_from_spec(spec_filepath: str) -> dict: """ Load worker specifications from a YAML file. @@ -108,4 +109,4 @@ def check_test_conditions(conditions: List[Condition], info: Dict[str, str]): f"Captured stderr: {info['stderr']}\n" f"Return code: {info['return_code']}\n" ) - raise AssertionError(error_message) \ No newline at end of file + raise AssertionError(error_message) From 0f0264c0be21926bd6d43b8c18282b9e9c401a1c Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 27 Sep 2024 12:21:08 -0700 Subject: [PATCH 160/201] run fix-style --- tests/integration/commands/test_stop_workers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py index ed2b360bc..c8b424f01 100644 --- a/tests/integration/commands/test_stop_workers.py +++ b/tests/integration/commands/test_stop_workers.py @@ -160,6 +160,7 @@ def test_spec_flag( created by the `redis_server` fixture. """ from merlin.celery import app as celery_app + conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped @@ -213,6 +214,7 @@ def test_workers_flag( created by the `redis_server` fixture. """ from merlin.celery import app as celery_app + conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped @@ -230,7 +232,6 @@ def test_workers_flag( worker_name = f"celery@{WorkerMessages.OTHER_WORKER.value}" assert worker_name in active_queues - def test_queues_flag( self, redis_server: str, @@ -268,6 +269,7 @@ def test_queues_flag( created by the `redis_server` fixture. """ from merlin.celery import app as celery_app + conditions = [ HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # One workers should be stopped HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped @@ -283,7 +285,8 @@ def test_queues_flag( ): active_queues = celery_app.control.inspect().active_queues() workers_that_should_be_alive = [ - f"celery@{WorkerMessages.OTHER_WORKER.value}", f"celery@{WorkerMessages.STEP_2_WORKER.value}" + f"celery@{WorkerMessages.OTHER_WORKER.value}", + f"celery@{WorkerMessages.STEP_2_WORKER.value}", ] for worker_name in workers_that_should_be_alive: assert worker_name in active_queues From 634060466561d9e313a19f1f368a68ac05e801f4 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 14:34:26 -0700 Subject: [PATCH 161/201] move stop and query workers tests to the same file --- tests/integration/commands/base_classes.py | 130 ------ .../commands/test_query_workers.py | 268 ----------- .../commands/test_stop_and_query_workers.py | 442 ++++++++++++++++++ .../integration/commands/test_stop_workers.py | 292 ------------ tests/integration/helper_funcs.py | 4 +- 5 files changed, 444 insertions(+), 692 deletions(-) delete mode 100644 tests/integration/commands/base_classes.py delete mode 100644 tests/integration/commands/test_query_workers.py create mode 100644 tests/integration/commands/test_stop_and_query_workers.py delete mode 100644 tests/integration/commands/test_stop_workers.py diff --git a/tests/integration/commands/base_classes.py b/tests/integration/commands/base_classes.py deleted file mode 100644 index 917e8db6a..000000000 --- a/tests/integration/commands/base_classes.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -This module will contain the base classes used for -the integration tests in this command directory. -""" - -import os -import subprocess -from contextlib import contextmanager -from typing import List - -from tests.context_managers.celery_workers_manager import CeleryWorkersManager -from tests.integration.conditions import Condition -from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec - - -class BaseStopWorkersAndQueryWorkersTest: - """ - Base class for `stop-workers` and `query-workers` tests. - Contains necessary methods for executing the tests. - """ - - @contextmanager - def run_test_with_workers( - self, - path_to_test_specs: str, - merlin_server_dir: str, - conditions: List[Condition], - command: str, - flag: str = None, - ): - """ - Helper method to run common testing logic for tests with workers started. - This method must also be a context manager so we can check the status of the - workers prior to the CeleryWorkersManager running it's exit code that shuts down - all active workers. - - This method will: - 0. Read in the necessary fixtures as parameters. These fixtures grab paths to - our test specs and the merlin server directory created from starting the - containerized redis server. - 1. Load in the worker specifications from the `multiple_workers.yaml` file. - 2. Use a context manager to start up the workers on the celery app connected to - the containerized redis server - 3. Copy the app.yaml file for the containerized redis server to the current working - directory so that merlin will connect to it when we run our test - 4. Run the test command that's provided and check that the conditions given are - passing. - 5. Yield control back to the calling method. - 6. Safely terminate workers that may have not been stopped once the calling method - completes. - - Parameters: - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - conditions: - A list of `Condition` instances that need to pass in order for this test to - be successful. - command: - The command that we're testing. E.g. "merlin stop-workers" - flag: - An optional flag to add to the command that we're testing so we can test - different functionality for the command. - """ - from merlin.celery import app as celery_app - - # Grab worker configurations from the spec file - multiple_worker_spec = os.path.join(path_to_test_specs, "multiple_workers.yaml") - workers_from_spec = load_workers_from_spec(multiple_worker_spec) - - # We use a context manager to start workers so that they'll safely stop even if this test fails - with CeleryWorkersManager(celery_app) as workers_manager: - workers_manager.launch_workers(workers_from_spec) - - # Copy the app.yaml to the cwd so merlin will connect to the testing server - copy_app_yaml_to_cwd(merlin_server_dir) - - # Run the test - cmd_to_test = f"{command} {flag}" if flag else command - result = subprocess.run(cmd_to_test, capture_output=True, text=True, shell=True) - - info = { - "stdout": result.stdout, - "stderr": result.stderr, - "return_code": result.returncode, - } - - # Ensure all test conditions are satisfied - check_test_conditions(conditions, info) - - yield - - def run_test_without_workers(self, merlin_server_dir: str, conditions: List[Condition], command: str): - """ - Helper function to run common testing logic for tests with no workers started. - - This test will: - 0. Read in the `merlin_server_dir` fixture as a parameter. This fixture is a - path to the merlin server directory created from starting the containerized - redis server. - 1. Copy the app.yaml file for the containerized redis server to the current working - directory so that merlin will connect to it when we run our test - 2. Run the test command that's provided and check that the conditions given are - passing. - - Parameters: - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - conditions: - A list of `Condition` instances that need to pass in order for this test to - be successful. - command: - The command that we're testing. E.g. "merlin stop-workers" - """ - # Copy the app.yaml to the cwd so merlin will connect to the testing server - copy_app_yaml_to_cwd(merlin_server_dir) - - # Run the test - result = subprocess.run(command, capture_output=True, text=True, shell=True) - info = { - "stdout": result.stdout, - "stderr": result.stderr, - "return_code": result.returncode, - } - - # Ensure all test conditions are satisfied - check_test_conditions(conditions, info) diff --git a/tests/integration/commands/test_query_workers.py b/tests/integration/commands/test_query_workers.py deleted file mode 100644 index d1b4cf9de..000000000 --- a/tests/integration/commands/test_query_workers.py +++ /dev/null @@ -1,268 +0,0 @@ -""" -Tests for the `merlin query-workers` command. -""" - -import os -from enum import Enum - -from tests.integration.commands.base_classes import BaseStopWorkersAndQueryWorkersTest -from tests.integration.conditions import HasRegex - - -class WorkerMessages(Enum): - """ - Enumerated strings to help keep track of the messages - that we're expecting (or not expecting) to see from the - tests in this module. - """ - - NO_WORKERS_MSG = "No workers found!" - STEP_1_WORKER = "step_1_merlin_test_worker" - STEP_2_WORKER = "step_2_merlin_test_worker" - OTHER_WORKER = "other_merlin_test_worker" - - -class TestQueryWorkers(BaseStopWorkersAndQueryWorkersTest): - """ - Tests for the `merlin query-workers` command. Most of these tests will: - 1. Start workers from a spec file used for testing - - Use CeleryWorkerManager for this to ensure safe stoppage of workers - if something goes wrong - 2. Run the `merlin query-workers` command from a subprocess - """ - - command_to_test = "merlin query-workers" - - def test_no_workers( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - merlin_server_dir: str, - ): - """ - Test the `merlin query-workers` command with no workers started in the first place. - - Run the `merlin query-workers` command and ensure that a "no workers found" message - is written to the output. To see more information on exactly what this test is doing, - see the `run_test_without_workers()` method of the base class. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value), # No workers should be launched so we should see this - HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), # None of these workers should be started - HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # None of these workers should be started - HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # None of these workers should be started - ] - self.run_test_without_workers(merlin_server_dir, conditions, self.command_to_test) - - def test_no_flags( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin query-workers` command with no flags. - - Run the `merlin query-workers` command and ensure that all workers are queried. - To see more information on exactly what this test is doing, see the - `run_test_with_workers()` method. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be found - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be queried - ] - with self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test): - pass - - def test_spec_flag( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin query-workers` command with the `--spec` flag. - - Run the `merlin query-workers` command with the `--spec` flag and ensure that all - workers are queried. To see more information on exactly what this test is doing, - see the `run_test_with_workers()` method. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be queried - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be queried - ] - with self.run_test_with_workers( - path_to_test_specs, - merlin_server_dir, - conditions, - self.command_to_test, - flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", - ): - pass - - def test_workers_flag( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin query-workers` command with the `--workers` flag. - - Run the `merlin query-workers` command with the `--workers` flag and ensure that - only the workers given with this flag are queried. To see more information on - exactly what this test is doing, see the `run_test_with_workers()` method. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be queried - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be queried - ] - with self.run_test_with_workers( - path_to_test_specs, - merlin_server_dir, - conditions, - self.command_to_test, - flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", - ): - pass - - def test_queues_flag( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin query-workers` command with the `--queues` flag. - - Run the `merlin query-workers` command with the `--queues` flag and ensure that - only the workers attached to the given queues are queried. To see more information - on exactly what this test is doing, see the `run_test_with_workers()` method. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # One worker should be queried - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be queried - HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # This worker should NOT be queried - HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be queried - ] - with self.run_test_with_workers( - path_to_test_specs, - merlin_server_dir, - conditions, - self.command_to_test, - flag="--queues hello_queue", - ): - pass diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py new file mode 100644 index 000000000..5d86e8d5e --- /dev/null +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -0,0 +1,442 @@ +""" +This module will contain the base class used for testing +the `stop-workers` and `query-workers` commands. +""" + +import os +import subprocess +from contextlib import contextmanager +from enum import Enum +from typing import List + +import pytest + +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.integration.conditions import Condition, HasRegex +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec + + +# pylint: disable=unused-argument,import-outside-toplevel,too-many-arguments + + +class WorkerMessages(Enum): + """ + Enumerated strings to help keep track of the messages + that we're expecting (or not expecting) to see from the + tests in this module. + """ + + NO_WORKERS_MSG_STOP = "No workers found to stop" + NO_WORKERS_MSG_QUERY = "No workers found!" + STEP_1_WORKER = "step_1_merlin_test_worker" + STEP_2_WORKER = "step_2_merlin_test_worker" + OTHER_WORKER = "other_merlin_test_worker" + + +class TestStopAndQueryWorkersCommands: + """ + Tests for the `merlin stop-workers` and `merlin query-workers` commands. + Most of these tests will: + 1. Start workers from a spec file used for testing + - Use CeleryWorkerManager for this to ensure safe stoppage of workers + if something goes wrong + 2. Run the test command from a subprocess + """ + + @contextmanager + def run_test_with_workers( + self, + path_to_test_specs: str, + merlin_server_dir: str, + conditions: List[Condition], + command: str, + flag: str = None, + ): + """ + Helper method to run common testing logic for tests with workers started. + This method must also be a context manager so we can check the status of the + workers prior to the CeleryWorkersManager running it's exit code that shuts down + all active workers. + + This method will: + 0. Read in the necessary fixtures as parameters. These fixtures grab paths to + our test specs and the merlin server directory created from starting the + containerized redis server. + 1. Load in the worker specifications from the `multiple_workers.yaml` file. + 2. Use a context manager to start up the workers on the celery app connected to + the containerized redis server + 3. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 4. Run the test command that's provided and check that the conditions given are + passing. + 5. Yield control back to the calling method. + 6. Safely terminate workers that may have not been stopped once the calling method + completes. + + Parameters: + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + conditions: + A list of `Condition` instances that need to pass in order for this test to + be successful. + command: + The command that we're testing. E.g. "merlin stop-workers" + flag: + An optional flag to add to the command that we're testing so we can test + different functionality for the command. + """ + from merlin.celery import app as celery_app + + # Grab worker configurations from the spec file + multiple_worker_spec = os.path.join(path_to_test_specs, "multiple_workers.yaml") + workers_from_spec = load_workers_from_spec(multiple_worker_spec) + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as workers_manager: + workers_manager.launch_workers(workers_from_spec) + + # Copy the app.yaml to the cwd so merlin will connect to the testing server + copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + cmd_to_test = f"{command} {flag}" if flag else command + result = subprocess.run(cmd_to_test, capture_output=True, text=True, shell=True) + + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + check_test_conditions(conditions, info) + + yield + + def get_no_workers_msg(self, command_to_test: str) -> WorkerMessages: + """ + Retrieve the appropriate "no workers" found message. + + This method checks the command to test and returns a corresponding + message based on whether the command is to stop workers or query for them. + + Returns: + The message indicating that no workers are available, depending on the + command being tested. + """ + no_workers_msg = None + if command_to_test == "merlin stop-workers": + no_workers_msg = WorkerMessages.NO_WORKERS_MSG_STOP.value + else: + no_workers_msg = WorkerMessages.NO_WORKERS_MSG_QUERY.value + return no_workers_msg + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_no_workers( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + merlin_server_dir: str, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with no workers + started in the first place. + + This test will: + 0. Setup the pytest fixtures which include: + - starting a containerized Redis server + - updating the CONFIG object to point to the containerized Redis server + - obtaining the path to the merlin server directory created from starting + the containerized Redis server + 1. Copy the app.yaml file for the containerized redis server to the current working + directory so that merlin will connect to it when we run our test + 2. Run the test command that's provided and check that the conditions given are + passing. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test)), + HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), + ] + + # Copy the app.yaml to the cwd so merlin will connect to the testing server + copy_app_yaml_to_cwd(merlin_server_dir) + + # Run the test + result = subprocess.run(command_to_test, capture_output=True, text=True, shell=True) + info = { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + # Ensure all test conditions are satisfied + check_test_conditions(conditions, info) + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_no_flags( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with no flags. + + Run the commands referenced above and ensure the text output from Merlin is correct. + For the `stop-workers` command, we check if all workers are stopped as well. + To see more information on exactly what this test is doing, see the + `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value), + HasRegex(WorkerMessages.OTHER_WORKER.value), + ] + with self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, command_to_test): + if command_to_test == "merlin stop-workers": + # After the test runs and before the CeleryWorkersManager exits, ensure there are no workers on the app + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + assert active_queues is None + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_spec_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with the `--spec` + flag. + + Run the commands referenced above with the `--spec` flag and ensure the text output + from Merlin is correct. For the `stop-workers` command, we check if all workers defined + in the spec file are stopped as well. To see more information on exactly what this test + is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value), + HasRegex(WorkerMessages.OTHER_WORKER.value), + ] + with self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + command_to_test, + flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", + ): + if command_to_test == "merlin stop-workers": + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + assert active_queues is None + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_workers_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with the `--workers` + flag. + + Run the commands referenced above with the `--workers` flag and ensure the text output + from Merlin is correct. For the `stop-workers` command, we check to make sure that all + workers given with this flag are stopped. To see more information on exactly what this + test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value), + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), + ] + with self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + command_to_test, + flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", + ): + if command_to_test == "merlin stop-workers": + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + worker_name = f"celery@{WorkerMessages.OTHER_WORKER.value}" + assert worker_name in active_queues + + @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) + def test_queues_flag( + self, + redis_server: str, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_test_specs: str, + merlin_server_dir: str, + command_to_test: str, + ): + """ + Test the `merlin stop-workers` and `merlin query-workers` commands with the `--queues` + flag. + + Run the commands referenced above with the `--queues` flag and ensure the text output + from Merlin is correct. For the `stop-workers` command, we check that only the workers + attached to the given queues are stopped. To see more information on exactly what this + test is doing, see the `run_test_with_workers()` method. + + Parameters: + redis_server: + A fixture that starts a containerized redis server instance that runs on + localhost:6379. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_test_specs: + A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + command_to_test: + The command that we're testing, obtained from the parametrize call. + """ + conditions = [ + HasRegex(self.get_no_workers_msg(command_to_test), negate=True), + HasRegex(WorkerMessages.STEP_1_WORKER.value), + HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), + HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), + ] + with self.run_test_with_workers( + path_to_test_specs, + merlin_server_dir, + conditions, + command_to_test, + flag="--queues hello_queue", + ): + if command_to_test == "merlin stop-workers": + from merlin.celery import app as celery_app + + active_queues = celery_app.control.inspect().active_queues() + workers_that_should_be_alive = [ + f"celery@{WorkerMessages.OTHER_WORKER.value}", + f"celery@{WorkerMessages.STEP_2_WORKER.value}", + ] + for worker_name in workers_that_should_be_alive: + assert worker_name in active_queues + +# pylint: enable=unused-argument,import-outside-toplevel,too-many-arguments diff --git a/tests/integration/commands/test_stop_workers.py b/tests/integration/commands/test_stop_workers.py deleted file mode 100644 index c8b424f01..000000000 --- a/tests/integration/commands/test_stop_workers.py +++ /dev/null @@ -1,292 +0,0 @@ -""" -Tests for the `merlin stop-workers` command. -""" - -import os -from enum import Enum - -from tests.integration.commands.base_classes import BaseStopWorkersAndQueryWorkersTest -from tests.integration.conditions import HasRegex - - -class WorkerMessages(Enum): - """ - Enumerated strings to help keep track of the messages - that we're expecting (or not expecting) to see from the - tests in this module. - """ - - NO_WORKERS_MSG = "No workers found to stop" - STEP_1_WORKER = "step_1_merlin_test_worker" - STEP_2_WORKER = "step_2_merlin_test_worker" - OTHER_WORKER = "other_merlin_test_worker" - - -class TestStopWorkers(BaseStopWorkersAndQueryWorkersTest): - """ - Tests for the `merlin stop-workers` command. Most of these tests will: - 1. Start workers from a spec file used for testing - - Use CeleryWorkerManager for this to ensure safe stoppage of workers - if something goes wrong - 2. Run the `merlin stop-workers` command from a subprocess - """ - - command_to_test = "merlin stop-workers" - - def test_no_workers( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - merlin_server_dir: str, - ): - """ - Test the `merlin stop-workers` command with no workers started in the first place. - - Run the `merlin stop-workers` command and ensure that a "no workers found" message - is written to the output. To see more information on exactly what this test is doing, - see the `run_test_without_workers()` method of the base class. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value), # No workers should be launched so we should see this - HasRegex(WorkerMessages.STEP_1_WORKER.value, negate=True), # None of these workers should be started - HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # None of these workers should be started - HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # None of these workers should be started - ] - self.run_test_without_workers(merlin_server_dir, conditions, self.command_to_test) - - def test_no_flags( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin stop-workers` command with no flags. - - Run the `merlin stop-workers` command and ensure that all workers are stopped. - To see more information on exactly what this test is doing, see the - `run_test_with_workers()` method of the base class. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - from merlin.celery import app as celery_app - - # Define test conditions - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped - ] - - # Run the test - with self.run_test_with_workers(path_to_test_specs, merlin_server_dir, conditions, self.command_to_test): - # After the test runs and before the CeleryWorkersManager exits, ensure there are no workers on the app - active_queues = celery_app.control.inspect().active_queues() - assert active_queues is None - - def test_spec_flag( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin stop-workers` command with the `--spec` flag. - - Run the `merlin stop-workers` command with the `--spec` flag and ensure that all - workers are stopped. To see more information on exactly what this test is doing, - see the `run_test_with_workers()` method of the base class. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - from merlin.celery import app as celery_app - - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.OTHER_WORKER.value), # This worker should be stopped - ] - with self.run_test_with_workers( - path_to_test_specs, - merlin_server_dir, - conditions, - self.command_to_test, - flag=f"--spec {os.path.join(path_to_test_specs, 'multiple_workers.yaml')}", - ): - active_queues = celery_app.control.inspect().active_queues() - assert active_queues is None - - def test_workers_flag( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin stop-workers` command with the `--workers` flag. - - Run the `merlin stop-workers` command with the `--workers` flag and ensure that - only the workers given with this flag are stopped. To see more information on - exactly what this test is doing, see the `run_test_with_workers()` method of the - base class. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - from merlin.celery import app as celery_app - - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # Some workers should be stopped - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.STEP_2_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped - ] - with self.run_test_with_workers( - path_to_test_specs, - merlin_server_dir, - conditions, - self.command_to_test, - flag=f"--workers {WorkerMessages.STEP_1_WORKER.value} {WorkerMessages.STEP_2_WORKER.value}", - ): - active_queues = celery_app.control.inspect().active_queues() - worker_name = f"celery@{WorkerMessages.OTHER_WORKER.value}" - assert worker_name in active_queues - - def test_queues_flag( - self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, - ): - """ - Test the `merlin stop-workers` command with the `--queues` flag. - - Run the `merlin stop-workers` command with the `--queues` flag and ensure that - only the workers attached to the given queues are stopped. To see more information - on exactly what this test is doing, see the `run_test_with_workers()` method of the - base class. - - Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - path_to_test_specs: - A fixture to provide the path to the directory containing test specifications. - merlin_server_dir: - A fixture to provide the path to the merlin_server directory that will be - created by the `redis_server` fixture. - """ - from merlin.celery import app as celery_app - - conditions = [ - HasRegex(WorkerMessages.NO_WORKERS_MSG.value, negate=True), # One workers should be stopped - HasRegex(WorkerMessages.STEP_1_WORKER.value), # This worker should be stopped - HasRegex(WorkerMessages.STEP_2_WORKER.value, negate=True), # This worker should NOT be stopped - HasRegex(WorkerMessages.OTHER_WORKER.value, negate=True), # This worker should NOT be stopped - ] - with self.run_test_with_workers( - path_to_test_specs, - merlin_server_dir, - conditions, - self.command_to_test, - flag="--queues hello_queue", - ): - active_queues = celery_app.control.inspect().active_queues() - workers_that_should_be_alive = [ - f"celery@{WorkerMessages.OTHER_WORKER.value}", - f"celery@{WorkerMessages.STEP_2_WORKER.value}", - ] - for worker_name in workers_that_should_be_alive: - assert worker_name in active_queues diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py index ac3d5d7de..fc976a68e 100644 --- a/tests/integration/helper_funcs.py +++ b/tests/integration/helper_funcs.py @@ -102,11 +102,11 @@ def check_test_conditions(conditions: List[Condition], info: Dict[str, str]): condition.ingest_info(info) try: assert condition.passes - except AssertionError: + except AssertionError as exc: error_message = ( f"Condition failed: {condition}\n" f"Captured stdout: {info['stdout']}\n" f"Captured stderr: {info['stderr']}\n" f"Return code: {info['return_code']}\n" ) - raise AssertionError(error_message) + raise AssertionError(error_message) from exc From 19c4bf7862c57b210eac55782fd68b7a30594a24 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 14:44:06 -0700 Subject: [PATCH 162/201] run fix-style --- tests/integration/commands/test_stop_and_query_workers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py index 5d86e8d5e..00215137d 100644 --- a/tests/integration/commands/test_stop_and_query_workers.py +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -439,4 +439,5 @@ def test_queues_flag( for worker_name in workers_that_should_be_alive: assert worker_name in active_queues + # pylint: enable=unused-argument,import-outside-toplevel,too-many-arguments From 99257d8a2bb598cc446de7b62ea460ae786c1ccd Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 14:52:46 -0700 Subject: [PATCH 163/201] go back to original cache setup --- .github/workflows/push-pr_workflow.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 59404df27..9085bdab6 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -101,6 +101,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Check cache + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Python dependencies # uses: actions/cache@v2 # with: @@ -172,6 +178,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Check cache + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Python dependencies # uses: actions/cache@v2 # with: @@ -256,6 +268,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Check cache + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + # - name: Cache Python dependencies # uses: actions/cache@v2 # with: From f947eaecc59411653225162e373ea81693797432 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 15:17:25 -0700 Subject: [PATCH 164/201] try new cache for singularity install --- .github/workflows/push-pr_workflow.yml | 61 +++++++++++--------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 9085bdab6..61d1e1f13 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -107,25 +107,19 @@ jobs: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - # - name: Cache Python dependencies - # uses: actions/cache@v2 - # with: - # path: ~/.cache/pip - # key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - name: Install dependencies run: | python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - # - name: Cache Singularity build artifacts - # uses: actions/cache@v2 - # with: - # path: | - # ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - # /usr/local/go - # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} + - name: Cache Singularity build artifacts + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/bin/singularity + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} - name: Install singularity run: | @@ -184,24 +178,24 @@ jobs: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - # - name: Cache Python dependencies - # uses: actions/cache@v2 - # with: - # path: ~/.cache/pip - # key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - name: Install dependencies run: | python3 -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt + - name: Cache Singularity build artifacts + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/bin/singularity + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} + # - name: Cache Singularity build artifacts # uses: actions/cache@v2 # with: - # path: | - # ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - # /usr/local/go + # path: ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity @@ -243,11 +237,6 @@ jobs: run: | python3 tests/integration/run_tests.py --verbose --local - # TODO: - # - create common-setup job again but with correct caches - # - make local/integration tests depend on common-setup and check caches (see if this works) - # - see if local tests even need singularity - # - likely will for now but won't once server tests are ported to pytest Integration-tests: needs: Common-setup runs-on: ubuntu-latest @@ -274,12 +263,6 @@ jobs: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - # - name: Cache Python dependencies - # uses: actions/cache@v2 - # with: - # path: ~/.cache/pip - # key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - name: Install dependencies run: | python3 -m pip install --upgrade pip @@ -291,12 +274,18 @@ jobs: pip3 install -e . merlin config + - name: Cache Singularity build artifacts + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/bin/singularity + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} + # - name: Cache Singularity build artifacts # uses: actions/cache@v2 # with: - # path: | - # ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - # /usr/local/go + # path: ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - name: Install singularity From beafb2265cd5a7a49fd8849fe7863b0131f8da24 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 15:19:27 -0700 Subject: [PATCH 165/201] fix syntax issue in github workflow --- .github/workflows/push-pr_workflow.yml | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 61d1e1f13..4a0163951 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -114,12 +114,12 @@ jobs: pip3 install -r requirements/dev.txt - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/bin/singularity - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/bin/singularity + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} - name: Install singularity run: | @@ -185,12 +185,12 @@ jobs: pip3 install -r requirements/dev.txt - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/bin/singularity - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/bin/singularity + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} # - name: Cache Singularity build artifacts # uses: actions/cache@v2 @@ -275,12 +275,12 @@ jobs: merlin config - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/bin/singularity - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} + uses: actions/cache@v2 + with: + path: | + ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION + /usr/local/bin/singularity + key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} # - name: Cache Singularity build artifacts # uses: actions/cache@v2 From fac28928deae3b012923d6f09e504bf7f4d1dc71 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 15:43:25 -0700 Subject: [PATCH 166/201] attempt to fix singularity cache --- .github/workflows/push-pr_workflow.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 4a0163951..c44079013 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -119,10 +119,15 @@ jobs: path: | ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION /usr/local/bin/singularity + /usr/local/libexec/singularity + /usr/local/etc/singularity key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} - name: Install singularity run: | + ls /usr/local/bin/ + ls /usr/local/libexec/ + ls /usr/local/etc/ sudo apt-get update && sudo apt-get install -y \ build-essential \ libssl-dev \ @@ -190,6 +195,8 @@ jobs: path: | ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION /usr/local/bin/singularity + /usr/local/libexec/singularity + /usr/local/etc/singularity key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} # - name: Cache Singularity build artifacts @@ -200,6 +207,9 @@ jobs: - name: Install singularity run: | + ls /usr/local/bin/ + ls /usr/local/libexec/ + ls /usr/local/etc/ sudo apt-get update && sudo apt-get install -y \ build-essential \ libssl-dev \ @@ -280,6 +290,8 @@ jobs: path: | ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION /usr/local/bin/singularity + /usr/local/libexec/singularity + /usr/local/etc/singularity key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} # - name: Cache Singularity build artifacts @@ -290,6 +302,9 @@ jobs: - name: Install singularity run: | + ls /usr/local/bin/ + ls /usr/local/libexec/ + ls /usr/local/etc/ sudo apt-get update && sudo apt-get install -y \ build-essential \ libssl-dev \ From c49660a64ff5b389055f42381176435a92292604 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 15:46:02 -0700 Subject: [PATCH 167/201] remove ls statement that breaks workflow --- .github/workflows/push-pr_workflow.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index c44079013..df06691d7 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -126,7 +126,6 @@ jobs: - name: Install singularity run: | ls /usr/local/bin/ - ls /usr/local/libexec/ ls /usr/local/etc/ sudo apt-get update && sudo apt-get install -y \ build-essential \ From ecb17622c5b13f31a797cde7aeba17e10e2ce5ce Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 15:54:36 -0700 Subject: [PATCH 168/201] revert back to no common setup --- .github/workflows/push-pr_workflow.yml | 106 ------------------------- 1 file changed, 106 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index df06691d7..b9b5d9ca1 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -82,80 +82,6 @@ jobs: python3 -m pylint merlin --rcfile=setup.cfg --exit-zero python3 -m pylint tests --rcfile=setup.cfg --exit-zero - Common-setup: - runs-on: ubuntu-latest - env: - GO_VERSION: 1.18.1 - SINGULARITY_VERSION: 3.9.9 - OS: linux - ARCH: amd64 - - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Check cache - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip3 install -r requirements/dev.txt - - - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/bin/singularity - /usr/local/libexec/singularity - /usr/local/etc/singularity - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} - - - name: Install singularity - run: | - ls /usr/local/bin/ - ls /usr/local/etc/ - sudo apt-get update && sudo apt-get install -y \ - build-essential \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config - wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz - sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz - rm go$GO_VERSION.$OS-$ARCH.tar.gz - export PATH=$PATH:/usr/local/go/bin - wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz - tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz - cd singularity-ce-$SINGULARITY_VERSION - ./mconfig && \ - make -C ./builddir && \ - sudo make -C ./builddir install - - - name: Install merlin to run unit tests - run: | - pip3 install -e . - merlin config - - - name: Install CLI task dependencies generated from the 'feature demo' workflow - run: | - merlin example feature_demo - pip3 install -r feature_demo/requirements.txt - Local-test-suite: needs: Common-setup runs-on: ubuntu-latest @@ -188,22 +114,6 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/bin/singularity - /usr/local/libexec/singularity - /usr/local/etc/singularity - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} - - # - name: Cache Singularity build artifacts - # uses: actions/cache@v2 - # with: - # path: ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - name: Install singularity run: | ls /usr/local/bin/ @@ -283,22 +193,6 @@ jobs: pip3 install -e . merlin config - - name: Cache Singularity build artifacts - uses: actions/cache@v2 - with: - path: | - ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - /usr/local/bin/singularity - /usr/local/libexec/singularity - /usr/local/etc/singularity - key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}-${{ env.SINGULARITY_VERSION }} - - # - name: Cache Singularity build artifacts - # uses: actions/cache@v2 - # with: - # path: ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION - # key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} - - name: Install singularity run: | ls /usr/local/bin/ From 39d09d6fb7390bd0846072e920eed8231b2dce03 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 16:02:09 -0700 Subject: [PATCH 169/201] remove unnecessary dependency --- .github/workflows/push-pr_workflow.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index b9b5d9ca1..f24d9e73a 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -83,7 +83,6 @@ jobs: python3 -m pylint tests --rcfile=setup.cfg --exit-zero Local-test-suite: - needs: Common-setup runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 @@ -116,9 +115,6 @@ jobs: - name: Install singularity run: | - ls /usr/local/bin/ - ls /usr/local/libexec/ - ls /usr/local/etc/ sudo apt-get update && sudo apt-get install -y \ build-essential \ libssl-dev \ @@ -157,7 +153,6 @@ jobs: python3 tests/integration/run_tests.py --verbose --local Integration-tests: - needs: Common-setup runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 @@ -195,9 +190,6 @@ jobs: - name: Install singularity run: | - ls /usr/local/bin/ - ls /usr/local/libexec/ - ls /usr/local/etc/ sudo apt-get update && sudo apt-get install -y \ build-essential \ libssl-dev \ From 5f4673b03637d9d1eaa933cd92807efbbdfd14f6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 16:16:31 -0700 Subject: [PATCH 170/201] update github actions versions to use latest --- .github/workflows/push-pr_workflow.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index f24d9e73a..be274b029 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -95,14 +95,14 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} @@ -165,14 +165,14 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} From 70e540fafab1cb7b0f3da78d4cf3bfe4f4de6c06 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 16:19:11 -0700 Subject: [PATCH 171/201] update action versions that didn't save --- .github/workflows/push-pr_workflow.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index be274b029..d5474da72 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 # Checkout the whole history, in case the target is way far behind @@ -40,14 +40,14 @@ jobs: MAX_COMPLEXITY: 15 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.x' - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} @@ -245,14 +245,14 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} From d2e85ecba67aa12c7190e4f2a78e59116baec47f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 30 Sep 2024 16:36:30 -0700 Subject: [PATCH 172/201] run fix-style --- merlin/managers/redis_connection.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py index 37ee149a4..50a63492c 100644 --- a/merlin/managers/redis_connection.py +++ b/merlin/managers/redis_connection.py @@ -92,9 +92,11 @@ def get_redis_connection(self) -> redis.Redis: # Add ssl settings if necessary if CONFIG.results_backend.name == "rediss": - redis_config.update({ - "ssl": True, - "ssl_cert_reqs": getattr(CONFIG.results_backend, "cert_reqs", "required"), - }) + redis_config.update( + { + "ssl": True, + "ssl_cert_reqs": getattr(CONFIG.results_backend, "cert_reqs", "required"), + } + ) return redis.Redis(**redis_config) From 9c4fca431a03a21701d36842a2400e85408459b8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 2 Oct 2024 17:53:35 -0700 Subject: [PATCH 173/201] move distributed test suite actions back to v2 --- .github/workflows/push-pr_workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index d5474da72..e03b939b1 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -245,14 +245,14 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v4 + uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} From 3a2cafa74330bc251ba03a51665ff1843c0e9a1c Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 15 Oct 2024 13:10:37 -0700 Subject: [PATCH 174/201] add 'merlin run' tests and port existing ones to pytest --- tests/conftest.py | 40 +- tests/context_managers/celery_task_manager.py | 126 +++++ tests/fixtures/run_command.py | 25 + tests/integration/commands/pgen.py | 37 ++ tests/integration/commands/test_run.py | 430 ++++++++++++++++++ .../commands/test_stop_and_query_workers.py | 2 +- 6 files changed, 656 insertions(+), 4 deletions(-) create mode 100644 tests/context_managers/celery_task_manager.py create mode 100644 tests/fixtures/run_command.py create mode 100644 tests/integration/commands/pgen.py create mode 100644 tests/integration/commands/test_run.py diff --git a/tests/conftest.py b/tests/conftest.py index 374b8d836..c79e29aa9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,7 @@ from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature +from redis import Redis from merlin.config.configfile import CONFIG from tests.constants import CERT_FILES, SERVER_PASS @@ -57,9 +58,10 @@ # Loading in Module Specific Fixtures # ####################################### - pytest_plugins = [ - fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) + fixture_file.replace("/", ".").replace(".py", "") + for fixture_file in glob("tests/fixtures/**/*.py", recursive=True) + if not fixture_file.endswith("__init__.py") ] @@ -112,7 +114,23 @@ def path_to_test_specs() -> str: The absolute path to the 'test_specs' directory. """ path_to_test_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))) - return os.path.join(path_to_test_dir, os.path.join("integration", "test_specs")) + return os.path.join(path_to_test_dir, "integration", "test_specs") + + +@pytest.fixture(scope="session") +def path_to_merlin_codebase() -> str: + """ + Fixture to provide the path to the directory containing the Merlin code. + + This fixture returns the absolute path to the 'merlin' directory at the + top level of this repository. It expands environment variables and user + home directory as necessary. + + Returns: + The absolute path to the 'merlin' directory. + """ + path_to_test_dir = os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))) + return os.path.join(path_to_test_dir, "..", "merlin") @pytest.fixture(scope="session") @@ -176,6 +194,22 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: # The server will be stopped once this context reaches the end of it's execution here +@pytest.fixture(scope="session") +def redis_client(redis_server: str) -> Redis: + """ + Fixture that provides a Redis client instance for the test session. + It connects to this client using the url created from the `redis_server` + fixture. + + Args: + redis_server: The redis server uri we'll use to connect to redis + + Returns: + An instance of the Redis client that can be used to interact + with the Redis server. + """ + return Redis.from_url(url=redis_server) + @pytest.fixture(scope="session") def celery_app(redis_server: str) -> Celery: """ diff --git a/tests/context_managers/celery_task_manager.py b/tests/context_managers/celery_task_manager.py new file mode 100644 index 000000000..bf6396070 --- /dev/null +++ b/tests/context_managers/celery_task_manager.py @@ -0,0 +1,126 @@ +""" +Module to define functionality for sending tasks to the server +and ensuring they're cleared from the server when the test finishes. +""" +from types import TracebackType +from typing import List, Type + +from celery import Celery +from celery.result import AsyncResult +from redis import Redis + + +class CeleryTaskManager: + """ + A context manager for managing Celery tasks. + + This class provides a way to send tasks to a Celery server and clean up + any tasks that were sent during its lifetime. It is designed to be used + as a context manager, ensuring that tasks are properly removed from the + server when the context is exited. + + Attributes: + celery_app: The Celery application instance. + redis_server: The Redis server connection string. + """ + + def __init__(self, app: Celery, redis_client: Redis): + self.celery_app: Celery = app + self.redis_client = redis_client + + def __enter__(self) -> "CeleryTaskManager": + """ + Enters the runtime context related to this object. + + Returns: + The current instance of the manager. + """ + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + Exits the runtime context and performs cleanup. + + This method removes any tasks currently in the server. + + Args: + exc_type: The exception type raised, if any. + exc_value: The exception instance raised, if any. + traceback: The traceback object, if an exception was raised. + """ + self.remove_tasks() + + def send_task(self, task_name: str, *args, **kwargs) -> AsyncResult: + """ + Sends a task to the Celery server. + + This method will be used for tests that don't call + `merlin run`, allowing for isolated test functionality. + + Args: + task_name: The name of the task to send to the server. + *args: Additional positional arguments to pass to the task. + **kwargs: Additional keyword arguments to pass to the task. + + Returns: + A Celery AsyncResult object containing information about the + task that was sent to the server. + """ + valid_kwargs = [ + 'add_to_parent', 'chain', 'chord', 'compression', 'connection', + 'countdown', 'eta', 'exchange', 'expires', 'group_id', + 'group_index', 'headers', 'ignore_result', 'link', 'link_error', + 'parent_id', 'priority', 'producer', 'publisher', 'queue', + 'replaced_task_nesting', 'reply_to', 'result_cls', 'retries', + 'retry', 'retry_policy', 'root_id', 'route_name', 'router', + 'routing_key', 'serializer', 'shadow', 'soft_time_limit', 'task_id', + 'task_type', 'time_limit' + ] + send_task_kwargs = {key: kwargs.pop(key) for key in valid_kwargs if key in kwargs} + + return self.celery_app.send_task(task_name, args=args, kwargs=kwargs, **send_task_kwargs) + + def remove_tasks(self): + """ + Removes tasks from the Celery server. + + Tasks are removed in two ways: + 1. By purging the Celery app queues, which will only purge tasks + sent with `send_task`. + 2. By deleting the remaining queues in the Redis server, which will + purge any tasks that weren't sent with `send_task` (e.g., tasks + sent with `merlin run`). + """ + # Purge the tasks + self.celery_app.control.purge() + + # Purge any remaining tasks directly through redis that may have been missed + queues = self.get_queue_list() + for queue in queues: + self.redis_client.delete(queue) + + def get_queue_list(self) -> List[str]: + """ + Builds a list of Celery queues that exist on the Redis server. + + Queries the Redis server for its keys and returns the keys + that represent the Celery queues. + + Returns: + A list of Celery queue names. + """ + cursor = 0 + queues = [] + while True: + # Get the 'merlin' queue if it exists + cursor, matching_queues = self.redis_client.scan(cursor=cursor, match="merlin") + queues.extend(matching_queues) + + # Get any queues that start with '[merlin]' + cursor, matching_queues = self.redis_client.scan(cursor=cursor, match="\[merlin\]*") + queues.extend(matching_queues) + + if cursor == 0: + break + + return queues diff --git a/tests/fixtures/run_command.py b/tests/fixtures/run_command.py new file mode 100644 index 000000000..959d864d3 --- /dev/null +++ b/tests/fixtures/run_command.py @@ -0,0 +1,25 @@ +""" +Fixtures specifically for help testing the `merlin run` command. +""" +import os + +import pytest + + +@pytest.fixture(scope="session") +def run_command_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to testing the + `merlin run` functionality. + + Args: + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run + + Returns: + The path to the temporary testing directory for `merlin run` tests + """ + testing_dir = f"{temp_output_dir}/run_command_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir diff --git a/tests/integration/commands/pgen.py b/tests/integration/commands/pgen.py new file mode 100644 index 000000000..aa846b79b --- /dev/null +++ b/tests/integration/commands/pgen.py @@ -0,0 +1,37 @@ +""" +This file contains pgen functionality for testing purposes. +It's specifically set up to work with the feature demo example. +""" +import random +import itertools as iter + +from maestrowf.datastructures.core import ParameterGenerator + +def get_custom_generator(env, **kwargs): + p_gen = ParameterGenerator() + + # Unpack any pargs passed in + x2_min = int(kwargs.get('X2_MIN', '0')) + x2_max = int(kwargs.get('X2_MAX', '1')) + n_name_min = int(kwargs.get('N_NAME_MIN', '0')) + n_name_max = int(kwargs.get('N_NAME_MAX', '10')) + + # We'll only have two parameter entries each just for testing + num_points = 2 + + params = { + "X2": { + "values": [random.uniform(x2_min, x2_max) for _ in range(num_points)], + "label": "X2.%%" + }, + "N_NEW": { + "values": [random.randint(n_name_min, n_name_max) for _ in range(num_points)], + "label": "N_NEW.%%" + } + } + + for key, value in params.items(): + p_gen.add_parameter(key, value["values"], value["label"]) + + return p_gen + diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py new file mode 100644 index 000000000..cf22d2995 --- /dev/null +++ b/tests/integration/commands/test_run.py @@ -0,0 +1,430 @@ +""" +This module will contain the testing logic +for the `merlin run` command. +""" +import csv +import os +import re +import subprocess +from typing import Tuple + +from redis import Redis + +from merlin.spec.specification import MerlinSpec +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd + + +class TestRunCommand: + """ + Base class for testing the `merlin run` command. + """ + + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + + def setup_test_environment(self, merlin_server_dir: str, run_command_testing_dir: str): + """ + Setup the test environment for these tests by: + 1. Moving into the temporary output directory created specifically for these tests. + 2. Copying the app.yaml file created by the `redis_server` fixture to the cwd so that + Merlin can connect to the test server. + + Args: + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + os.chdir(run_command_testing_dir) + copy_app_yaml_to_cwd(merlin_server_dir) + + def run_merlin_command(self, command: str) -> subprocess.CompletedProcess: + """ + Open a subprocess and run the command specified by the `command` parameter. + Ensure this command runs successfully and return the process results. + + Args: + command: The command to execute in a subprocess. + + Returns: + The results from executing the command in a subprocess. + + Raises: + AssertionError: If the command fails (non-zero return code). + """ + result = subprocess.run(command, shell=True, capture_output=True, text=True) + assert result.returncode == 0, f"Command failed with return code {result.returncode}. Output: {result.stdout} Error: {result.stderr}" + return result + + def get_output_workspace_from_logs(self, stdout_logs: str, stderr_logs: str) -> str: + """ + Extracts the workspace path from the provided standard output and error logs. + + This method searches for a specific message indicating the study workspace + in the combined logs (both stdout and stderr). The expected message format + is: "Study workspace is ''". If the message is found, + the method returns the extracted workspace path. If the message is not + found, an assertion error is raised. + + Args: + stdout_logs: The standard output logs as a string. + stderr_logs: The standard error logs as a string. + + Returns: + The extracted workspace path from the logs. + + Raises: + AssertionError: If the expected message is not found in the combined logs. + """ + workspace_pattern = re.compile(r"Study workspace is '(\S+)'") + combined_output = stdout_logs + stderr_logs + match = workspace_pattern.search(combined_output) + assert match, "No 'Study workspace is...' message found in command output." + return match.group(1) + + def validate_workspace(self, result: subprocess.CompletedProcess) -> str: + """ + Validate the workspace path extracted from the command output logs. + + This method retrieves the expected workspace path from the standard output and error logs + of the executed command. It checks if the workspace path exists in the filesystem. + + Args: + result: The result of the executed command, containing stdout and stderr. + + Returns: + The path to the expected workspace. + + Raises: + AssertionError: If the expected workspace path is not found in the filesystem. + """ + expected_workspace_path = self.get_output_workspace_from_logs(result.stdout, result.stderr) + assert os.path.exists(expected_workspace_path), f"Expected workspace not found: {expected_workspace_path}" + return expected_workspace_path + + +class TestRunCommandDistributed(TestRunCommand): + """ + Tests for the `merlin run` command that are run in a distributed manner + rather than being run locally. + """ + + def test_distributed_run( + self, + redis_client: Redis, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_merlin_codebase: str, + merlin_server_dir: str, + run_command_testing_dir: str, + ): + """ + This test verifies that tasks can be successfully sent to a Redis server + using the `merlin run` command with no flags. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + from merlin.celery import app as celery_app + + # Setup the testing environment + feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) + self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server + self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_distributed_run") + + # Get the queues we need to query + spec = MerlinSpec.load_specification(feature_demo) + queues_in_spec = spec.get_task_queues() + + for queue in queues_in_spec.values(): + # Brackets are special chars in regex so we have to add \ to make them literal + queue = queue.replace("[", "\\[").replace("]", "\\]") + matching_queues_on_server = redis_client.keys(pattern=f"{queue}*") + + # Make sure any queues that exist on the server have tasks in them + for matching_queue in matching_queues_on_server: + tasks = redis_client.lrange(matching_queue, 0, -1) + assert len(tasks) > 0 + + def test_samplesfile_option( + self, + redis_client: Redis, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_merlin_codebase: str, + merlin_server_dir: str, + run_command_testing_dir: str, + ): + """ + This test verifies that passing in a samples filepath from the command line will + substitute in the file properly. It should copy the samples file that's passed + in to the merlin_info subdirectory. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + from merlin.celery import app as celery_app + + # Setup the testing environment + feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) + self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + + # Create a new samples file to pass into our test workflow + data = [ + ["X1, Value 1", "X2, Value 1"], + ["X1, Value 2", "X2, Value 2"], + ["X1, Value 3", "X2, Value 3"], + ] + sample_filename = "test_samplesfile.csv" + new_samples_file = os.path.join(run_command_testing_dir, sample_filename) + with open(new_samples_file, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerows(data) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server + result = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_samplesfile_option --samplesfile {new_samples_file}") + + # Check that the new samples file is written to the merlin_info directory + expected_workspace_path = self.validate_workspace(result) + assert os.path.exists(os.path.join(expected_workspace_path, "merlin_info", sample_filename)) + + def test_pgen_and_pargs_options( + self, + redis_client: Redis, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_merlin_codebase: str, + merlin_server_dir: str, + run_command_testing_dir: str, + ): + """ + Test the `--pgen` and `--pargs` options with the `merlin run` command. + This should update the parameter block of the expanded yaml file to have + 2 entries for both `X2` and `N_NEW`. The `X2` parameter should be between + `X2_MIN` and `X2_MAX`, and the `N_NEW` parameter should be between `N_NEW_MIN` + and `N_NEW_MAX`. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + from merlin.celery import app as celery_app + + # Setup test vars and the testing environment + feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) + new_x2_min, new_x2_max = 1, 2 + new_n_new_min, new_n_new_max = 5, 15 + pgen_filepath = os.path.join(os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))), "pgen.py") + self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server + result = self.run_merlin_command( + f'merlin run {feature_demo} --vars NAME=run_command_test_pgen_and_pargs_options --pgen {pgen_filepath} --parg "X2_MIN:{new_x2_min}" --parg "X2_MAX:{new_x2_max}" --parg "N_NAME_MIN:{new_n_new_min}" --parg "N_NAME_MAX:{new_n_new_max}"' + ) + + # Check that an expanded yaml file exists + expected_workspace_path = self.validate_workspace(result) + merlin_info_dir = os.path.join(expected_workspace_path, "merlin_info") + expanded_yaml = os.path.join(merlin_info_dir, "feature_demo.expanded.yaml") + assert os.path.exists(expanded_yaml), f"Expected YAML file not found: {expanded_yaml}" + + # Read in the parameters from the expanded yaml and ensure they're within the new bounds we provided + expanded_spec = MerlinSpec.load_specification(expanded_yaml) + params = expanded_spec.get_parameters() + for x2_param in params.parameters["X2"]: + assert new_x2_min <= x2_param <= new_x2_max + for n_new_param in params.parameters["N_NEW"]: + assert new_n_new_min <= n_new_param <= new_n_new_max + + +class TestRunCommandLocal(TestRunCommand): + """ + Tests for the `merlin run` command that are run in a locally rather + than in a distributed manner. + """ + + def test_dry_run( + self, + redis_client: Redis, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_merlin_codebase: str, + merlin_server_dir: str, + run_command_testing_dir: str, + ): + """ + Test the `merlin run` command's `--dry` option. This should create all the output + subdirectories for each step but it shouldn't execute anything for the steps. In + other words, the only file in each step subdirectory should be the .sh file. + + Note: + This test will run locally so that we don't have to worry about starting + & stopping workers. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + # Setup the test environment + feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) + self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + + # Run the test and grab the output workspace generated from it + result = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_dry_run --local --dry") + expected_workspace_path = self.validate_workspace(result) + + # Check that every step was ran by looking for an existing output workspace + spec = MerlinSpec.load_specification(feature_demo) + for step in spec.get_study_steps(): + step_directory = os.path.join(expected_workspace_path, step.name) + assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" + + allowed_dry_run_files = {"MERLIN_STATUS.json", "status.lock"} + for dirpath, dirnames, filenames in os.walk(step_directory): + # Check if the current directory has no subdirectories (leaf directory) + if not dirnames: + # Check for unexpected files + unexpected_files = [file for file in filenames if file not in allowed_dry_run_files and not file.endswith(".sh")] + assert not unexpected_files, f"Unexpected files found in {dirpath}: {unexpected_files}. Expected only .sh files or {allowed_dry_run_files}." + + # Check that there is exactly one .sh file + sh_file_count = sum(1 for file in filenames if file.endswith(".sh")) + assert sh_file_count == 1, f"Expected exactly one .sh file in {dirpath} but found {sh_file_count} .sh files." + + def test_local_run( + self, + redis_client: Redis, + redis_results_backend_config: "Fixture", # noqa: F821 + redis_broker_config: "Fixture", # noqa: F821 + path_to_merlin_codebase: str, + merlin_server_dir: str, + run_command_testing_dir: str, + ): + """ + This test verifies that tasks can be successfully executed locally using + the `merlin run` command with the `--local` flag. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + run_command_testing_dir: + The path to the the temp output directory for `merlin run` tests. + """ + # Setup the test environment + feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) + self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + + # Run the test and grab the output workspace generated from it + result = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_local_run --local") + expected_workspace_path = self.validate_workspace(result) + + # Check that every step was ran by looking for an existing output workspace and MERLIN_FINISHED files + spec = MerlinSpec.load_specification(feature_demo) + for step in spec.get_study_steps(): + step_directory = os.path.join(expected_workspace_path, step.name) + assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" + for dirpath, dirnames, filenames in os.walk(step_directory): + # Check if the current directory has no subdirectories (leaf directory) + if not dirnames: + # Check for the existence of the MERLIN_FINISHED file + assert "MERLIN_FINISHED" in filenames, f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" + +# TODO +# - commit these changes +# - do something similar for `merlin purge` tests \ No newline at end of file diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py index 00215137d..853158470 100644 --- a/tests/integration/commands/test_stop_and_query_workers.py +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -1,5 +1,5 @@ """ -This module will contain the base class used for testing +This module will contain the testing logic for the `stop-workers` and `query-workers` commands. """ From 15d66654d1dee6dd89e26692d2e03bc166189747 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 15 Oct 2024 13:11:10 -0700 Subject: [PATCH 175/201] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9799ee32..1eeff3df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `CeleryWorkersManager`: context to help with starting/stopping workers for tests - Ability to copy and print the `Config` object from `merlin/config/__init__.py` - Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py` +- Added additional tests for the `merlin run` command ### Changed - Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier - Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier - Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot` - Moved stop-workers and query-workers integration tests to pytest tests +- Ported `run and purge feature demo` test to pytest ## [1.12.2b1] ### Added From 5844e914e561a8e91544fe3e8ff8012808af3a52 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 15 Oct 2024 14:55:39 -0700 Subject: [PATCH 176/201] add aliased fixture types for typehinting --- tests/conftest.py | 50 ++++++++------ tests/fixture_types.py | 39 +++++++++++ tests/fixtures/examples.py | 4 +- tests/fixtures/run_command.py | 4 +- tests/fixtures/server.py | 48 ++++++------- tests/fixtures/status.py | 13 ++-- tests/integration/commands/test_run.py | 67 +++++++++---------- .../commands/test_stop_and_query_workers.py | 53 ++++++++------- 8 files changed, 166 insertions(+), 112 deletions(-) create mode 100644 tests/fixture_types.py diff --git a/tests/conftest.py b/tests/conftest.py index c79e29aa9..22da1a061 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,6 +48,16 @@ from tests.constants import CERT_FILES, SERVER_PASS from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager +from tests.fixture_types import ( + FixtureBytes, + FixtureCelery, + FixtureDict, + FixtureInt, + FixtureModification, + FixtureRedis, + FixtureSignature, + FixtureStr, +) from tests.utils import create_cert_files, create_pass_file @@ -102,7 +112,7 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi @pytest.fixture(scope="session") -def path_to_test_specs() -> str: +def path_to_test_specs() -> FixtureStr: """ Fixture to provide the path to the directory containing test specifications. @@ -118,7 +128,7 @@ def path_to_test_specs() -> str: @pytest.fixture(scope="session") -def path_to_merlin_codebase() -> str: +def path_to_merlin_codebase() -> FixtureStr: """ Fixture to provide the path to the directory containing the Merlin code. @@ -134,7 +144,7 @@ def path_to_merlin_codebase() -> str: @pytest.fixture(scope="session") -def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: +def temp_output_dir(tmp_path_factory: TempPathFactory) -> FixtureStr: """ This fixture will create a temporary directory to store output files of integration tests. The temporary directory will be stored at /tmp/`whoami`/pytest-of-`whoami`/. There can be at most @@ -157,7 +167,7 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str) -> str: +def merlin_server_dir(temp_output_dir: FixtureStr) -> FixtureStr: """ The path to the merlin_server directory that will be created by the `redis_server` fixture. @@ -171,7 +181,7 @@ def merlin_server_dir(temp_output_dir: str) -> str: @pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: +def redis_server(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureStr: """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. @@ -195,7 +205,7 @@ def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: @pytest.fixture(scope="session") -def redis_client(redis_server: str) -> Redis: +def redis_client(redis_server: FixtureStr) -> FixtureRedis: """ Fixture that provides a Redis client instance for the test session. It connects to this client using the url created from the `redis_server` @@ -211,7 +221,7 @@ def redis_client(redis_server: str) -> Redis: return Redis.from_url(url=redis_server) @pytest.fixture(scope="session") -def celery_app(redis_server: str) -> Celery: +def celery_app(redis_server: FixtureStr) -> FixtureCelery: """ Create the celery app to be used throughout our integration tests. @@ -223,7 +233,7 @@ def celery_app(redis_server: str) -> Celery: @pytest.fixture(scope="session") -def sleep_sig(celery_app: Celery) -> Signature: +def sleep_sig(celery_app: FixtureCelery) -> FixtureSignature: """ Create a task registered to our celery app and return a signature for it. Once requested by a test, you can set the queue you'd like to send this to @@ -245,7 +255,7 @@ def sleep_task(): @pytest.fixture(scope="session") -def worker_queue_map() -> Dict[str, str]: +def worker_queue_map() -> FixtureDict[str, str]: """ Worker and queue names to be used throughout tests @@ -255,7 +265,7 @@ def worker_queue_map() -> Dict[str, str]: @pytest.fixture(scope="class") -def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): +def launch_workers(celery_app: FixtureCelery, worker_queue_map: FixtureDict[str, str]): """ Launch the workers on the celery app fixture using the worker and queue names defined in the worker_queue_map fixture. @@ -273,7 +283,7 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): @pytest.fixture(scope="session") -def test_encryption_key() -> bytes: +def test_encryption_key() -> FixtureBytes: """ An encryption key to be used for tests that need it. @@ -298,7 +308,7 @@ def test_encryption_key() -> bytes: @pytest.fixture(scope="function") -def config(merlin_server_dir: str, test_encryption_key: bytes): +def config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: """ DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` @@ -350,8 +360,8 @@ def config(merlin_server_dir: str, test_encryption_key: bytes): @pytest.fixture(scope="function") def redis_broker_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): + merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis broker and results_backend. @@ -371,8 +381,8 @@ def redis_broker_config( @pytest.fixture(scope="function") def redis_results_backend_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): + merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a Redis results_backend. @@ -392,8 +402,8 @@ def redis_results_backend_config( @pytest.fixture(scope="function") def rabbit_broker_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): + merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a RabbitMQ broker. @@ -413,8 +423,8 @@ def rabbit_broker_config( @pytest.fixture(scope="function") def mysql_results_backend_config( - merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument -): + merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase that uses the CONFIG object with a MySQL results_backend. diff --git a/tests/fixture_types.py b/tests/fixture_types.py new file mode 100644 index 000000000..3b02188d2 --- /dev/null +++ b/tests/fixture_types.py @@ -0,0 +1,39 @@ +""" +It's hard to type hint pytest fixtures in a way that makes it clear +that the variable being used is a fixture. This module will created +aliases for these fixtures in order to make it easier to track what's +happening. + +The types here will be defined as such: +- `FixtureBytes`: A fixture that returns bytes +- `FixtureCelery`: A fixture that returns a Celery app object +- `FixtureDict`: A fixture that returns a dictionary +- `FixtureInt`: A fixture that returns an integer +- `FixtureModification`: A fixture that modifies something but never actually + returns/yields a value to be used in the test. +- `FixtureRedis`: A fixture that returns a Redis client +- `FixtureSignature`: A fixture that returns a Celery Signature object +- `FixtureStr`: A fixture that returns a string +""" +import pytest +from argparse import Namespace +from celery import Celery +from celery.canvas import Signature +from redis import Redis +from typing import Any, Annotated, Dict, TypeVar + +# TODO convert unit test type hinting to use these +# - likely will do this when I work on API docs for test library + +K = TypeVar('K') +V = TypeVar('V') + +FixtureBytes = Annotated[bytes, pytest.fixture] +FixtureCelery = Annotated[Celery, pytest.fixture] +FixtureDict = Annotated[Dict[K, V], pytest.fixture] +FixtureInt = Annotated[int, pytest.fixture] +FixtureModification = Annotated[Any, pytest.fixture] +FixtureNamespace = Annotated[Namespace, pytest.fixture] +FixtureRedis = Annotated[Redis, pytest.fixture] +FixtureSignature = Annotated[Signature, pytest.fixture] +FixtureStr = Annotated[str, pytest.fixture] diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py index 7c4626e3e..949234577 100644 --- a/tests/fixtures/examples.py +++ b/tests/fixtures/examples.py @@ -6,9 +6,11 @@ import pytest +from tests.fixture_types import FixtureStr + @pytest.fixture(scope="session") -def examples_testing_dir(temp_output_dir: str) -> str: +def examples_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to the examples functionality. diff --git a/tests/fixtures/run_command.py b/tests/fixtures/run_command.py index 959d864d3..bdf0ff13d 100644 --- a/tests/fixtures/run_command.py +++ b/tests/fixtures/run_command.py @@ -5,9 +5,11 @@ import pytest +from tests.fixture_types import FixtureStr + @pytest.fixture(scope="session") -def run_command_testing_dir(temp_output_dir: str) -> str: +def run_command_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to testing the `merlin run` functionality. diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index 4f4a07a2c..cd641aa30 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -9,12 +9,14 @@ import pytest import yaml +from tests.fixture_types import FixtureDict, FixtureNamespace, FixtureStr + # pylint: disable=redefined-outer-name @pytest.fixture(scope="session") -def server_testing_dir(temp_output_dir: str) -> str: +def server_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to the server functionality. @@ -29,7 +31,7 @@ def server_testing_dir(temp_output_dir: str) -> str: @pytest.fixture(scope="session") -def server_redis_conf_file(server_testing_dir: str) -> str: +def server_redis_conf_file(server_testing_dir: FixtureStr) -> FixtureStr: """ Fixture to write a redis.conf file to the temporary output directory. @@ -77,7 +79,7 @@ def server_redis_conf_file(server_testing_dir: str) -> str: @pytest.fixture(scope="session") -def server_redis_pass_file(server_testing_dir: str) -> str: +def server_redis_pass_file(server_testing_dir: FixtureStr) -> FixtureStr: """ Fixture to create a redis password file in the temporary output directory. @@ -96,7 +98,7 @@ def server_redis_pass_file(server_testing_dir: str) -> str: @pytest.fixture(scope="session") -def server_users() -> Dict[str, Dict[str, str]]: +def server_users() -> FixtureDict[str, Dict[str, str]]: """ Create a dictionary of two test users with identical configuration settings. @@ -122,7 +124,7 @@ def server_users() -> Dict[str, Dict[str, str]]: @pytest.fixture(scope="session") -def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: +def server_redis_users_file(server_testing_dir: FixtureStr, server_users: FixtureDict[str, Dict[str, str]]) -> FixtureStr: """ Fixture to write a redis.users file to the temporary output directory. @@ -143,11 +145,11 @@ def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: @pytest.fixture(scope="class") def server_container_config_data( - server_testing_dir: str, - server_redis_conf_file: str, - server_redis_pass_file: str, - server_redis_users_file: str, -) -> Dict[str, str]: + server_testing_dir: FixtureStr, + server_redis_conf_file: FixtureStr, + server_redis_pass_file: FixtureStr, + server_redis_users_file: FixtureStr, +) -> FixtureDict[str, str]: """ Fixture to provide sample data for ContainerConfig tests. @@ -172,7 +174,7 @@ def server_container_config_data( @pytest.fixture(scope="class") -def server_container_format_config_data() -> Dict[str, str]: +def server_container_format_config_data() -> FixtureDict[str, str]: """ Fixture to provide sample data for ContainerFormatConfig tests @@ -187,7 +189,7 @@ def server_container_format_config_data() -> Dict[str, str]: @pytest.fixture(scope="class") -def server_process_config_data() -> Dict[str, str]: +def server_process_config_data() -> FixtureDict[str, str]: """ Fixture to provide sample data for ProcessConfig tests @@ -201,10 +203,10 @@ def server_process_config_data() -> Dict[str, str]: @pytest.fixture(scope="class") def server_server_config( - server_container_config_data: Dict[str, str], - server_process_config_data: Dict[str, str], - server_container_format_config_data: Dict[str, str], -) -> Dict[str, Dict[str, str]]: + server_container_config_data: FixtureDict[str, str], + server_process_config_data: FixtureDict[str, str], + server_container_format_config_data: FixtureDict[str, str], +) -> FixtureDict[str, FixtureDict[str, str]]: """ Fixture to provide sample data for ServerConfig tests @@ -222,10 +224,10 @@ def server_server_config( @pytest.fixture(scope="function") def server_app_yaml_contents( - server_redis_pass_file: str, - server_container_config_data: Dict[str, str], - server_process_config_data: Dict[str, str], -) -> Dict[str, Union[str, int]]: + server_redis_pass_file: FixtureStr, + server_container_config_data: FixtureDict[str, str], + server_process_config_data: FixtureDict[str, str], +) -> FixtureDict[str, Union[str, int]]: """ Fixture to create the contents of an app.yaml file. @@ -260,7 +262,7 @@ def server_app_yaml_contents( @pytest.fixture(scope="function") -def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> str: +def server_app_yaml(server_testing_dir: FixtureStr, server_app_yaml_contents: FixtureDict[str, Union[str, int]]) -> FixtureStr: """ Fixture to create an app.yaml file in the temporary output directory. @@ -280,13 +282,13 @@ def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> @pytest.fixture(scope="function") -def server_process_file_contents() -> str: +def server_process_file_contents() -> FixtureDict[str, Union[str, int]]: """Fixture to represent process file contents.""" return {"parent_pid": 123, "image_pid": 456, "port": 6379, "hostname": "dummy_server"} @pytest.fixture(scope="function") -def server_config_server_args() -> Namespace: +def server_config_server_args() -> FixtureNamespace: """ Setup an argparse Namespace with all args that the `config_server` function will need. These can be modified on a test-by-test basis. diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index 39a36f9bf..166d27eb5 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -11,6 +11,7 @@ import pytest import yaml +from tests.fixture_types import FixtureNamespace, FixtureStr from tests.unit.study.status_test_files import status_test_variables @@ -18,7 +19,7 @@ @pytest.fixture(scope="session") -def status_testing_dir(temp_output_dir: str) -> str: +def status_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: """ A pytest fixture to set up a temporary directory to write files to for testing status. @@ -33,7 +34,7 @@ def status_testing_dir(temp_output_dir: str) -> str: @pytest.fixture(scope="class") -def status_empty_file(status_testing_dir: str) -> str: +def status_empty_file(status_testing_dir: FixtureStr) -> FixtureStr: """ A pytest fixture to create an empty status file. @@ -49,7 +50,7 @@ def status_empty_file(status_testing_dir: str) -> str: @pytest.fixture(scope="session") -def status_spec_path(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_spec_path(status_testing_dir: FixtureStr) -> FixtureStr: # pylint: disable=W0621 """ Copy the test spec to the temp directory and modify the OUTPUT_PATH in the spec to point to the temp location. @@ -94,7 +95,7 @@ def set_sample_path(output_workspace: str): @pytest.fixture(scope="session") -def status_output_workspace(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_output_workspace(status_testing_dir: FixtureStr) -> FixtureStr: # pylint: disable=W0621 """ A pytest fixture to copy the test output workspace for status to the temporary status testing directory. @@ -110,7 +111,7 @@ def status_output_workspace(status_testing_dir: str) -> str: # pylint: disable= @pytest.fixture(scope="function") -def status_args(): +def status_args() -> FixtureNamespace: """ A pytest fixture to set up a namespace with all the arguments necessary for the Status object. @@ -130,7 +131,7 @@ def status_args(): @pytest.fixture(scope="session") -def status_nested_workspace(status_testing_dir: str) -> str: # pylint: disable=W0621 +def status_nested_workspace(status_testing_dir: FixtureStr) -> FixtureStr: # pylint: disable=W0621 """ Create an output workspace that contains another output workspace within one of its steps. In this case it will copy the status test workspace then within the 'just_samples' diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index cf22d2995..e55c15e9d 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -12,6 +12,7 @@ from merlin.spec.specification import MerlinSpec from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd @@ -22,7 +23,7 @@ class TestRunCommand: demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") - def setup_test_environment(self, merlin_server_dir: str, run_command_testing_dir: str): + def setup_test_environment(self, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr): """ Setup the test environment for these tests by: 1. Moving into the temporary output directory created specifically for these tests. @@ -112,12 +113,12 @@ class TestRunCommandDistributed(TestRunCommand): def test_distributed_run( self, - redis_client: Redis, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_merlin_codebase: str, - merlin_server_dir: str, - run_command_testing_dir: str, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, ): """ This test verifies that tasks can be successfully sent to a Redis server @@ -171,12 +172,12 @@ def test_distributed_run( def test_samplesfile_option( self, - redis_client: Redis, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_merlin_codebase: str, - merlin_server_dir: str, - run_command_testing_dir: str, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, ): """ This test verifies that passing in a samples filepath from the command line will @@ -233,12 +234,12 @@ def test_samplesfile_option( def test_pgen_and_pargs_options( self, - redis_client: Redis, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_merlin_codebase: str, - merlin_server_dir: str, - run_command_testing_dir: str, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, ): """ Test the `--pgen` and `--pargs` options with the `merlin run` command. @@ -307,12 +308,12 @@ class TestRunCommandLocal(TestRunCommand): def test_dry_run( self, - redis_client: Redis, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_merlin_codebase: str, - merlin_server_dir: str, - run_command_testing_dir: str, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, ): """ Test the `merlin run` command's `--dry` option. This should create all the output @@ -373,12 +374,12 @@ def test_dry_run( def test_local_run( self, - redis_client: Redis, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_merlin_codebase: str, - merlin_server_dir: str, - run_command_testing_dir: str, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr, ): """ This test verifies that tasks can be successfully executed locally using @@ -424,7 +425,3 @@ def test_local_run( if not dirnames: # Check for the existence of the MERLIN_FINISHED file assert "MERLIN_FINISHED" in filenames, f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" - -# TODO -# - commit these changes -# - do something similar for `merlin purge` tests \ No newline at end of file diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py index 853158470..1144ff972 100644 --- a/tests/integration/commands/test_stop_and_query_workers.py +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -12,6 +12,7 @@ import pytest from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.fixture_types import FixtureModification, FixtureStr from tests.integration.conditions import Condition, HasRegex from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec @@ -46,8 +47,8 @@ class TestStopAndQueryWorkersCommands: @contextmanager def run_test_with_workers( self, - path_to_test_specs: str, - merlin_server_dir: str, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, conditions: List[Condition], command: str, flag: str = None, @@ -137,10 +138,10 @@ def get_no_workers_msg(self, command_to_test: str) -> WorkerMessages: @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_no_workers( self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - merlin_server_dir: str, + redis_server: FixtureStr, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + merlin_server_dir: FixtureStr, command_to_test: str, ): """ @@ -202,11 +203,11 @@ def test_no_workers( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_no_flags( self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, + redis_server: FixtureStr, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, command_to_test: str, ): """ @@ -256,11 +257,11 @@ def test_no_flags( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_spec_flag( self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, + redis_server: FixtureStr, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, command_to_test: str, ): """ @@ -316,11 +317,11 @@ def test_spec_flag( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_workers_flag( self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, + redis_server: FixtureStr, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, command_to_test: str, ): """ @@ -377,11 +378,11 @@ def test_workers_flag( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_queues_flag( self, - redis_server: str, - redis_results_backend_config: "Fixture", # noqa: F821 - redis_broker_config: "Fixture", # noqa: F821 - path_to_test_specs: str, - merlin_server_dir: str, + redis_server: FixtureStr, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, command_to_test: str, ): """ From 085d4b2916e57670ee735d2ff8134436c17566b7 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 16 Oct 2024 16:51:35 -0700 Subject: [PATCH 177/201] add tests for the purge command --- tests/integration/commands/test_purge.py | 422 +++++++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 tests/integration/commands/test_purge.py diff --git a/tests/integration/commands/test_purge.py b/tests/integration/commands/test_purge.py new file mode 100644 index 000000000..a3c6d8011 --- /dev/null +++ b/tests/integration/commands/test_purge.py @@ -0,0 +1,422 @@ +""" +This module will contain the testing logic +for the `merlin purge` command. +""" +import os +import subprocess +from typing import Dict, List, Tuple, Union + +from merlin.spec.expansion import get_spec_with_expansion +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr +from tests.integration.conditions import HasRegex, HasReturnCode +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd + + +class TestPurgeCommand: + """ + Tests for the `merlin purge` command. + """ + + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + + def setup_test(self, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr) -> str: + """ + Setup the test environment for these tests by: + 1. Copying the app.yaml file created by the `redis_server` fixture to the cwd so that + Merlin can connect to the test server. + 2. Obtaining the path to the feature_demo spec that we'll use for these tests. + + Args: + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + + Returns: + The path to the feature_demo spec file. + """ + copy_app_yaml_to_cwd(merlin_server_dir) + return os.path.join(path_to_merlin_codebase, self.demo_workflow) + + def setup_tasks(self, CTM: CeleryTaskManager, spec_file: str) -> Tuple[Dict[str, str], int]: + """ + Helper method to setup tasks in the specified queues. + + This method sends tasks named 'task_for_{queue}' to each queue defined in the + provided spec file and returns the total number of queues that received tasks. + + Args: + CTM: + A context manager for managing Celery tasks, used to send tasks to the server. + spec_file: + The path to the spec file from which queues will be extracted. + + Returns: + A tuple with: + - A dictionary where the keys are step names and values are their associated queues. + - The number of queues that received tasks + """ + spec = get_spec_with_expansion(spec_file) + queues_in_spec = spec.get_task_queues() + + for queue in queues_in_spec.values(): + CTM.send_task(f"task_for_{queue}", queue=queue) + + return queues_in_spec, len(queues_in_spec.values()) + + def run_purge( + self, + spec_file: str, + input_value: str = None, + force: bool = False, + steps_to_purge: List[str] = None, + ) -> Dict[str, Union[str, int]]: + """ + Helper method to run the purge command. + + Args: + spec_file: The path to the spec file from which queues will be purged. + input_value: Any input we need to send to the subprocess. + force: If True, add the `-f` option to the purge command. + steps_to_purge: An optional list of steps to send to the purge command. + + Returns: + The result from executing the command in a subprocess. + """ + purge_cmd = ( + "merlin purge" + (" -f" if force else "") + f" {spec_file}" + + (f" --steps {' '.join(steps_to_purge)}" if steps_to_purge is not None else "") + ) + result = subprocess.run( + purge_cmd, + shell=True, + capture_output=True, + text=True, + input=input_value + ) + return { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } + + def check_queues( + self, + redis_client: FixtureRedis, + queues_in_spec: Dict[str, str], + expected_task_count: int, + steps_to_purge: List[str] = None, + ): + """ + Check the state of queues in Redis against expected task counts. + + When `steps_to_purge` is set, the `expected_task_count` will represent the + number of expected tasks in the queues that _are not_ associated with the + steps in the `steps_to_purge` list. + + Args: + redis_client: The Redis client instance. + queues_in_spec: A dictionary of queues to check. + expected_task_count: The expected number of tasks in the queues (0 or 1). + steps_to_purge: Optional list of steps to determine which queues should be purged. + """ + for queue in queues_in_spec.values(): + # Brackets are special chars in regex so we have to add \ to make them literal + queue = queue.replace("[", "\\[").replace("]", "\\]") + matching_queues_on_server = redis_client.keys(pattern=f"{queue}*") + + for matching_queue in matching_queues_on_server: + tasks = redis_client.lrange(matching_queue, 0, -1) + if steps_to_purge and matching_queue in [queues_in_spec[step] for step in steps_to_purge]: + assert len(tasks) == 0, f"Expected 0 tasks in {matching_queue}, found {len(tasks)}." + else: + assert len(tasks) == expected_task_count, f"Expected {expected_task_count} tasks in {matching_queue}, found {len(tasks)}." + + def test_no_options_tasks_exist_y( + self, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with no options added and + tasks sent to the server. This should come up with a y/N + prompt in which we type 'y'. This should then purge the + tasks from the server. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + + # Run the purge test + test_info = self.run_purge(feature_demo, input_value="y") + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"Purged {num_queues} messages from {num_queues} known task queues."), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were purged + self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + + def test_no_options_no_tasks_y( + self, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with no options added and + no tasks sent to the server. This should come up with a y/N + prompt in which we type 'y'. This should then give us a "No + messages purged" log. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Get the queues from the spec file + spec = get_spec_with_expansion(feature_demo) + queues_in_spec = spec.get_task_queues() + num_queues = len(queues_in_spec.values()) + + # Check that there are no tasks in the queues before we run the purge command + self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + + # Run the purge test + test_info = self.run_purge(feature_demo, input_value="y") + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"No messages purged from {num_queues} queues."), + ] + check_test_conditions(conditions, test_info) + + # Check that the Redis server still has no tasks + self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + + + def test_no_options_N( + self, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with no options added and + tasks sent to the server. This should come up with a y/N + prompt in which we type 'N'. This should take us out of the + command without purging the tasks. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + + # Run the purge test + test_info = self.run_purge(feature_demo, input_value="N") + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"Purged {num_queues} messages from {num_queues} known task queues.", negate=True), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were not purged + self.check_queues(redis_client, queues_in_spec, expected_task_count=1) + + def test_force_option( + self, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with the `--force` option + enabled. This should not bring up a y/N prompt and should + immediately purge all tasks. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + + # Run the purge test + test_info = self.run_purge(feature_demo, force=True) + + # Make sure the subprocess ran and the correct output messages are given + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?", negate=True), + HasRegex(f"Purged {num_queues} messages from {num_queues} known task queues."), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were purged + self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + + def test_steps_option( + self, + redis_client: FixtureRedis, + redis_results_backend_config: FixtureModification, + redis_broker_config: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + ): + """ + Test the `merlin purge` command with the `--steps` option + enabled. This should only purge the tasks in the task queues + associated with the steps provided. + + Args: + redis_client: + A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config: + A fixture that modifies the CONFIG object so that it points the results + backend configuration to the containerized redis server we start up with + the `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + redis_broker_config: + A fixture that modifies the CONFIG object so that it points the broker + configuration to the containerized redis server we start up with the + `redis_server` fixture. The CONFIG object is what merlin uses to connect + to a server. + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. + merlin_server_dir: + A fixture to provide the path to the merlin_server directory that will be + created by the `redis_server` fixture. + """ + from merlin.celery import app as celery_app + + feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send tasks to the server for every queue in the spec + queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + + # Run the purge test + steps_to_purge = ["hello", "collect"] + test_info = self.run_purge(feature_demo, input_value="y", steps_to_purge=steps_to_purge) + + # Make sure the subprocess ran and the correct output messages are given + num_steps_to_purge = len(steps_to_purge) + conditions = [ + HasReturnCode(), + HasRegex("Are you sure you want to delete all tasks?"), + HasRegex(f"Purged {num_steps_to_purge} messages from {num_steps_to_purge} known task queues."), + ] + check_test_conditions(conditions, test_info) + + # Check on the Redis queues to ensure they were not purged + self.check_queues(redis_client, queues_in_spec, expected_task_count=1, steps_to_purge=steps_to_purge) From b36dedcc81771a1fe8929deb1ad7b4e552d26499 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 16 Oct 2024 16:51:56 -0700 Subject: [PATCH 178/201] update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eeff3df0..f8fd49fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `CeleryWorkersManager`: context to help with starting/stopping workers for tests - Ability to copy and print the `Config` object from `merlin/config/__init__.py` - Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py` -- Added additional tests for the `merlin run` command +- Added additional tests for the `merlin run` and `merlin purge` commands +- Aliased types to represent different types of pytest fixtures ### Changed - Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier From 6758282bf239163e313e16592f4c2926ee094cf0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 16 Oct 2024 16:52:59 -0700 Subject: [PATCH 179/201] update run command tests to use conditions when appropriate --- tests/integration/commands/test_run.py | 139 ++++++++++++++----------- tests/integration/conditions.py | 4 +- 2 files changed, 82 insertions(+), 61 deletions(-) diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index e55c15e9d..b04d51a22 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -6,13 +6,14 @@ import os import re import subprocess -from typing import Tuple +from typing import Dict, Tuple, Union from redis import Redis -from merlin.spec.specification import MerlinSpec +from merlin.spec.expansion import get_spec_with_expansion from tests.context_managers.celery_task_manager import CeleryTaskManager from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr +from tests.integration.conditions import HasReturnCode, PathExists, StepFileExists from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd @@ -23,24 +24,37 @@ class TestRunCommand: demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") - def setup_test_environment(self, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr): + def setup_test_environment( + self, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + run_command_testing_dir: FixtureStr + ) -> str: """ Setup the test environment for these tests by: 1. Moving into the temporary output directory created specifically for these tests. 2. Copying the app.yaml file created by the `redis_server` fixture to the cwd so that Merlin can connect to the test server. + 3. Obtaining the path to the feature_demo spec that we'll use for these tests. Args: + path_to_merlin_codebase: + A fixture to provide the path to the directory containing Merlin's core + functionality. merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. run_command_testing_dir: The path to the the temp output directory for `merlin run` tests. + + Returns: + The path to the feature_demo spec file. """ os.chdir(run_command_testing_dir) copy_app_yaml_to_cwd(merlin_server_dir) + return os.path.join(path_to_merlin_codebase, self.demo_workflow) - def run_merlin_command(self, command: str) -> subprocess.CompletedProcess: + def run_merlin_command(self, command: str) -> Dict[str, Union[str, int]]: """ Open a subprocess and run the command specified by the `command` parameter. Ensure this command runs successfully and return the process results. @@ -55,10 +69,13 @@ def run_merlin_command(self, command: str) -> subprocess.CompletedProcess: AssertionError: If the command fails (non-zero return code). """ result = subprocess.run(command, shell=True, capture_output=True, text=True) - assert result.returncode == 0, f"Command failed with return code {result.returncode}. Output: {result.stdout} Error: {result.stderr}" - return result + return { + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, + } - def get_output_workspace_from_logs(self, stdout_logs: str, stderr_logs: str) -> str: + def get_output_workspace_from_logs(self, test_info: Dict[str, Union[str, int]]) -> str: """ Extracts the workspace path from the provided standard output and error logs. @@ -69,8 +86,7 @@ def get_output_workspace_from_logs(self, stdout_logs: str, stderr_logs: str) -> found, an assertion error is raised. Args: - stdout_logs: The standard output logs as a string. - stderr_logs: The standard error logs as a string. + test_info: The results from executing our test. Returns: The extracted workspace path from the logs. @@ -79,31 +95,11 @@ def get_output_workspace_from_logs(self, stdout_logs: str, stderr_logs: str) -> AssertionError: If the expected message is not found in the combined logs. """ workspace_pattern = re.compile(r"Study workspace is '(\S+)'") - combined_output = stdout_logs + stderr_logs + combined_output = test_info["stdout"] + test_info["stderr"] match = workspace_pattern.search(combined_output) assert match, "No 'Study workspace is...' message found in command output." return match.group(1) - def validate_workspace(self, result: subprocess.CompletedProcess) -> str: - """ - Validate the workspace path extracted from the command output logs. - - This method retrieves the expected workspace path from the standard output and error logs - of the executed command. It checks if the workspace path exists in the filesystem. - - Args: - result: The result of the executed command, containing stdout and stderr. - - Returns: - The path to the expected workspace. - - Raises: - AssertionError: If the expected workspace path is not found in the filesystem. - """ - expected_workspace_path = self.get_output_workspace_from_logs(result.stdout, result.stderr) - assert os.path.exists(expected_workspace_path), f"Expected workspace not found: {expected_workspace_path}" - return expected_workspace_path - class TestRunCommandDistributed(TestRunCommand): """ @@ -149,15 +145,17 @@ def test_distributed_run( from merlin.celery import app as celery_app # Setup the testing environment - feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) - self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) with CeleryTaskManager(celery_app, redis_client) as CTM: # Send tasks to the server - self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_distributed_run") + test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_distributed_run") + + # Check that the test ran properly + check_test_conditions([HasReturnCode()], test_info) # Get the queues we need to query - spec = MerlinSpec.load_specification(feature_demo) + spec = get_spec_with_expansion(feature_demo) queues_in_spec = spec.get_task_queues() for queue in queues_in_spec.values(): @@ -209,8 +207,7 @@ def test_samplesfile_option( from merlin.celery import app as celery_app # Setup the testing environment - feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) - self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) # Create a new samples file to pass into our test workflow data = [ @@ -226,11 +223,16 @@ def test_samplesfile_option( with CeleryTaskManager(celery_app, redis_client) as CTM: # Send tasks to the server - result = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_samplesfile_option --samplesfile {new_samples_file}") + test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_samplesfile_option --samplesfile {new_samples_file}") - # Check that the new samples file is written to the merlin_info directory - expected_workspace_path = self.validate_workspace(result) - assert os.path.exists(os.path.join(expected_workspace_path, "merlin_info", sample_filename)) + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + PathExists(os.path.join(expected_workspace_path, "merlin_info", sample_filename)) + ] + check_test_conditions(conditions, test_info) def test_pgen_and_pargs_options( self, @@ -273,26 +275,29 @@ def test_pgen_and_pargs_options( from merlin.celery import app as celery_app # Setup test vars and the testing environment - feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) new_x2_min, new_x2_max = 1, 2 new_n_new_min, new_n_new_max = 5, 15 pgen_filepath = os.path.join(os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))), "pgen.py") - self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) with CeleryTaskManager(celery_app, redis_client) as CTM: # Send tasks to the server - result = self.run_merlin_command( + test_info = self.run_merlin_command( f'merlin run {feature_demo} --vars NAME=run_command_test_pgen_and_pargs_options --pgen {pgen_filepath} --parg "X2_MIN:{new_x2_min}" --parg "X2_MAX:{new_x2_max}" --parg "N_NAME_MIN:{new_n_new_min}" --parg "N_NAME_MAX:{new_n_new_max}"' ) - # Check that an expanded yaml file exists - expected_workspace_path = self.validate_workspace(result) - merlin_info_dir = os.path.join(expected_workspace_path, "merlin_info") - expanded_yaml = os.path.join(merlin_info_dir, "feature_demo.expanded.yaml") - assert os.path.exists(expanded_yaml), f"Expected YAML file not found: {expanded_yaml}" + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + expanded_yaml = os.path.join(expected_workspace_path, "merlin_info", "feature_demo.expanded.yaml") + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + PathExists(os.path.join(expanded_yaml)) + ] + check_test_conditions(conditions, test_info) # Read in the parameters from the expanded yaml and ensure they're within the new bounds we provided - expanded_spec = MerlinSpec.load_specification(expanded_yaml) + expanded_spec = get_spec_with_expansion(expanded_yaml) params = expanded_spec.get_parameters() for x2_param in params.parameters["X2"]: assert new_x2_min <= x2_param <= new_x2_max @@ -347,15 +352,21 @@ def test_dry_run( The path to the the temp output directory for `merlin run` tests. """ # Setup the test environment - feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) - self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) # Run the test and grab the output workspace generated from it - result = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_dry_run --local --dry") - expected_workspace_path = self.validate_workspace(result) + test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_dry_run --local --dry") + + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + ] + check_test_conditions(conditions, test_info) # Check that every step was ran by looking for an existing output workspace - spec = MerlinSpec.load_specification(feature_demo) + spec = get_spec_with_expansion(feature_demo) for step in spec.get_study_steps(): step_directory = os.path.join(expected_workspace_path, step.name) assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" @@ -408,15 +419,25 @@ def test_local_run( The path to the the temp output directory for `merlin run` tests. """ # Setup the test environment - feature_demo = os.path.join(path_to_merlin_codebase, self.demo_workflow) - self.setup_test_environment(merlin_server_dir, run_command_testing_dir) + feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) # Run the test and grab the output workspace generated from it - result = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_local_run --local") - expected_workspace_path = self.validate_workspace(result) + study_name = "run_command_test_local_run" + test_info = self.run_merlin_command( + f"merlin run {feature_demo} --vars NAME={study_name} OUTPUT_PATH={run_command_testing_dir} --local" + ) + + # Check that the test ran properly and created the correct directories/files + expected_workspace_path = self.get_output_workspace_from_logs(test_info) + + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + ] + check_test_conditions(conditions, test_info) # Check that every step was ran by looking for an existing output workspace and MERLIN_FINISHED files - spec = MerlinSpec.load_specification(feature_demo) + spec = get_spec_with_expansion(feature_demo) for step in spec.get_study_steps(): step_directory = os.path.join(expected_workspace_path, step.name) assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 83f07aafe..ad029ffb4 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -176,8 +176,8 @@ def glob_string(self): """ param_glob = "" if self.params: - param_glob = "*/" - return f"{self.dirpath_glob}/{self.step}/{param_glob}{self.filename}" + param_glob = "*" + return os.path.join(self.dirpath_glob, self.step, param_glob, self.filename) def file_exists(self): """Check if the file path created by glob_string exists""" From 466c27673b5f25c36fec833cf5209c35ba64aaa9 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 28 Oct 2024 09:50:38 -0700 Subject: [PATCH 180/201] start work on adding workflow tests --- tests/fixtures/feature_demo.py | 112 ++++++++++++++++++ .../workflows/test_feature_demo.py | 73 ++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 tests/fixtures/feature_demo.py create mode 100644 tests/integration/workflows/test_feature_demo.py diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py new file mode 100644 index 000000000..0710c5b24 --- /dev/null +++ b/tests/fixtures/feature_demo.py @@ -0,0 +1,112 @@ +""" +Fixtures specifically for help testing the feature_demo workflow. +""" +import os + +import pytest + +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr, FixtureTuple +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec + + +@pytest.fixture(scope="session") +def feature_demo_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: + """ + Fixture to create a temporary output directory for tests related to testing the + feature_demo workflow. + + Args: + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. + + Returns: + The path to the temporary testing directory for feature_demo workflow tests. + """ + testing_dir = f"{temp_output_dir}/feature_demo_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir + + +@pytest.fixture(scope="session") +def feature_demo_num_samples() -> FixtureInt: + """ + Defines a specific number of samples to use for the feature_demo workflow. + This helps ensure that even if changes were made to the feature_demo workflow, + tests using this fixture should still run the same thing. + + Returns: + An integer representing the number of samples to use in the feature_demo workflow. + """ + return 8 + + +@pytest.fixture(scope="session") +def feature_demo_name() -> FixtureStr: + """ + Defines a specific name to use for the feature_demo workflow. This helps ensure + that even if changes were made to the feature_demo workflow, tests using this fixture + should still run the same thing. + + Returns: + An string representing the name to use for the feature_demo workflow. + """ + return "feature_demo_test" + + +@pytest.fixture(scope="class") +def feature_demo_run_workflow( + redis_client: FixtureRedis, + redis_results_backend_config_class: FixtureModification, + redis_broker_config_class: FixtureModification, + path_to_merlin_codebase: FixtureStr, + merlin_server_dir: FixtureStr, + feature_demo_testing_dir: FixtureStr, + feature_demo_num_samples: FixtureInt, + feature_demo_name: FixtureStr, +) -> FixtureTuple[str, str]: + """ + """ + from merlin.celery import app as celery_app + + # os.chdir(feature_demo_testing_dir) + copy_app_yaml_to_cwd(merlin_server_dir) + feature_demo_path = os.path.join(path_to_merlin_codebase, self.demo_workflow) + # test_output_path = os.path.join(feature_demo_testing_dir, self.get_test_name()) + test_name = self.get_test_name() + + vars_to_substitute = [ + f"N_SAMPLES={feature_demo_num_samples}", + f"NAME={feature_demo_name}", + f"OUTPUT_PATH={feature_demo_testing_dir}" + ] + + run_workers_proc = None + + with CeleryTaskManager(celery_app, redis_client) as CTM: + run_proc = subprocess.run( + f"merlin run {feature_demo_path} --vars {' '.join(vars_to_substitute)}", + shell=True, + capture_output=True, + text=True, + ) + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as CWM: + # Start the workers then add them to the context manager so they can be stopped safely later + run_workers_proc = subprocess.Popen( + f"merlin run-workers {feature_demo_path}".split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True + ) + CWM.add_run_workers_process(run_workers_proc.pid) + + # Let the workflow try to run for 30 seconds + sleep(30) + + stdout, stderr = run_workers_proc.communicate() + return stdout, stderr diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py new file mode 100644 index 000000000..09d95b60a --- /dev/null +++ b/tests/integration/workflows/test_feature_demo.py @@ -0,0 +1,73 @@ +""" +This module contains tests for the feature_demo workflow. +""" +import inspect +import os +import signal +import subprocess +from time import sleep +from subprocess import TimeoutExpired + +# from tests.context_managers.celery_task_manager import CeleryTaskManager +# from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr, FixtureTuple +from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec + + +# NOTE maybe this workflow only needs to run twice? +# - once for a normal run +# - can test e2e, data passing, and step execution order with a single run +# - another time for error testing + +class TestFeatureDemo: + """ + Tests for the feature_demo workflow. + """ + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + + def get_test_name(self): + """ + """ + stack = inspect.stack() + return stack[1].function + + def test_end_to_end_run( + self, + feature_demo_testing_dir: FixtureStr, + feature_demo_num_samples: FixtureInt, + feature_demo_name: FixtureStr, + feature_demo_run_workflow: FixtureTuple[str, str], + ): + """ + Test that the workflow runs from start to finish with no problems. + """ + # TODO check if the workflow ran to completion in 30 seconds + # if not assert a failure happened + # - to check, see if all MERLIN_FINISHED files exist + + StepFinishedFilesCount( + step="hello", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=feature_demo_num_samples, + ) + + def test_step_execution_order(self): + """ + Test that steps are executed in the correct order. + """ + # TODO build a list with the correct order that steps should be ran + # TODO compare the list against the logs from the worker + + def test_workflow_error_handling(self): + """ + Test the behavior when errors arise during the worfklow. + + TODO should this test both soft and hard fails? should this test all return codes? + """ + + def test_data_passing(self): + """ + Test that data can be successfully passed between steps using built-in Merlin variables. + """ From 144246f41f1b456301f76368f91764884aa1d7f2 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 31 Oct 2024 12:48:11 -0700 Subject: [PATCH 181/201] create function and class scoped config fixtures --- tests/conftest.py | 208 ++++++++++++++---- tests/integration/commands/test_purge.py | 40 ++-- tests/integration/commands/test_run.py | 40 ++-- .../commands/test_stop_and_query_workers.py | 40 ++-- tests/unit/common/test_encryption.py | 16 +- tests/unit/config/test_broker.py | 86 ++++---- tests/unit/config/test_results_backend.py | 56 ++--- tests/unit/config/test_utils.py | 12 +- 8 files changed, 313 insertions(+), 185 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 22da1a061..9e17b0723 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,9 +68,10 @@ # Loading in Module Specific Fixtures # ####################################### +fixture_glob = os.path.join("tests", "fixtures", "**", "*.py") pytest_plugins = [ - fixture_file.replace("/", ".").replace(".py", "") - for fixture_file in glob("tests/fixtures/**/*.py", recursive=True) + fixture_file.replace(os.sep, ".").replace(".py", "") + for fixture_file in glob(fixture_glob, recursive=True) if not fixture_file.endswith("__init__.py") ] @@ -106,6 +107,31 @@ def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_fi yaml.dump(app_yaml, app_yaml_file) +def setup_redis_config(config_type: str, merlin_server_dir: str): + """ + Sets up the Redis configuration for either broker or results backend. + + Args: + config_type: The type of configuration to set up ('broker' or 'results_backend'). + merlin_server_dir: The directory to the merlin test server configuration. + """ + port = 6379 + name = "redis" + pass_file = os.path.join(merlin_server_dir, "redis.pass") + create_pass_file(pass_file) + + if config_type == 'broker': + CONFIG.broker.password = pass_file + CONFIG.broker.port = port + CONFIG.broker.name = name + elif config_type == 'results_backend': + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.port = port + CONFIG.results_backend.name = name + else: + raise ValueError("Invalid config_type. Must be 'broker' or 'results_backend'.") + + ####################################### ######### Fixture Definitions ######### ####################################### @@ -307,17 +333,22 @@ def test_encryption_key() -> FixtureBytes: ####################################### -@pytest.fixture(scope="function") -def config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: +def _config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes): """ - DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. - This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` - fixtures. It sets up the CONFIG object but leaves certain broker settings unset. + Sets up the configuration for testing purposes by modifying the global CONFIG object. - :param merlin_server_dir: The directory to the merlin test server configuration - :param test_encryption_key: An encryption key to be used for testing - """ + This helper function prepares the broker and results backend configurations for testing + by creating necessary encryption key files and resetting the CONFIG object to its + original state after the tests are executed. + Args: + merlin_server_dir: The directory to the merlin test server configuration + test_encryption_key: An encryption key to be used for testing + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ # Create a copy of the CONFIG option so we can reset it after the test orig_config = copy(CONFIG) @@ -326,9 +357,9 @@ def config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> create_encryption_file(key_file, test_encryption_key) # Set the broker configuration for testing - CONFIG.broker.password = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` - CONFIG.broker.port = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` - CONFIG.broker.name = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.password = None # This will be updated in `redis_broker_config_*` or `rabbit_broker_config` + CONFIG.broker.port = None # This will be updated in `redis_broker_config_*` or `rabbit_broker_config` + CONFIG.broker.name = None # This will be updated in `redis_broker_config_*` or `rabbit_broker_config` CONFIG.broker.server = "127.0.0.1" CONFIG.broker.username = "default" CONFIG.broker.vhost = "host4testing" @@ -336,11 +367,11 @@ def config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> # Set the results_backend configuration for testing CONFIG.results_backend.password = ( - None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + None # This will be updated in `redis_results_backend_config_function` or `mysql_results_backend_config` ) - CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` + CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config_function` CONFIG.results_backend.name = ( - None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + None # This will be updated in `redis_results_backend_config_function` or `mysql_results_backend_config` ) CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` CONFIG.results_backend.server = "127.0.0.1" @@ -359,50 +390,147 @@ def config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> @pytest.fixture(scope="function") -def redis_broker_config( - merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +def config_function(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: + """ + Sets up the configuration for testing with a function scope. + + Warning: + DO NOT USE THIS FIXTURE IN A TEST, USE ONE OF THE SERVER SPECIFIC CONFIGURATIONS + (LIKE `redis_broker_config_function`, `rabbit_broker_config`, etc.) INSTEAD. + + This fixture modifies the global CONFIG object to prepare the broker and results backend + configurations for testing. It creates necessary encryption key files and ensures that + the original configuration is restored after the tests are executed. + + Args: + merlin_server_dir: The directory to the merlin test server configuration + test_encryption_key: An encryption key to be used for testing + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + yield from _config(merlin_server_dir, test_encryption_key) + +@pytest.fixture(scope="class") +def config_class(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: + """ + Sets up the configuration for testing with a class scope. + + Warning: + DO NOT USE THIS FIXTURE IN A TEST, USE ONE OF THE SERVER SPECIFIC CONFIGURATIONS + (LIKE `redis_broker_config_class`, `rabbit_broker_config`, etc.) INSTEAD. + + This fixture modifies the global CONFIG object to prepare the broker and results backend + configurations for testing. It creates necessary encryption key files and ensures that + the original configuration is restored after the tests are executed. + + Args: + merlin_server_dir: The directory to the merlin test server configuration + test_encryption_key: An encryption key to be used for testing + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + yield from _config(merlin_server_dir, test_encryption_key) + + +@pytest.fixture(scope="function") +def redis_broker_config_function( + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument ) -> FixtureModification: """ - This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a Redis broker and results_backend. + Fixture for configuring the Redis broker for testing with a function scope. - :param merlin_server_dir: The directory to the merlin test server configuration - :param config: The fixture that sets up most of the CONFIG object for testing + This fixture sets up the CONFIG object to use a Redis broker for testing any functionality + in the codebase that interacts with the broker. It modifies the configuration to point + to the specified Redis broker settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. """ - pass_file = os.path.join(merlin_server_dir, "redis.pass") - create_pass_file(pass_file) + setup_redis_config("broker", merlin_server_dir) + yield - CONFIG.broker.password = pass_file - CONFIG.broker.port = 6379 - CONFIG.broker.name = "redis" +@pytest.fixture(scope="class") +def redis_broker_config_class( + merlin_server_dir: FixtureStr, config_class: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: + """ + Fixture for configuring the Redis broker for testing with a class scope. + + This fixture sets up the CONFIG object to use a Redis broker for testing any functionality + in the codebase that interacts with the broker. It modifies the configuration to point + to the specified Redis broker settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + setup_redis_config("broker", merlin_server_dir) yield @pytest.fixture(scope="function") -def redis_results_backend_config( - merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +def redis_results_backend_config_function( + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument ) -> FixtureModification: """ - This fixture is intended to be used for testing any functionality in the codebase - that uses the CONFIG object with a Redis results_backend. + Fixture for configuring the Redis results backend for testing with a function scope. - :param merlin_server_dir: The directory to the merlin test server configuration - :param config: The fixture that sets up most of the CONFIG object for testing + This fixture sets up the CONFIG object to use a Redis results backend for testing any + functionality in the codebase that interacts with the results backend. It modifies the + configuration to point to the specified Redis results backend settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. """ - pass_file = os.path.join(merlin_server_dir, "redis.pass") - create_pass_file(pass_file) + setup_redis_config("results_backend", merlin_server_dir) + yield - CONFIG.results_backend.password = pass_file - CONFIG.results_backend.port = 6379 - CONFIG.results_backend.name = "redis" +@pytest.fixture(scope="class") +def redis_results_backend_config_class( + merlin_server_dir: FixtureStr, config_class: FixtureModification # pylint: disable=redefined-outer-name,unused-argument +) -> FixtureModification: + """ + Fixture for configuring the Redis results backend for testing with a class scope. + + This fixture sets up the CONFIG object to use a Redis results backend for testing any + functionality in the codebase that interacts with the results backend. It modifies the + configuration to point to the specified Redis results backend settings. + + Args: + merlin_server_dir: The directory to the merlin test server configuration. + config_function: The fixture that sets up most of the CONFIG object for testing. + + Yields: + This function yields control back to the test function, allowing tests to run + with the modified CONFIG settings. + """ + setup_redis_config("results_backend", merlin_server_dir) yield @pytest.fixture(scope="function") def rabbit_broker_config( - merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument ) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase @@ -423,7 +551,7 @@ def rabbit_broker_config( @pytest.fixture(scope="function") def mysql_results_backend_config( - merlin_server_dir: FixtureStr, config: FixtureModification # pylint: disable=redefined-outer-name,unused-argument + merlin_server_dir: FixtureStr, config_function: FixtureModification # pylint: disable=redefined-outer-name,unused-argument ) -> FixtureModification: """ This fixture is intended to be used for testing any functionality in the codebase diff --git a/tests/integration/commands/test_purge.py b/tests/integration/commands/test_purge.py index a3c6d8011..3c7b284d7 100644 --- a/tests/integration/commands/test_purge.py +++ b/tests/integration/commands/test_purge.py @@ -138,8 +138,8 @@ def check_queues( def test_no_options_tasks_exist_y( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -152,12 +152,12 @@ def test_no_options_tasks_exist_y( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -194,8 +194,8 @@ def test_no_options_tasks_exist_y( def test_no_options_no_tasks_y( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -208,12 +208,12 @@ def test_no_options_no_tasks_y( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -256,8 +256,8 @@ def test_no_options_no_tasks_y( def test_no_options_N( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -270,12 +270,12 @@ def test_no_options_N( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -312,8 +312,8 @@ def test_no_options_N( def test_force_option( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -325,12 +325,12 @@ def test_force_option( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -367,8 +367,8 @@ def test_force_option( def test_steps_option( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -380,12 +380,12 @@ def test_steps_option( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index b04d51a22..f1e7fec09 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -110,8 +110,8 @@ class TestRunCommandDistributed(TestRunCommand): def test_distributed_run( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -123,12 +123,12 @@ def test_distributed_run( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -171,8 +171,8 @@ def test_distributed_run( def test_samplesfile_option( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -185,12 +185,12 @@ def test_samplesfile_option( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -237,8 +237,8 @@ def test_samplesfile_option( def test_pgen_and_pargs_options( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -253,12 +253,12 @@ def test_pgen_and_pargs_options( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -314,8 +314,8 @@ class TestRunCommandLocal(TestRunCommand): def test_dry_run( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -332,12 +332,12 @@ def test_dry_run( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -386,8 +386,8 @@ def test_dry_run( def test_local_run( self, redis_client: FixtureRedis, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -399,12 +399,12 @@ def test_local_run( Args: redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py index 1144ff972..3cbbdc861 100644 --- a/tests/integration/commands/test_stop_and_query_workers.py +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -139,8 +139,8 @@ def get_no_workers_msg(self, command_to_test: str) -> WorkerMessages: def test_no_workers( self, redis_server: FixtureStr, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, merlin_server_dir: FixtureStr, command_to_test: str, ): @@ -163,12 +163,12 @@ def test_no_workers( redis_server: A fixture that starts a containerized redis server instance that runs on localhost:6379. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -204,8 +204,8 @@ def test_no_workers( def test_no_flags( self, redis_server: FixtureStr, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -222,12 +222,12 @@ def test_no_flags( redis_server: A fixture that starts a containerized redis server instance that runs on localhost:6379. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -258,8 +258,8 @@ def test_no_flags( def test_spec_flag( self, redis_server: FixtureStr, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -277,12 +277,12 @@ def test_spec_flag( redis_server: A fixture that starts a containerized redis server instance that runs on localhost:6379. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -318,8 +318,8 @@ def test_spec_flag( def test_workers_flag( self, redis_server: FixtureStr, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -337,12 +337,12 @@ def test_workers_flag( redis_server: A fixture that starts a containerized redis server instance that runs on localhost:6379. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect @@ -379,8 +379,8 @@ def test_workers_flag( def test_queues_flag( self, redis_server: FixtureStr, - redis_results_backend_config: FixtureModification, - redis_broker_config: FixtureModification, + redis_results_backend_config_function: FixtureModification, + redis_broker_config_function: FixtureModification, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -398,12 +398,12 @@ def test_queues_flag( redis_server: A fixture that starts a containerized redis server instance that runs on localhost:6379. - redis_results_backend_config: + redis_results_backend_config_function: A fixture that modifies the CONFIG object so that it points the results backend configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect to a server. - redis_broker_config: + redis_broker_config_function: A fixture that modifies the CONFIG object so that it points the broker configuration to the containerized redis server we start up with the `redis_server` fixture. The CONFIG object is what merlin uses to connect diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 3e37cef84..4fb775dc0 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -17,34 +17,34 @@ class TestEncryption: This class will house all tests necessary for our encryption modules. """ - def test_encrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_encrypt(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test that our encryption function is encrypting the bytes that we're passing to it. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ str_to_encrypt = b"super secret string shhh" encrypted_str = encrypt(str_to_encrypt) for word in str_to_encrypt.decode("utf-8").split(" "): assert word not in encrypted_str.decode("utf-8") - def test_decrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_decrypt(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test that our decryption function is decrypting the bytes that we're passing to it. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # This is the output of the bytes from the encrypt test str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" decrypted_str = decrypt(str_to_decrypt) assert decrypted_str == b"super secret string shhh" - def test_get_key_path(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_key_path(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `_get_key_path` function. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) @@ -89,14 +89,14 @@ def test_gen_key(self, temp_output_dir: str): assert key_gen_contents != "" def test_get_key( - self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture" # noqa: F821 + self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config_function: "fixture" # noqa: F821 ): """ Test the `_get_key` function. :param merlin_server_dir: The directory to the merlin test server configuration :param test_encryption_key: A fixture to establish a fixed encryption key for testing - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Test the default functionality actual_default = _get_key() diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py index 581b19488..3b44c5261 100644 --- a/tests/unit/config/test_broker.py +++ b/tests/unit/config/test_broker.py @@ -36,37 +36,37 @@ def test_read_file(merlin_server_dir: str): assert actual == SERVER_PASS -def test_get_connection_string_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_connection_string_invalid_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "invalid_broker" with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_no_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_connection_string_no_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function without a broker name value in the CONFIG object. This should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name with pytest.raises(ValueError): get_connection_string() -def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: F821 +def test_get_connection_string_simple(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function in the simplest way that we can. This function will automatically check for a broker url and if it finds one in the CONFIG object it will just return the value it finds. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_url = "test_url" CONFIG.broker.url = test_url @@ -74,11 +74,11 @@ def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: assert actual == test_url -def test_get_ssl_config_no_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_ssl_config_no_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function without a broker. This should return False. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.name assert not get_ssl_config() @@ -274,11 +274,11 @@ def run_get_redissock_connection(self, expected_vals: Dict[str, str]): actual = get_redissock_connection() assert actual == expected - def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with both a db_num and a broker path set. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path and db_num for testing test_path = "/fake/path/to/broker" @@ -290,12 +290,12 @@ def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa expected_vals = {"db_num": test_db_num, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_db(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a broker path set but no db num. This should default the db_num to 0. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Create and store a fake path for testing test_path = "/fake/path/to/broker" @@ -305,25 +305,25 @@ def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): expected_vals = {"db_num": 0, "path": test_path} self.run_get_redissock_connection(expected_vals) - def test_get_redissock_connection_no_path(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with a db num set but no broker path. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.db_num = "45" with pytest.raises(AttributeError): get_redissock_connection() - def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redissock_connection` function with neither a broker path nor a db num set. This should raise an AttributeError since there will be no path value to read from in `CONFIG.broker`. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ with pytest.raises(AttributeError): get_redissock_connection() @@ -341,11 +341,11 @@ def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_passwo actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) assert expected == actual - def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -356,13 +356,13 @@ def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F8 } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_port(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the port setting from the CONFIG object. This should still run and give us port = 6379. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.port expected_vals = { @@ -374,12 +374,12 @@ def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_with_db(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after adding the db_num setting to the CONFIG object. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ test_db_num = "45" CONFIG.broker.db_num = test_db_num @@ -392,25 +392,25 @@ def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_no_username(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_username(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the username setting from the CONFIG object. This should still run and give us username = ''. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.username expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_invalid_pass_file(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after changing the permissions of the password file so it can't be opened. This should still run and give us password = CONFIG.broker.password. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Capture the initial permissions of the password file so we can reset them orig_file_permissions = os.stat(CONFIG.broker.password).st_mode @@ -435,21 +435,21 @@ def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixt os.chmod(CONFIG.broker.password, orig_file_permissions) - def test_get_redis_connection_dont_include_password(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_dont_include_password(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function without including the password. This should place 6 *s where the password would normally be placed in spass. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) - def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_use_ssl(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "rediss", @@ -460,24 +460,24 @@ def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # } self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) - def test_get_redis_connection_no_password(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_redis_connection_no_password(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_redis_connection` function with default functionality (including password and not using ssl). We'll run this after deleting the password setting from the CONFIG object. This should still run and give us spass = ''. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.broker.password expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) - def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis as the broker (this is what our CONFIG - is set to by default with the redis_broker_config fixture). + is set to by default with the redis_broker_config_function fixture). - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -490,11 +490,11 @@ def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # n actual = get_connection_string() assert expected == actual - def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_connection_string_rediss(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rediss (with two 's') as the broker. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected_vals = { @@ -508,11 +508,11 @@ def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # actual = get_connection_string() assert expected == actual - def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis_socket(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis+socket as the broker. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Change our broker CONFIG.broker.name = "redis+socket" @@ -529,21 +529,21 @@ def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture" actual = get_connection_string() assert actual == expected - def test_get_ssl_config_redis(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_ssl_config_redis(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). This should return False. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert not get_ssl_config() - def test_get_ssl_config_rediss(self, redis_broker_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss(self, redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss (with two 's') as the broker. This should return a dict of cert reqs with ssl.CERT_NONE as the value. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "rediss" expected = {"ssl_cert_reqs": CERT_NONE} diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py index f49e3e897..55459f3ec 100644 --- a/tests/unit/config/test_results_backend.py +++ b/tests/unit/config/test_results_backend.py @@ -102,7 +102,7 @@ def test_get_backend_password_using_certs_path(temp_output_dir: str): assert get_backend_password(pass_filename, certs_path=test_dir) == SERVER_PASS -def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 +def test_get_ssl_config_no_results_backend(config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with no results_backend set. This should return False. NOTE: we're using the config fixture here to make sure values are reset after this test finishes. @@ -114,7 +114,7 @@ def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 assert get_ssl_config() is False -def test_get_connection_string_no_results_backend(config: "fixture"): # noqa: F821 +def test_get_connection_string_no_results_backend(config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with no results_backend set. This should raise a ValueError. @@ -160,11 +160,11 @@ def run_get_redis( actual = get_redis(certs_path=certs_path, include_password=include_password, ssl=ssl) assert actual == expected - def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with default functionality. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -175,11 +175,11 @@ def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_dont_include_password(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_dont_include_password(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with the password hidden. This should * out the password. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -190,11 +190,11 @@ def test_get_redis_dont_include_password(self, redis_results_backend_config: "fi } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=False, ssl=False) - def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_using_ssl(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with ssl enabled. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "rediss", @@ -205,11 +205,11 @@ def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=True) - def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_port(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no port in our CONFIG object. This should default to port=6379. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.port expected_vals = { @@ -221,11 +221,11 @@ def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # no } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_db_num(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no db_num in our CONFIG object. This should default to db_num=0. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.db_num expected_vals = { @@ -237,11 +237,11 @@ def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_username(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no username in our CONFIG object. This should default to username=''. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.username expected_vals = { @@ -253,11 +253,11 @@ def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_no_password_file(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function with no password filepath in our CONFIG object. This should default to spass=''. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.password expected_vals = { @@ -269,12 +269,12 @@ def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture } self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) - def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_redis_invalid_pass_file(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_redis` function. We'll run this after changing the permissions of the password file so it can't be opened. This should still run and give us password=CONFIG.results_backend.password. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ # Capture the initial permissions of the password file so we can reset them @@ -299,41 +299,41 @@ def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixtur os.chmod(CONFIG.results_backend.password, orig_file_permissions) raise AssertionError from exc - def test_get_ssl_config_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_redis(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with redis as the results_backend. This should return False since ssl requires using rediss (with two 's'). - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_ssl_config() is False - def test_get_ssl_config_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss as the results_backend. This should return a dict of cert reqs with ssl.CERT_NONE as the value. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.results_backend.name = "rediss" assert get_ssl_config() == {"ssl_cert_reqs": CERT_NONE} - def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_ssl_config` function with rediss as the results_backend and no cert_reqs set. This should return True. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ del CONFIG.results_backend.cert_reqs CONFIG.results_backend.name = "rediss" assert get_ssl_config() is True - def test_get_connection_string_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_connection_string_redis(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with redis as the results_backend. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ expected_vals = { "urlbase": "redis", @@ -346,11 +346,11 @@ def test_get_connection_string_redis(self, redis_results_backend_config: "fixtur actual = get_connection_string() assert actual == expected - def test_get_connection_string_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + def test_get_connection_string_rediss(self, redis_results_backend_config_function: "fixture"): # noqa: F821 """ Test the `get_connection_string` function with rediss as the results_backend. - :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_results_backend_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.results_backend.name = "rediss" expected_vals = { diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py index 9d64c10c7..b4f7c52fa 100644 --- a/tests/unit/config/test_utils.py +++ b/tests/unit/config/test_utils.py @@ -47,12 +47,12 @@ def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F assert get_priority(Priority.RETRY) == 10 -def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_priority_redis_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_priority` function with redis as the broker. Low priority for redis is 10 and high is 2. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ assert get_priority(Priority.LOW) == 10 assert get_priority(Priority.MID) == 5 @@ -60,12 +60,12 @@ def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F82 assert get_priority(Priority.RETRY) == 1 -def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 +def test_get_priority_invalid_broker(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_priority` function with an invalid broker. This should raise a ValueError. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ CONFIG.broker.name = "invalid" with pytest.raises(ValueError) as excinfo: @@ -73,12 +73,12 @@ def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F assert "Unsupported broker name: invalid" in str(excinfo.value) -def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 +def test_get_priority_invalid_priority(redis_broker_config_function: "fixture"): # noqa: F821 """ Test the `get_priority` function with an invalid priority. This should raise a TypeError. - :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param redis_broker_config_function: A fixture to set the CONFIG object to a test configuration that we'll use here """ with pytest.raises(ValueError) as excinfo: get_priority("invalid_priority") From 0518a48db1089e61c8c1fef1292f8a14b3e758a0 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 31 Oct 2024 12:49:34 -0700 Subject: [PATCH 182/201] add Tuple fixture type --- tests/fixture_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fixture_types.py b/tests/fixture_types.py index 3b02188d2..5d9902491 100644 --- a/tests/fixture_types.py +++ b/tests/fixture_types.py @@ -20,7 +20,7 @@ from celery import Celery from celery.canvas import Signature from redis import Redis -from typing import Any, Annotated, Dict, TypeVar +from typing import Any, Annotated, Dict, Tuple, TypeVar # TODO convert unit test type hinting to use these # - likely will do this when I work on API docs for test library @@ -37,3 +37,4 @@ FixtureRedis = Annotated[Redis, pytest.fixture] FixtureSignature = Annotated[Signature, pytest.fixture] FixtureStr = Annotated[str, pytest.fixture] +FixtureTuple = Annotated[Tuple[K, V], pytest.fixture] From d57765311de6a315592d09e6880511024aadfd5f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 31 Oct 2024 15:22:48 -0700 Subject: [PATCH 183/201] get e2e test of feature_demo workflow running --- .../celery_workers_manager.py | 35 +++++- tests/fixtures/feature_demo.py | 53 ++++++-- tests/integration/conditions.py | 97 +++++++++++++-- tests/integration/helper_funcs.py | 62 +++++----- .../workflows/test_feature_demo.py | 115 +++++++++++++----- 5 files changed, 273 insertions(+), 89 deletions(-) diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 2daecb612..c38173df9 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -52,7 +52,8 @@ class CeleryWorkersManager: def __init__(self, app: Celery): self.app = app - self.running_workers = [] + self.running_workers = set() + self.run_worker_processes = set() self.worker_processes = {} self.echo_processes = {} @@ -145,7 +146,7 @@ def start_worker(self, worker_launch_cmd: List[str]): """ self.app.worker_main(worker_launch_cmd) - def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = 1): + def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = 1, prefetch: int = 1): """ Launch a single worker. We'll add the process that the worker is running in to the list of worker processes. We'll also create an echo process to simulate a celery worker command that will show up with 'ps ux'. @@ -170,6 +171,8 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = ",".join(queues), "--concurrency", str(concurrency), + "--prefetch-multiplier", + str(prefetch), f"--logfile={worker_name}.log", "--loglevel=DEBUG", ] @@ -187,7 +190,7 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = worker_process = multiprocessing.Process(target=self.start_worker, args=(worker_launch_cmd,)) worker_process.start() self.worker_processes[worker_name] = worker_process - self.running_workers.append(worker_name) + self.running_workers.add(worker_name) # Wait for the worker to launch properly try: @@ -207,6 +210,24 @@ def launch_workers(self, worker_info: Dict[str, Dict]): for worker_name, worker_settings in worker_info.items(): self.launch_worker(worker_name, worker_settings["queues"], worker_settings["concurrency"]) + def add_run_workers_process(self, pid: int): + """ + Add a process ID for a `merlin run-workers` process to the + set that tracks all `merlin run-workers` processes that are + currently running. + + Warning: + The process that's added here must utilize the + `start_new_session=True` setting of subprocess.Popen. This + is necessary for us to be able to terminate all the workers + that are started with it safely since they will be seen as + child processes of the `merlin run-workers` process. + + Args: + pid: The process ID running `merlin run-workers`. + """ + self.run_worker_processes.add(pid) + def stop_worker(self, worker_name: str): """ Stop a single running worker and its associated processes. @@ -226,12 +247,16 @@ def stop_worker(self, worker_name: str): self.worker_processes[worker_name].kill() # Terminate the echo process and its sleep inf subprocess - os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGTERM) - sleep(2) + if self.echo_processes[worker_name] is not None: + os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGTERM) + sleep(2) def stop_all_workers(self): """ Stop all of the running workers and the processes associated with them. """ + for run_worker_pid in self.run_worker_processes: + os.killpg(os.getpgid(run_worker_pid), signal.SIGTERM) + for worker_name in self.running_workers: self.stop_worker(worker_name) diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py index 0710c5b24..573e54f49 100644 --- a/tests/fixtures/feature_demo.py +++ b/tests/fixtures/feature_demo.py @@ -2,12 +2,14 @@ Fixtures specifically for help testing the feature_demo workflow. """ import os +import subprocess +from time import sleep import pytest from tests.context_managers.celery_task_manager import CeleryTaskManager from tests.context_managers.celery_workers_manager import CeleryWorkersManager -from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr, FixtureTuple +from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec @@ -51,7 +53,7 @@ def feature_demo_name() -> FixtureStr: should still run the same thing. Returns: - An string representing the name to use for the feature_demo workflow. + A string representing the name to use for the feature_demo workflow. """ return "feature_demo_test" @@ -66,17 +68,48 @@ def feature_demo_run_workflow( feature_demo_testing_dir: FixtureStr, feature_demo_num_samples: FixtureInt, feature_demo_name: FixtureStr, -) -> FixtureTuple[str, str]: +) -> subprocess.CompletedProcess: """ + Run the feature demo workflow. + + This fixture sets up and executes the feature demo workflow using the specified configurations + and parameters. It prepares the environment by modifying the CONFIG object to connect to a + Redis server and runs the demo workflow with the provided sample size and name. It utilizes + context managers to safely send tasks to the server and start up workers. The workflow is given + 30 seconds to complete which should be plenty of time. + + Args: + redis_client: A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config_class: A fixture that modifies the CONFIG object so that it + points the results backend configuration to the containerized redis server we start up + with the [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin + uses to connect to a server. + redis_broker_config_class: A fixture that modifies the CONFIG object so that it points + the broker configuration to the containerized redis server we start up with the + [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin uses + to connect to a server. + path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's + core functionality. + merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be + created by the [`redis_server`][conftest.redis_server] fixture. + feature_demo_testing_dir: The path to the temp output directory for feature_demo workflow tests. + feature_demo_num_samples: An integer representing the number of samples to use in the feature_demo + workflow. + feature_demo_name: A string representing the name to use for the feature_demo workflow. + + Returns: + The completed process object containing information about the execution of the workflow, including + return code, stdout, and stderr. """ + # TODO might want to generalize the logic in this function into a new function that runs workflows from merlin.celery import app as celery_app - - # os.chdir(feature_demo_testing_dir) + + # Setup the test copy_app_yaml_to_cwd(merlin_server_dir) - feature_demo_path = os.path.join(path_to_merlin_codebase, self.demo_workflow) - # test_output_path = os.path.join(feature_demo_testing_dir, self.get_test_name()) - test_name = self.get_test_name() + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + feature_demo_path = os.path.join(path_to_merlin_codebase, demo_workflow) + # Create the variables to pass in to the workflow vars_to_substitute = [ f"N_SAMPLES={feature_demo_num_samples}", f"NAME={feature_demo_name}", @@ -86,6 +119,7 @@ def feature_demo_run_workflow( run_workers_proc = None with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send the tasks to the server run_proc = subprocess.run( f"merlin run {feature_demo_path} --vars {' '.join(vars_to_substitute)}", shell=True, @@ -108,5 +142,4 @@ def feature_demo_run_workflow( # Let the workflow try to run for 30 seconds sleep(30) - stdout, stderr = run_workers_proc.communicate() - return stdout, stderr + return run_workers_proc diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index ad029ffb4..24ffe64b1 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -34,7 +34,6 @@ from re import search -# TODO when moving command line tests to pytest, change Condition boolean returns to assertions class Condition(ABC): """Abstract Condition class that other conditions will inherit from""" @@ -131,7 +130,7 @@ def __init__(self, study_name, output_path): """ self.study_name = study_name self.output_path = output_path - self.dirpath_glob = f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*" + self.dirpath_glob = os.path.join(self.output_path, f"{self.study_name}_[0-9]*-[0-9]*") def glob(self, glob_string): """ @@ -154,7 +153,7 @@ class StepFileExists(StudyOutputAware): A StudyOutputAware that checks for a particular file's existence. """ - def __init__(self, step, filename, study_name, output_path, params=False): # pylint: disable=R0913 + def __init__(self, step, filename, study_name, output_path, params=False, samples=False): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -165,6 +164,7 @@ def __init__(self, step, filename, study_name, output_path, params=False): # py self.step = step self.filename = filename self.params = params + self.samples = samples def __str__(self): return f"{__class__.__name__} expected to find file '{self.glob_string}', but file did not exist" @@ -174,10 +174,9 @@ def glob_string(self): """ Returns a regex string for the glob library to recursively find files with. """ - param_glob = "" - if self.params: - param_glob = "*" - return os.path.join(self.dirpath_glob, self.step, param_glob, self.filename) + param_glob = "*" if self.params else "" + samples_glob = "**" if self.samples else "" + return os.path.join(self.dirpath_glob, self.step, param_glob, samples_glob, self.filename) def file_exists(self): """Check if the file path created by glob_string exists""" @@ -243,6 +242,90 @@ def passes(self): return self.contains() +# TODO when writing API docs for tests make sure this looks correct and has functioning links +# - Do we want to list expected_count, glob_string, and passes as methods since they're already attributes? +class StepFinishedFilesCount(StudyOutputAware): + """ + A [`StudyOutputAware`][integration.conditions.StudyOutputAware] that checks for the + exact number of `MERLIN_FINISHED` files in a specified step's output directory based + on the number of parameters and samples. + + Attributes: + step: The name of the step to check. + study_name: The name of the study. + output_path: The output path of the study. + num_parameters: The number of parameters for the step. + num_samples: The number of samples for the step. + expected_count: The expected number of `MERLIN_FINISHED` files based on parameters and samples. + glob_string: The glob pattern to find `MERLIN_FINISHED` files in the specified step's output directory. + passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. + + Methods: + expected_count: Calculates the expected number of `MERLIN_FINISHED` files. + glob_string: Constructs the glob pattern for searching `MERLIN_FINISHED` files. + count_finished_files: Counts the number of `MERLIN_FINISHED` files found. + passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. + """ + + def __init__(self, step: str, study_name: str, output_path: str, num_parameters: int = 0, num_samples: int = 0): + super().__init__(study_name, output_path) + self.step = step + self.num_parameters = num_parameters + self.num_samples = num_samples + + @property + def expected_count(self) -> int: + """ + Calculate the expected number of `MERLIN_FINISHED` files. + + Returns: + The expected number of `MERLIN_FINISHED` files. + """ + if self.num_parameters > 0 and self.num_samples > 0: + return self.num_parameters * self.num_samples + elif self.num_parameters > 0: + return self.num_parameters + elif self.num_samples > 0: + return self.num_samples + else: + return 1 # Default case when there are no parameters or samples + + @property + def glob_string(self) -> str: + """ + Glob pattern to find `MERLIN_FINISHED` files in the specified step's output directory. + + Returns: + A glob pattern to find `MERLIN_FINISHED` files. + """ + param_glob = "*" if self.num_parameters > 0 else "" + samples_glob = "**" if self.num_samples > 0 else "" + return os.path.join(self.dirpath_glob, self.step, param_glob, samples_glob, "MERLIN_FINISHED") + + def count_finished_files(self) -> int: + """ + Count the number of `MERLIN_FINISHED` files found. + + Returns: + The actual number of `MERLIN_FINISHED` files that exist in the step's output directory. + """ + finished_files = glob(self.glob_string) # Adjust the glob pattern as needed + return len(finished_files) + + @property + def passes(self) -> bool: + """ + Check if the count of `MERLIN_FINISHED` files matches the expected count. + + Returns: + True if the expected count matches the actual count. False otherwise. + """ + return self.count_finished_files() == self.expected_count + + def __str__(self) -> str: + return f"{__class__.__name__} expected {self.expected_count} `MERLIN_FINISHED` files, but found {self.count_finished_files()}" + + class ProvenanceYAMLFileHasRegex(HasRegex): """ A condition that a Merlin provenance yaml spec in the 'merlin_info' directory diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py index fc976a68e..d22989eff 100644 --- a/tests/integration/helper_funcs.py +++ b/tests/integration/helper_funcs.py @@ -10,6 +10,7 @@ import yaml +from merlin.spec.expansion import get_spec_with_expansion from tests.integration.conditions import Condition @@ -19,33 +20,29 @@ def load_workers_from_spec(spec_filepath: str) -> dict: This function reads a YAML file containing study specifications and extracts the worker information under the "merlin" section. It - constructs a dictionary in the form that CeleryWorkersManager.launch_workers + constructs a dictionary in the form that + [`CeleryWorkersManager.launch_workers`][context_managers.celery_workers_manager.CeleryWorkersManager.launch_workers] requires. - Parameters: + Args: spec_filepath: The file path to the YAML specification file. Returns: A dictionary containing the worker specifications from the "merlin" section of the YAML file. """ - # Read in the contents of the spec file - with open(spec_filepath, "r") as spec_file: - spec_contents = yaml.load(spec_file, yaml.Loader) - - # Initialize an empty dictionary to hold worker_info worker_info = {} + spec = get_spec_with_expansion(spec_filepath) + steps_and_queues = spec.get_task_queues(omit_tag=True) - # Access workers and steps from spec_contents - workers = spec_contents["merlin"]["resources"]["workers"] - study_steps = {step["name"]: step["run"]["task_queue"] for step in spec_contents["study"]} - - # Grab the concurrency and queues from each worker and add it to the worker_info dict - for worker_name, worker_settings in workers.items(): + for worker_name, worker_settings in spec.merlin["resources"]["workers"].items(): match = re.search(r"--concurrency\s+(\d+)", worker_settings["args"]) concurrency = int(match.group(1)) if match else 1 - queues = [study_steps[step] for step in worker_settings["steps"]] - worker_info[worker_name] = {"concurrency": concurrency, "queues": queues} + worker_info[worker_name] = {"concurrency": concurrency} + if worker_settings["steps"] == ["all"]: + worker_info[worker_name]["queues"] = list(steps_and_queues.values()) + else: + worker_info[worker_name]["queues"] = [steps_and_queues[step] for step in worker_settings["steps"]] return worker_info @@ -59,10 +56,9 @@ def copy_app_yaml_to_cwd(merlin_server_dir: str): working directory so that Merlin will read this in as the server configuration for whatever test is calling this. - Parameters: - merlin_server_dir: - The path to the `merlin_server` directory that should be created by the - `redis_server` fixture. + Args: + merlin_server_dir: The path to the `merlin_server` directory that should be created by the + [`redis_server`][conftest.redis_server] fixture. """ copied_app_yaml = os.path.join(os.getcwd(), "app.yaml") if not os.path.exists(copied_app_yaml): @@ -75,26 +71,24 @@ def check_test_conditions(conditions: List[Condition], info: Dict[str, str]): Ensure all specified test conditions are satisfied based on the output from a subprocess. - This function iterates through a list of `Condition` instances, ingests - the provided information (stdout, stderr, and return code) for each - condition, and checks if each condition passes. If any condition fails, - an AssertionError is raised with a detailed message that includes the - condition that failed, along with the captured output and return code. - - Parameters: - conditions: - A list of Condition instances that define the expectations for the test. - info: - A dictionary containing the output from the subprocess, which should - include the following keys: + This function iterates through a list of [`Condition`][integration.conditions.Condition] + instances, ingests the provided information (stdout, stderr, and return + code) for each condition, and checks if each condition passes. If any + condition fails, an AssertionError is raised with a detailed message that + includes the condition that failed, along with the captured output and + return code. + + Args: + conditions: A list of Condition instances that define the expectations for the test. + info: A dictionary containing the output from the subprocess, which should + include the following keys:\n - 'stdout': The standard output captured from the subprocess. - 'stderr': The standard error output captured from the subprocess. - 'return_code': The return code of the subprocess, indicating success - or failure of the command executed. + or failure of the command executed. Raises: - AssertionError - If any of the conditions do not pass, an AssertionError is raised with + AssertionError: If any of the conditions do not pass, an AssertionError is raised with a detailed message including the failed condition and the subprocess output. """ diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py index 09d95b60a..2688b40aa 100644 --- a/tests/integration/workflows/test_feature_demo.py +++ b/tests/integration/workflows/test_feature_demo.py @@ -1,57 +1,106 @@ """ This module contains tests for the feature_demo workflow. """ -import inspect -import os -import signal import subprocess -from time import sleep -from subprocess import TimeoutExpired -# from tests.context_managers.celery_task_manager import CeleryTaskManager -# from tests.context_managers.celery_workers_manager import CeleryWorkersManager -from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr, FixtureTuple -from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec +from tests.fixture_types import FixtureInt, FixtureStr +from tests.integration.conditions import StepFinishedFilesCount -# NOTE maybe this workflow only needs to run twice? -# - once for a normal run -# - can test e2e, data passing, and step execution order with a single run -# - another time for error testing - class TestFeatureDemo: """ Tests for the feature_demo workflow. """ - demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") - - def get_test_name(self): - """ - """ - stack = inspect.stack() - return stack[1].function def test_end_to_end_run( self, feature_demo_testing_dir: FixtureStr, feature_demo_num_samples: FixtureInt, feature_demo_name: FixtureStr, - feature_demo_run_workflow: FixtureTuple[str, str], + feature_demo_run_workflow: subprocess.CompletedProcess, ): """ Test that the workflow runs from start to finish with no problems. - """ - # TODO check if the workflow ran to completion in 30 seconds - # if not assert a failure happened - # - to check, see if all MERLIN_FINISHED files exist - StepFinishedFilesCount( - step="hello", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, - num_parameters=1, - num_samples=feature_demo_num_samples, - ) + This will check that each step has the proper amount of `MERLIN_FINISHED` files. + The workflow will be run via the + [`feature_demo_run_workflow`][fixtures.feature_demo.feature_demo_run_workflow] + fixture. + + Args: + feature_demo_testing_dir: The directory containing the output of the feature + demo run. + feature_demo_num_samples: The number of samples we give to the feature demo run. + feature_demo_name: The name of the feature demo study. + feature_demo_run_workflow: A fixture to run the feature demo study. + """ + conditions = [ + StepFinishedFilesCount( + step="hello", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=feature_demo_num_samples, + ), + StepFinishedFilesCount( + step="python2_hello", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="python3_hello", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="collect", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="translate", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="learn", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="make_new_samples", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="predict", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="verify", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + num_parameters=1, + num_samples=0, + ), + ] + for condition in conditions: + assert condition.passes def test_step_execution_order(self): """ From 983215b2e3c42bd1361a87fe277053a514746a7d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 31 Oct 2024 15:32:46 -0700 Subject: [PATCH 184/201] add check for proper variable substitution in e2e test --- tests/integration/workflows/test_feature_demo.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py index 2688b40aa..329c578ed 100644 --- a/tests/integration/workflows/test_feature_demo.py +++ b/tests/integration/workflows/test_feature_demo.py @@ -4,7 +4,7 @@ import subprocess from tests.fixture_types import FixtureInt, FixtureStr -from tests.integration.conditions import StepFinishedFilesCount +from tests.integration.conditions import ProvenanceYAMLFileHasRegex, StepFinishedFilesCount class TestFeatureDemo: @@ -35,7 +35,14 @@ def test_end_to_end_run( feature_demo_run_workflow: A fixture to run the feature demo study. """ conditions = [ - StepFinishedFilesCount( + ProvenanceYAMLFileHasRegex( # This condition will check that variable substitution worked + regex=f"N_SAMPLES: {feature_demo_num_samples}", + spec_file_name="feature_demo", + study_name=feature_demo_name, + output_path=feature_demo_testing_dir, + provenance_type="expanded", + ), + StepFinishedFilesCount( # The rest of the conditions will ensure every step ran to completion step="hello", study_name=feature_demo_name, output_path=feature_demo_testing_dir, From 977e91cd23dc9058a65ae95032ddb95e2be44d68 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 4 Nov 2024 15:00:46 -0800 Subject: [PATCH 185/201] generalize functionality to run workflows --- tests/fixtures/feature_demo.py | 43 ++----------- tests/integration/helper_funcs.py | 61 ++++++++++++++++++- .../workflows/test_feature_demo.py | 31 +++++----- 3 files changed, 81 insertions(+), 54 deletions(-) diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py index 573e54f49..7de88c8e6 100644 --- a/tests/fixtures/feature_demo.py +++ b/tests/fixtures/feature_demo.py @@ -3,14 +3,11 @@ """ import os import subprocess -from time import sleep import pytest -from tests.context_managers.celery_task_manager import CeleryTaskManager -from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr -from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec +from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow @pytest.fixture(scope="session") @@ -74,9 +71,7 @@ def feature_demo_run_workflow( This fixture sets up and executes the feature demo workflow using the specified configurations and parameters. It prepares the environment by modifying the CONFIG object to connect to a - Redis server and runs the demo workflow with the provided sample size and name. It utilizes - context managers to safely send tasks to the server and start up workers. The workflow is given - 30 seconds to complete which should be plenty of time. + Redis server and runs the demo workflow with the provided sample size and name. Args: redis_client: A fixture that connects us to a redis client that we can interact with. @@ -100,10 +95,7 @@ def feature_demo_run_workflow( Returns: The completed process object containing information about the execution of the workflow, including return code, stdout, and stderr. - """ - # TODO might want to generalize the logic in this function into a new function that runs workflows - from merlin.celery import app as celery_app - + """ # Setup the test copy_app_yaml_to_cwd(merlin_server_dir) demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") @@ -116,30 +108,5 @@ def feature_demo_run_workflow( f"OUTPUT_PATH={feature_demo_testing_dir}" ] - run_workers_proc = None - - with CeleryTaskManager(celery_app, redis_client) as CTM: - # Send the tasks to the server - run_proc = subprocess.run( - f"merlin run {feature_demo_path} --vars {' '.join(vars_to_substitute)}", - shell=True, - capture_output=True, - text=True, - ) - - # We use a context manager to start workers so that they'll safely stop even if this test fails - with CeleryWorkersManager(celery_app) as CWM: - # Start the workers then add them to the context manager so they can be stopped safely later - run_workers_proc = subprocess.Popen( - f"merlin run-workers {feature_demo_path}".split(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - start_new_session=True - ) - CWM.add_run_workers_process(run_workers_proc.pid) - - # Let the workflow try to run for 30 seconds - sleep(30) - - return run_workers_proc + # Run the workflow + return run_workflow(redis_client, feature_demo_path, vars_to_substitute) diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py index d22989eff..268687313 100644 --- a/tests/integration/helper_funcs.py +++ b/tests/integration/helper_funcs.py @@ -2,15 +2,19 @@ This module contains helper functions for the integration test suite. """ - import os import re import shutil +import subprocess +from time import sleep from typing import Dict, List import yaml from merlin.spec.expansion import get_spec_with_expansion +from tests.context_managers.celery_task_manager import CeleryTaskManager +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.fixture_types import FixtureRedis from tests.integration.conditions import Condition @@ -104,3 +108,58 @@ def check_test_conditions(conditions: List[Condition], info: Dict[str, str]): f"Return code: {info['return_code']}\n" ) raise AssertionError(error_message) from exc + + +def run_workflow(redis_client: FixtureRedis, workflow_path: str, vars_to_substitute: List[str]) -> subprocess.CompletedProcess: + """ + Run a Merlin workflow using the `merlin run` and `merlin run-workers` commands. + + This function executes a Merlin workflow using a specified path to a study and variables to + configure the study with. It utilizes context managers to safely send tasks to the server + and start up workers. The tasks are given 15 seconds to be sent to the server. Once tasks + exist on the server, the workflow is given 30 seconds to run to completion, which should be + plenty of time. + + Args: + redis_client: A fixture that connects us to a redis client that we can interact with. + workflow_path: The path to the study that we're going to run here + vars_to_substitute: A list of variables in the form ["VAR_NAME=var_value"] to be modified + in the workflow. + + Returns: + The completed process object containing information about the execution of the workflow, including + return code, stdout, and stderr. + """ + from merlin.celery import app as celery_app + + run_workers_proc = None + + with CeleryTaskManager(celery_app, redis_client) as CTM: + # Send the tasks to the server + try: + run_proc = subprocess.run( + f"merlin run {workflow_path} --vars {' '.join(vars_to_substitute)}", + shell=True, + capture_output=True, + text=True, + timeout=15, + ) + except subprocess.TimeoutExpired as exc: + raise TimeoutError("Could not send tasks to the server within the allotted time.") from exc + + # We use a context manager to start workers so that they'll safely stop even if this test fails + with CeleryWorkersManager(celery_app) as CWM: + # Start the workers then add them to the context manager so they can be stopped safely later + run_workers_proc = subprocess.Popen( + f"merlin run-workers {workflow_path}".split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True + ) + CWM.add_run_workers_process(run_workers_proc.pid) + + # Let the workflow try to run for 30 seconds + sleep(30) + + return run_workers_proc diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py index 329c578ed..9d10b9ce6 100644 --- a/tests/integration/workflows/test_feature_demo.py +++ b/tests/integration/workflows/test_feature_demo.py @@ -109,21 +109,22 @@ def test_end_to_end_run( for condition in conditions: assert condition.passes - def test_step_execution_order(self): - """ - Test that steps are executed in the correct order. - """ - # TODO build a list with the correct order that steps should be ran - # TODO compare the list against the logs from the worker + # TODO implement the below tests + # def test_step_execution_order(self): + # """ + # Test that steps are executed in the correct order. + # """ + # # TODO build a list with the correct order that steps should be ran + # # TODO compare the list against the logs from the worker - def test_workflow_error_handling(self): - """ - Test the behavior when errors arise during the worfklow. + # def test_workflow_error_handling(self): + # """ + # Test the behavior when errors arise during the worfklow. - TODO should this test both soft and hard fails? should this test all return codes? - """ + # TODO should this test both soft and hard fails? should this test all return codes? + # """ - def test_data_passing(self): - """ - Test that data can be successfully passed between steps using built-in Merlin variables. - """ + # def test_data_passing(self): + # """ + # Test that data can be successfully passed between steps using built-in Merlin variables. + # """ From ba3f066d97c723c1ca53d7ed93e9811b9e45f3bb Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 4 Nov 2024 16:02:14 -0800 Subject: [PATCH 186/201] add create_testing_dir fixture --- tests/conftest.py | 30 +++++++++++++++++++++++++++++- tests/fixture_types.py | 3 ++- tests/fixtures/examples.py | 19 ++++++++++--------- tests/fixtures/feature_demo.py | 11 ++++------- tests/fixtures/run_command.py | 15 ++++++--------- tests/fixtures/server.py | 18 +++++++++--------- tests/fixtures/status.py | 18 +++++++++--------- 7 files changed, 69 insertions(+), 45 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e17b0723..d287e4e73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,7 +35,7 @@ from copy import copy from glob import glob from time import sleep -from typing import Dict +from typing import Callable, Dict import pytest import yaml @@ -169,6 +169,34 @@ def path_to_merlin_codebase() -> FixtureStr: return os.path.join(path_to_test_dir, "..", "merlin") +@pytest.fixture(scope="session") +def create_testing_dir() -> Callable: + """ + Fixture to create a temporary testing directory. + + Returns: + A function that creates the testing directory. + """ + + def _create_testing_dir(base_dir: str, sub_dir: str) -> str: + """ + Helper function to create a temporary testing directory. + + Args: + base_dir: The base directory where the testing directory will be created. + sub_dir: The name of the subdirectory to create. + + Returns: + The path to the created testing directory. + """ + testing_dir = os.path.join(base_dir, sub_dir) + if not os.path.exists(testing_dir): + os.makedirs(testing_dir) # Use makedirs to create intermediate directories if needed + return testing_dir + + return _create_testing_dir + + @pytest.fixture(scope="session") def temp_output_dir(tmp_path_factory: TempPathFactory) -> FixtureStr: """ diff --git a/tests/fixture_types.py b/tests/fixture_types.py index 5d9902491..93229bdf0 100644 --- a/tests/fixture_types.py +++ b/tests/fixture_types.py @@ -20,7 +20,7 @@ from celery import Celery from celery.canvas import Signature from redis import Redis -from typing import Any, Annotated, Dict, Tuple, TypeVar +from typing import Any, Annotated, Callable, Dict, Tuple, TypeVar # TODO convert unit test type hinting to use these # - likely will do this when I work on API docs for test library @@ -29,6 +29,7 @@ V = TypeVar('V') FixtureBytes = Annotated[bytes, pytest.fixture] +FixtureCallable = Annotated[Callable, pytest.fixture] FixtureCelery = Annotated[Celery, pytest.fixture] FixtureDict = Annotated[Dict[K, V], pytest.fixture] FixtureInt = Annotated[int, pytest.fixture] diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py index 949234577..3fe444c1d 100644 --- a/tests/fixtures/examples.py +++ b/tests/fixtures/examples.py @@ -3,22 +3,23 @@ """ import os +from typing import Callable import pytest -from tests.fixture_types import FixtureStr +from tests.fixture_types import FixtureCallable, FixtureStr @pytest.fixture(scope="session") -def examples_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: +def examples_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to the examples functionality. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: The path to the temporary testing directory for examples tests - """ - testing_dir = f"{temp_output_dir}/examples_testing" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary output directory we'll be using for this test run. - return testing_dir + Returns: + The path to the temporary testing directory for examples tests. + """ + return create_testing_dir(temp_output_dir, "examples_testing") diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py index 7de88c8e6..ec9ccdb61 100644 --- a/tests/fixtures/feature_demo.py +++ b/tests/fixtures/feature_demo.py @@ -6,27 +6,24 @@ import pytest -from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr +from tests.fixture_types import FixtureCallable, FixtureInt, FixtureModification, FixtureRedis, FixtureStr from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow @pytest.fixture(scope="session") -def feature_demo_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: +def feature_demo_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to testing the feature_demo workflow. Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. Returns: The path to the temporary testing directory for feature_demo workflow tests. """ - testing_dir = f"{temp_output_dir}/feature_demo_testing" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) - - return testing_dir + return create_testing_dir(temp_output_dir, "feature_demo_testing") @pytest.fixture(scope="session") diff --git a/tests/fixtures/run_command.py b/tests/fixtures/run_command.py index bdf0ff13d..842f03fca 100644 --- a/tests/fixtures/run_command.py +++ b/tests/fixtures/run_command.py @@ -5,23 +5,20 @@ import pytest -from tests.fixture_types import FixtureStr +from tests.fixture_types import FixtureCallable, FixtureStr @pytest.fixture(scope="session") -def run_command_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: +def run_command_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to testing the `merlin run` functionality. Args: - temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. Returns: - The path to the temporary testing directory for `merlin run` tests + The path to the temporary testing directory for `merlin run` tests. """ - testing_dir = f"{temp_output_dir}/run_command_testing" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) - - return testing_dir + return create_testing_dir(temp_output_dir, "run_command_testing") diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py index cd641aa30..c2bcdc762 100644 --- a/tests/fixtures/server.py +++ b/tests/fixtures/server.py @@ -9,25 +9,25 @@ import pytest import yaml -from tests.fixture_types import FixtureDict, FixtureNamespace, FixtureStr +from tests.fixture_types import FixtureCallable, FixtureDict, FixtureNamespace, FixtureStr # pylint: disable=redefined-outer-name @pytest.fixture(scope="session") -def server_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: +def server_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ Fixture to create a temporary output directory for tests related to the server functionality. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: The path to the temporary testing directory for server tests - """ - testing_dir = f"{temp_output_dir}/server_testing" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - return testing_dir + Returns: + The path to the temporary testing directory for server tests. + """ + return create_testing_dir(temp_output_dir, "server_testing") @pytest.fixture(scope="session") diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index 166d27eb5..57ff16bac 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -11,7 +11,7 @@ import pytest import yaml -from tests.fixture_types import FixtureNamespace, FixtureStr +from tests.fixture_types import FixtureCallable, FixtureNamespace, FixtureStr from tests.unit.study.status_test_files import status_test_variables @@ -19,18 +19,18 @@ @pytest.fixture(scope="session") -def status_testing_dir(temp_output_dir: FixtureStr) -> FixtureStr: +def status_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ A pytest fixture to set up a temporary directory to write files to for testing status. - :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :returns: The path to the temporary testing directory for status testing - """ - testing_dir = f"{temp_output_dir}/status_testing/" - if not os.path.exists(testing_dir): - os.mkdir(testing_dir) + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - return testing_dir + Returns: + The path to the temporary testing directory for status tests. + """ + return create_testing_dir(temp_output_dir, "status_testing") @pytest.fixture(scope="class") From b30d796e3df5be12d6aca8b5a8702485f2b0d5bc Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 5 Nov 2024 09:05:27 -0800 Subject: [PATCH 187/201] port chord error workflow to pytest --- tests/fixtures/chord_err.py | 90 +++++++++++++++++++ tests/integration/conditions.py | 18 ++-- tests/integration/definitions.py | 55 ------------ tests/integration/test_specs/chord_err.yaml | 3 +- .../integration/workflows/test_chord_error.py | 62 +++++++++++++ 5 files changed, 166 insertions(+), 62 deletions(-) create mode 100644 tests/fixtures/chord_err.py create mode 100644 tests/integration/workflows/test_chord_error.py diff --git a/tests/fixtures/chord_err.py b/tests/fixtures/chord_err.py new file mode 100644 index 000000000..4ae0934ed --- /dev/null +++ b/tests/fixtures/chord_err.py @@ -0,0 +1,90 @@ +""" +Fixtures specifically for help testing the chord_err workflow. +""" +import os +import subprocess + +import pytest + +from tests.fixture_types import FixtureCallable, FixtureModification, FixtureRedis, FixtureStr +from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow + + +@pytest.fixture(scope="session") +def chord_err_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: + """ + Fixture to create a temporary output directory for tests related to testing the + chord_err workflow. + + Args: + create_testing_dir: A fixture which returns a function that creates the testing directory. + temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. + + Returns: + The path to the temporary testing directory for chord_err workflow tests. + """ + return create_testing_dir(temp_output_dir, "chord_err_testing") + + +@pytest.fixture(scope="session") +def chord_err_name() -> FixtureStr: + """ + Defines a specific name to use for the chord_err workflow. This helps ensure + that even if changes were made to the chord_err workflow, tests using this fixture + should still run the same thing. + + Returns: + A string representing the name to use for the chord_err workflow. + """ + return "chord_err_test" + + +@pytest.fixture(scope="class") +def chord_err_run_workflow( + redis_client: FixtureRedis, + redis_results_backend_config_class: FixtureModification, + redis_broker_config_class: FixtureModification, + path_to_test_specs: FixtureStr, + merlin_server_dir: FixtureStr, + chord_err_testing_dir: FixtureStr, + chord_err_name: FixtureStr, +) -> subprocess.CompletedProcess: + """ + Run the chord error workflow. + + This fixture sets up and executes the chord error workflow using the specified configurations + and parameters. It prepares the environment by modifying the CONFIG object to connect to a + Redis server and runs the workflow with the provided name and output path. + + Args: + redis_client: A fixture that connects us to a redis client that we can interact with. + redis_results_backend_config_class: A fixture that modifies the CONFIG object so that it + points the results backend configuration to the containerized redis server we start up + with the [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin + uses to connect to a server. + redis_broker_config_class: A fixture that modifies the CONFIG object so that it points + the broker configuration to the containerized redis server we start up with the + [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin uses + to connect to a server. + path_to_test_specs: A fixture to provide the path to the directory containing test specifications. + merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be + created by the [`redis_server`][conftest.redis_server] fixture. + chord_err_testing_dir: The path to the temporary testing directory for chord_err workflow tests. + chord_err_name: A string representing the name to use for the chord_err workflow. + + Returns: + The completed process object containing information about the execution of the workflow, including + return code, stdout, and stderr. + """ + # Setup the test + copy_app_yaml_to_cwd(merlin_server_dir) + chord_err_path = os.path.join(path_to_test_specs, "chord_err.yaml") + + # Create the variables to pass in to the workflow + vars_to_substitute = [ + f"NAME={chord_err_name}", + f"OUTPUT_PATH={chord_err_testing_dir}" + ] + + # Run the workflow + return run_workflow(redis_client, chord_err_path, vars_to_substitute) diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 24ffe64b1..04ec8fa28 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -256,7 +256,7 @@ class StepFinishedFilesCount(StudyOutputAware): output_path: The output path of the study. num_parameters: The number of parameters for the step. num_samples: The number of samples for the step. - expected_count: The expected number of `MERLIN_FINISHED` files based on parameters and samples. + expected_count: The expected number of `MERLIN_FINISHED` files based on parameters and samples or explicitly set. glob_string: The glob pattern to find `MERLIN_FINISHED` files in the specified step's output directory. passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. @@ -267,11 +267,12 @@ class StepFinishedFilesCount(StudyOutputAware): passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. """ - def __init__(self, step: str, study_name: str, output_path: str, num_parameters: int = 0, num_samples: int = 0): + def __init__(self, step: str, study_name: str, output_path: str, num_parameters: int = 0, num_samples: int = 0, expected_count: int = None): super().__init__(study_name, output_path) self.step = step self.num_parameters = num_parameters self.num_samples = num_samples + self._expected_count = expected_count @property def expected_count(self) -> int: @@ -281,14 +282,19 @@ def expected_count(self) -> int: Returns: The expected number of `MERLIN_FINISHED` files. """ + # Return the explicitly set expected count if given + if self._expected_count is not None: + return self._expected_count + + # Otherwise calculate the correct number of MERLIN_FINISHED files to expect if self.num_parameters > 0 and self.num_samples > 0: return self.num_parameters * self.num_samples - elif self.num_parameters > 0: + if self.num_parameters > 0: return self.num_parameters - elif self.num_samples > 0: + if self.num_samples > 0: return self.num_samples - else: - return 1 # Default case when there are no parameters or samples + + return 1 # Default case when there are no parameters or samples @property def glob_string(self) -> str: diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index eeb23fc71..a6f66020c 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -647,59 +647,6 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } - distributed_tests = { # noqa: F841 - "run and purge feature_demo": { - "cmds": f"{run} {demo}; {purge} {demo} -f", - "conditions": HasReturnCode(), - "run type": "distributed", - }, - "remote feature_demo": { - "cmds": f"""{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers; - {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers""", - "conditions": [ - HasReturnCode(), - ProvenanceYAMLFileHasRegex( - regex="cli_test_demo_workers:", - spec_file_name="remote_feature_demo", - study_name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - ), - StepFileExists( - "verify", - "MERLIN_FINISHED", - "feature_demo", - OUTPUT_DIR, - params=True, - ), - ], - "run type": "distributed", - }, - } - distributed_error_checks = { - "check chord error continues wf": { - "cmds": [ - f"{workers} {chord_err_wf} --vars OUTPUT_PATH=./{OUTPUT_DIR}", - f"{run} {chord_err_wf} --vars OUTPUT_PATH=./{OUTPUT_DIR}; sleep 40; tree {OUTPUT_DIR}", - ], - "conditions": [ - HasReturnCode(), - PathExists( # Check that the sample that's supposed to raise an error actually raises an error - f"{OUTPUT_DIR}/process_samples/01/MERLIN_FINISHED", - negate=True, - ), - StepFileExists( # Check that step 3 is actually started and completes - "step_3", - "MERLIN_FINISHED", - "chord_err", - OUTPUT_DIR, - ), - ], - "run type": "distributed", - "cleanup": KILL_WORKERS, - "num procs": 2, - } - } # combine and return test dictionaries all_tests = {} @@ -719,8 +666,6 @@ def define_tests(): # pylint: disable=R0914,R0915 # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, - distributed_tests, - distributed_error_checks, ]: all_tests.update(test_dict) diff --git a/tests/integration/test_specs/chord_err.yaml b/tests/integration/test_specs/chord_err.yaml index 3da99ae03..9fe7d55ea 100644 --- a/tests/integration/test_specs/chord_err.yaml +++ b/tests/integration/test_specs/chord_err.yaml @@ -1,10 +1,11 @@ description: - name: chord_err + name: $(NAME) description: test the chord err problem env: variables: OUTPUT_PATH: ./studies + NAME: chord_err global.parameters: TEST_PARAM: diff --git a/tests/integration/workflows/test_chord_error.py b/tests/integration/workflows/test_chord_error.py new file mode 100644 index 000000000..46530a832 --- /dev/null +++ b/tests/integration/workflows/test_chord_error.py @@ -0,0 +1,62 @@ +""" +This module contains tests for the feature_demo workflow. +""" +import subprocess + +from tests.fixture_types import FixtureStr +from tests.integration.conditions import HasRegex, StepFinishedFilesCount +from tests.integration.helper_funcs import check_test_conditions + +class TestChordError: + """ + Tests for the chord error workflow. + """ + + def test_chord_error_continues( + self, + chord_err_testing_dir: FixtureStr, + chord_err_name: FixtureStr, + chord_err_run_workflow: subprocess.CompletedProcess + ): + """ + Test that this workflow continues through to the end of its execution, even + though a ChordError will be raised. + + Args: + chord_err_testing_dir: The directory containing the output of the chord error run. + chord_err_name: The name of the chord error study. + chord_err_run_workflow: A fixture to run the chord error study. + """ + + conditions = [ + HasRegex("Exception raised by request from the user"), + StepFinishedFilesCount( # Check that the `process_samples` step has only 2 MERLIN_FINISHED files + step="process_samples", + study_name=chord_err_name, + output_path=chord_err_testing_dir, + expected_count=2, + num_samples=3, + ), + StepFinishedFilesCount( # Check that the `samples_and_params` step has all of its MERLIN_FINISHED files + step="samples_and_params", + study_name=chord_err_name, + output_path=chord_err_testing_dir, + num_parameters=2, + num_samples=3, + ), + StepFinishedFilesCount( # Check that the final step has a MERLIN_FINISHED file + step="step_3", + study_name=chord_err_name, + output_path=chord_err_testing_dir, + num_parameters=0, + num_samples=0, + ), + ] + + info = { + "return_code": chord_err_run_workflow.returncode, + "stdout": chord_err_run_workflow.stdout.read(), + "stderr": chord_err_run_workflow.stderr.read(), + } + + check_test_conditions(conditions, info) From 8b76db846dc026468432ab7511519edc32e2b96c Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 7 Nov 2024 16:10:47 -0800 Subject: [PATCH 188/201] create dataclasses to house common fixtures and reduce fixture import requirements --- tests/fixture_data_classes.py | 76 ++++++++ tests/fixtures/chord_err.py | 73 +++++--- tests/fixtures/feature_demo.py | 84 +++++---- tests/integration/commands/pgen.py | 29 ++-- tests/integration/commands/test_purge.py | 163 ++++++------------ tests/integration/commands/test_run.py | 163 ++++++------------ .../commands/test_stop_and_query_workers.py | 100 ++--------- .../integration/workflows/test_chord_error.py | 24 +-- .../workflows/test_feature_demo.py | 60 +++---- 9 files changed, 354 insertions(+), 418 deletions(-) create mode 100644 tests/fixture_data_classes.py diff --git a/tests/fixture_data_classes.py b/tests/fixture_data_classes.py new file mode 100644 index 000000000..588a21aa2 --- /dev/null +++ b/tests/fixture_data_classes.py @@ -0,0 +1,76 @@ +""" +This module houses dataclasses to be used with pytest fixtures. +""" +from dataclasses import dataclass + +from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr + + +@dataclass +class RedisBrokerAndBackend: + """ + Data class to encapsulate all Redis-related fixtures required for + establishing connections to Redis for both the broker and backend. + + This class simplifies the management of Redis fixtures by grouping + them into a single object, reducing the number of individual fixture + imports needed in tests that require Redis functionality. + + Attributes: + client: A fixture that provides a client for interacting + with the Redis server. + server: A fixture providing the connection string to the + Redis server instance. + broker_config: A fixture that modifies the configuration + to point to the Redis server used as the message broker. + results_backend_config: A fixture that modifies the + configuration to point to the Redis server used for storing + results. + """ + client: FixtureRedis + server: FixtureStr + results_backend_config: FixtureModification + broker_config: FixtureModification + + +@dataclass +class FeatureDemoSetup: + """ + Data class to encapsulate all feature-demo-related fixtures required + for testing the feature demo workflow. + + This class simplifies the management of feature demo setup fixtures + by grouping them into a single object, reducing the number of individual + fixture imports needed in tests that require feature demo setup. + + Attributes: + testing_dir: The path to the temp output directory for feature_demo workflow tests. + num_samples: An integer representing the number of samples to use in the feature_demo + workflow. + name: A string representing the name to use for the feature_demo workflow. + path: The path to the feature demo YAML file. + """ + testing_dir: FixtureStr + num_samples: FixtureInt + name: FixtureStr + path: FixtureStr + + +@dataclass +class ChordErrorSetup: + """ + Data class to encapsulate all chord-error-related fixtures required + for testing the chord error workflow. + + This class simplifies the management of chord error setup fixtures + by grouping them into a single object, reducing the number of individual + fixture imports needed in tests that require chord error setup. + + Attributes: + testing_dir: The path to the temp output directory for chord_err workflow tests. + name: A string representing the name to use for the chord_err workflow. + path: The path to the chord error YAML file. + """ + testing_dir: FixtureStr + name: FixtureStr + path: FixtureStr diff --git a/tests/fixtures/chord_err.py b/tests/fixtures/chord_err.py index 4ae0934ed..3e93f3f77 100644 --- a/tests/fixtures/chord_err.py +++ b/tests/fixtures/chord_err.py @@ -1,11 +1,13 @@ """ Fixtures specifically for help testing the chord_err workflow. """ + import os import subprocess import pytest +from tests.fixture_data_classes import ChordErrorSetup, RedisBrokerAndBackend from tests.fixture_types import FixtureCallable, FixtureModification, FixtureRedis, FixtureStr from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow @@ -19,7 +21,7 @@ def chord_err_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: Args: create_testing_dir: A fixture which returns a function that creates the testing directory. temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - + Returns: The path to the temporary testing directory for chord_err workflow tests. """ @@ -39,15 +41,45 @@ def chord_err_name() -> FixtureStr: return "chord_err_test" +@pytest.fixture(scope="session") +def chord_err_setup( + chord_err_testing_dir: FixtureStr, + chord_err_name: FixtureStr, + path_to_test_specs: FixtureStr, +) -> ChordErrorSetup: + """ + Fixture for setting up the environment required for testing the chord error workflow. + + This fixture prepares the necessary configuration and paths for executing tests related + to the chord error workflow. It aggregates the required parameters into a single + [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] data class instance, which + simplifies the management of these parameters in tests. + + Args: + chord_err_testing_dir: The path to the temporary output directory where chord error + workflow tests will store their results. + chord_err_name: A string representing the name to use for the chord error workflow. + path_to_test_specs: The base path to the Merlin test specs directory, which is + used to locate the chord error YAML file. + + Returns: + A [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] instance containing + the testing directory, name, and path to the chord error YAML file, which can + be used in tests that require this setup. + """ + chord_err_path = os.path.join(path_to_test_specs, "chord_err.yaml") + return ChordErrorSetup( + testing_dir=chord_err_testing_dir, + name=chord_err_name, + path=chord_err_path, + ) + + @pytest.fixture(scope="class") def chord_err_run_workflow( - redis_client: FixtureRedis, - redis_results_backend_config_class: FixtureModification, - redis_broker_config_class: FixtureModification, - path_to_test_specs: FixtureStr, + redis_broker_and_backend_class: RedisBrokerAndBackend, + chord_err_setup: ChordErrorSetup, merlin_server_dir: FixtureStr, - chord_err_testing_dir: FixtureStr, - chord_err_name: FixtureStr, ) -> subprocess.CompletedProcess: """ Run the chord error workflow. @@ -57,34 +89,23 @@ def chord_err_run_workflow( Redis server and runs the workflow with the provided name and output path. Args: - redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_class: A fixture that modifies the CONFIG object so that it - points the results backend configuration to the containerized redis server we start up - with the [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin - uses to connect to a server. - redis_broker_config_class: A fixture that modifies the CONFIG object so that it points - the broker configuration to the containerized redis server we start up with the - [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin uses - to connect to a server. - path_to_test_specs: A fixture to provide the path to the directory containing test specifications. + redis_broker_and_backend_class: Fixture for setting up Redis broker and + backend for class-scoped tests. + chord_err_setup: A fixture that returns a [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] + instance. merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be created by the [`redis_server`][conftest.redis_server] fixture. - chord_err_testing_dir: The path to the temporary testing directory for chord_err workflow tests. - chord_err_name: A string representing the name to use for the chord_err workflow. Returns: The completed process object containing information about the execution of the workflow, including return code, stdout, and stderr. - """ + """ # Setup the test copy_app_yaml_to_cwd(merlin_server_dir) - chord_err_path = os.path.join(path_to_test_specs, "chord_err.yaml") + # chord_err_path = os.path.join(path_to_test_specs, "chord_err.yaml") # Create the variables to pass in to the workflow - vars_to_substitute = [ - f"NAME={chord_err_name}", - f"OUTPUT_PATH={chord_err_testing_dir}" - ] + vars_to_substitute = [f"NAME={chord_err_setup.name}", f"OUTPUT_PATH={chord_err_setup.testing_dir}"] # Run the workflow - return run_workflow(redis_client, chord_err_path, vars_to_substitute) + return run_workflow(redis_broker_and_backend_class.client, chord_err_setup.path, vars_to_substitute) diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py index ec9ccdb61..39ef2251f 100644 --- a/tests/fixtures/feature_demo.py +++ b/tests/fixtures/feature_demo.py @@ -1,11 +1,13 @@ """ Fixtures specifically for help testing the feature_demo workflow. """ + import os import subprocess import pytest +from tests.fixture_data_classes import FeatureDemoSetup, RedisBrokerAndBackend from tests.fixture_types import FixtureCallable, FixtureInt, FixtureModification, FixtureRedis, FixtureStr from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow @@ -19,7 +21,7 @@ def feature_demo_testing_dir(create_testing_dir: FixtureCallable, temp_output_di Args: create_testing_dir: A fixture which returns a function that creates the testing directory. temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - + Returns: The path to the temporary testing directory for feature_demo workflow tests. """ @@ -52,16 +54,51 @@ def feature_demo_name() -> FixtureStr: return "feature_demo_test" -@pytest.fixture(scope="class") -def feature_demo_run_workflow( - redis_client: FixtureRedis, - redis_results_backend_config_class: FixtureModification, - redis_broker_config_class: FixtureModification, - path_to_merlin_codebase: FixtureStr, - merlin_server_dir: FixtureStr, +@pytest.fixture(scope="session") +def feature_demo_setup( feature_demo_testing_dir: FixtureStr, feature_demo_num_samples: FixtureInt, feature_demo_name: FixtureStr, + path_to_merlin_codebase: FixtureStr, +) -> FeatureDemoSetup: + """ + Fixture for setting up the environment required for testing the feature demo workflow. + + This fixture prepares the necessary configuration and paths for executing tests related + to the feature demo workflow. It aggregates the required parameters into a single + [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] data class instance, which + simplifies the management of these parameters in tests. + + Args: + feature_demo_testing_dir: The path to the temporary output directory where + feature demo workflow tests will store their results. + feature_demo_num_samples: An integer representing the number of samples + to use in the feature demo workflow. + feature_demo_name: A string representing the name to use for the feature + demo workflow. + path_to_merlin_codebase: The base path to the Merlin codebase, which is + used to locate the feature demo YAML file. + + Returns: + A [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] instance containing + the testing directory, number of samples, name, and path to the feature demo + YAML file, which can be used in tests that require this setup. + """ + demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") + feature_demo_path = os.path.join(path_to_merlin_codebase, demo_workflow) + return FeatureDemoSetup( + testing_dir=feature_demo_testing_dir, + num_samples=feature_demo_num_samples, + name=feature_demo_name, + path=feature_demo_path, + ) + + +@pytest.fixture(scope="class") +def feature_demo_run_workflow( + redis_broker_and_backend_class: RedisBrokerAndBackend, + feature_demo_setup: FeatureDemoSetup, + merlin_server_dir: FixtureStr, ) -> subprocess.CompletedProcess: """ Run the feature demo workflow. @@ -71,39 +108,26 @@ def feature_demo_run_workflow( Redis server and runs the demo workflow with the provided sample size and name. Args: - redis_client: A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_class: A fixture that modifies the CONFIG object so that it - points the results backend configuration to the containerized redis server we start up - with the [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin - uses to connect to a server. - redis_broker_config_class: A fixture that modifies the CONFIG object so that it points - the broker configuration to the containerized redis server we start up with the - [`redis_server`][conftest.redis_server] fixture. The CONFIG object is what merlin uses - to connect to a server. - path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's - core functionality. + redis_broker_and_backend_class: Fixture for setting up Redis broker and + backend for class-scoped tests. + feature_demo_setup: A fixture that returns a [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] + instance. merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be created by the [`redis_server`][conftest.redis_server] fixture. - feature_demo_testing_dir: The path to the temp output directory for feature_demo workflow tests. - feature_demo_num_samples: An integer representing the number of samples to use in the feature_demo - workflow. - feature_demo_name: A string representing the name to use for the feature_demo workflow. Returns: The completed process object containing information about the execution of the workflow, including return code, stdout, and stderr. - """ + """ # Setup the test copy_app_yaml_to_cwd(merlin_server_dir) - demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") - feature_demo_path = os.path.join(path_to_merlin_codebase, demo_workflow) # Create the variables to pass in to the workflow vars_to_substitute = [ - f"N_SAMPLES={feature_demo_num_samples}", - f"NAME={feature_demo_name}", - f"OUTPUT_PATH={feature_demo_testing_dir}" + f"N_SAMPLES={feature_demo_setup.num_samples}", + f"NAME={feature_demo_setup.name}", + f"OUTPUT_PATH={feature_demo_setup.testing_dir}", ] # Run the workflow - return run_workflow(redis_client, feature_demo_path, vars_to_substitute) + return run_workflow(redis_broker_and_backend_class.client, feature_demo_setup.path, vars_to_substitute) diff --git a/tests/integration/commands/pgen.py b/tests/integration/commands/pgen.py index aa846b79b..6973317d6 100644 --- a/tests/integration/commands/pgen.py +++ b/tests/integration/commands/pgen.py @@ -2,36 +2,35 @@ This file contains pgen functionality for testing purposes. It's specifically set up to work with the feature demo example. """ + import random -import itertools as iter from maestrowf.datastructures.core import ParameterGenerator -def get_custom_generator(env, **kwargs): + +# pylint complains about unused argument `env` but it's necessary for Maestro +def get_custom_generator(env, **kwargs): # pylint: disable=unused-argument + """ + Custom parameter generator that's used for testing the `--pgen` flag + of the `merlin run` command. + """ p_gen = ParameterGenerator() # Unpack any pargs passed in - x2_min = int(kwargs.get('X2_MIN', '0')) - x2_max = int(kwargs.get('X2_MAX', '1')) - n_name_min = int(kwargs.get('N_NAME_MIN', '0')) - n_name_max = int(kwargs.get('N_NAME_MAX', '10')) + x2_min = int(kwargs.get("X2_MIN", "0")) + x2_max = int(kwargs.get("X2_MAX", "1")) + n_name_min = int(kwargs.get("N_NAME_MIN", "0")) + n_name_max = int(kwargs.get("N_NAME_MAX", "10")) # We'll only have two parameter entries each just for testing num_points = 2 params = { - "X2": { - "values": [random.uniform(x2_min, x2_max) for _ in range(num_points)], - "label": "X2.%%" - }, - "N_NEW": { - "values": [random.randint(n_name_min, n_name_max) for _ in range(num_points)], - "label": "N_NEW.%%" - } + "X2": {"values": [random.uniform(x2_min, x2_max) for _ in range(num_points)], "label": "X2.%%"}, + "N_NEW": {"values": [random.randint(n_name_min, n_name_max) for _ in range(num_points)], "label": "N_NEW.%%"}, } for key, value in params.items(): p_gen.add_parameter(key, value["values"], value["label"]) return p_gen - diff --git a/tests/integration/commands/test_purge.py b/tests/integration/commands/test_purge.py index 3c7b284d7..69c04c0dd 100644 --- a/tests/integration/commands/test_purge.py +++ b/tests/integration/commands/test_purge.py @@ -2,11 +2,13 @@ This module will contain the testing logic for the `merlin purge` command. """ + import os import subprocess from typing import Dict, List, Tuple, Union from merlin.spec.expansion import get_spec_with_expansion +from tests.fixture_data_classes import RedisBrokerAndBackend from tests.context_managers.celery_task_manager import CeleryTaskManager from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr from tests.integration.conditions import HasRegex, HasReturnCode @@ -26,7 +28,7 @@ def setup_test(self, path_to_merlin_codebase: FixtureStr, merlin_server_dir: Fix 1. Copying the app.yaml file created by the `redis_server` fixture to the cwd so that Merlin can connect to the test server. 2. Obtaining the path to the feature_demo spec that we'll use for these tests. - + Args: path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core @@ -41,15 +43,15 @@ def setup_test(self, path_to_merlin_codebase: FixtureStr, merlin_server_dir: Fix copy_app_yaml_to_cwd(merlin_server_dir) return os.path.join(path_to_merlin_codebase, self.demo_workflow) - def setup_tasks(self, CTM: CeleryTaskManager, spec_file: str) -> Tuple[Dict[str, str], int]: + def setup_tasks(self, celery_task_manager: CeleryTaskManager, spec_file: str) -> Tuple[Dict[str, str], int]: """ Helper method to setup tasks in the specified queues. - + This method sends tasks named 'task_for_{queue}' to each queue defined in the provided spec file and returns the total number of queues that received tasks. Args: - CTM: + celery_task_manager: A context manager for managing Celery tasks, used to send tasks to the server. spec_file: The path to the spec file from which queues will be extracted. @@ -63,7 +65,7 @@ def setup_tasks(self, CTM: CeleryTaskManager, spec_file: str) -> Tuple[Dict[str, queues_in_spec = spec.get_task_queues() for queue in queues_in_spec.values(): - CTM.send_task(f"task_for_{queue}", queue=queue) + celery_task_manager.send_task(f"task_for_{queue}", queue=queue) return queues_in_spec, len(queues_in_spec.values()) @@ -76,7 +78,7 @@ def run_purge( ) -> Dict[str, Union[str, int]]: """ Helper method to run the purge command. - + Args: spec_file: The path to the spec file from which queues will be purged. input_value: Any input we need to send to the subprocess. @@ -87,16 +89,12 @@ def run_purge( The result from executing the command in a subprocess. """ purge_cmd = ( - "merlin purge" + (" -f" if force else "") + f" {spec_file}" + - (f" --steps {' '.join(steps_to_purge)}" if steps_to_purge is not None else "") - ) - result = subprocess.run( - purge_cmd, - shell=True, - capture_output=True, - text=True, - input=input_value + "merlin purge" + + (" -f" if force else "") + + f" {spec_file}" + + (f" --steps {' '.join(steps_to_purge)}" if steps_to_purge is not None else "") ) + result = subprocess.run(purge_cmd, shell=True, capture_output=True, text=True, input=input_value) return { "stdout": result.stdout, "stderr": result.stderr, @@ -133,13 +131,13 @@ def check_queues( if steps_to_purge and matching_queue in [queues_in_spec[step] for step in steps_to_purge]: assert len(tasks) == 0, f"Expected 0 tasks in {matching_queue}, found {len(tasks)}." else: - assert len(tasks) == expected_task_count, f"Expected {expected_task_count} tasks in {matching_queue}, found {len(tasks)}." + assert ( + len(tasks) == expected_task_count + ), f"Expected {expected_task_count} tasks in {matching_queue}, found {len(tasks)}." def test_no_options_tasks_exist_y( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -150,18 +148,8 @@ def test_no_options_tasks_exist_y( tasks from the server. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -169,13 +157,13 @@ def test_no_options_tasks_exist_y( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ - from merlin.celery import app as celery_app + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: # Send tasks to the server for every queue in the spec - queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) # Run the purge test test_info = self.run_purge(feature_demo, input_value="y") @@ -189,13 +177,11 @@ def test_no_options_tasks_exist_y( check_test_conditions(conditions, test_info) # Check on the Redis queues to ensure they were purged - self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) def test_no_options_no_tasks_y( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -206,18 +192,8 @@ def test_no_options_no_tasks_y( messages purged" log. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -225,18 +201,18 @@ def test_no_options_no_tasks_y( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ - from merlin.celery import app as celery_app + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): # Get the queues from the spec file spec = get_spec_with_expansion(feature_demo) queues_in_spec = spec.get_task_queues() num_queues = len(queues_in_spec.values()) # Check that there are no tasks in the queues before we run the purge command - self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) # Run the purge test test_info = self.run_purge(feature_demo, input_value="y") @@ -250,14 +226,11 @@ def test_no_options_no_tasks_y( check_test_conditions(conditions, test_info) # Check that the Redis server still has no tasks - self.check_queues(redis_client, queues_in_spec, expected_task_count=0) - + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) def test_no_options_N( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -268,18 +241,8 @@ def test_no_options_N( command without purging the tasks. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -287,13 +250,13 @@ def test_no_options_N( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ - from merlin.celery import app as celery_app + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: # Send tasks to the server for every queue in the spec - queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) # Run the purge test test_info = self.run_purge(feature_demo, input_value="N") @@ -307,13 +270,11 @@ def test_no_options_N( check_test_conditions(conditions, test_info) # Check on the Redis queues to ensure they were not purged - self.check_queues(redis_client, queues_in_spec, expected_task_count=1) + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=1) def test_force_option( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -323,18 +284,8 @@ def test_force_option( immediately purge all tasks. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -342,13 +293,13 @@ def test_force_option( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ - from merlin.celery import app as celery_app + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: # Send tasks to the server for every queue in the spec - queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) # Run the purge test test_info = self.run_purge(feature_demo, force=True) @@ -362,13 +313,11 @@ def test_force_option( check_test_conditions(conditions, test_info) # Check on the Redis queues to ensure they were purged - self.check_queues(redis_client, queues_in_spec, expected_task_count=0) + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) def test_steps_option( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, ): @@ -378,18 +327,8 @@ def test_steps_option( associated with the steps provided. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -397,13 +336,13 @@ def test_steps_option( A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. """ - from merlin.celery import app as celery_app + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel feature_demo = self.setup_test(path_to_merlin_codebase, merlin_server_dir) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: # Send tasks to the server for every queue in the spec - queues_in_spec, num_queues = self.setup_tasks(CTM, feature_demo) + queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) # Run the purge test steps_to_purge = ["hello", "collect"] @@ -419,4 +358,4 @@ def test_steps_option( check_test_conditions(conditions, test_info) # Check on the Redis queues to ensure they were not purged - self.check_queues(redis_client, queues_in_spec, expected_task_count=1, steps_to_purge=steps_to_purge) + self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=1, steps_to_purge=steps_to_purge) diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index f1e7fec09..7f5f48f78 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -2,18 +2,18 @@ This module will contain the testing logic for the `merlin run` command. """ + import csv import os import re import subprocess -from typing import Dict, Tuple, Union - -from redis import Redis +from typing import Dict, Union from merlin.spec.expansion import get_spec_with_expansion +from tests.fixture_data_classes import RedisBrokerAndBackend from tests.context_managers.celery_task_manager import CeleryTaskManager from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr -from tests.integration.conditions import HasReturnCode, PathExists, StepFileExists +from tests.integration.conditions import HasReturnCode, PathExists from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd @@ -25,10 +25,7 @@ class TestRunCommand: demo_workflow = os.path.join("examples", "workflows", "feature_demo", "feature_demo.yaml") def setup_test_environment( - self, - path_to_merlin_codebase: FixtureStr, - merlin_server_dir: FixtureStr, - run_command_testing_dir: FixtureStr + self, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr ) -> str: """ Setup the test environment for these tests by: @@ -79,10 +76,10 @@ def get_output_workspace_from_logs(self, test_info: Dict[str, Union[str, int]]) """ Extracts the workspace path from the provided standard output and error logs. - This method searches for a specific message indicating the study workspace - in the combined logs (both stdout and stderr). The expected message format - is: "Study workspace is ''". If the message is found, - the method returns the extracted workspace path. If the message is not + This method searches for a specific message indicating the study workspace + in the combined logs (both stdout and stderr). The expected message format + is: "Study workspace is ''". If the message is found, + the method returns the extracted workspace path. If the message is not found, an assertion error is raised. Args: @@ -109,9 +106,7 @@ class TestRunCommandDistributed(TestRunCommand): def test_distributed_run( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -121,18 +116,8 @@ def test_distributed_run( using the `merlin run` command with no flags. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -147,7 +132,7 @@ def test_distributed_run( # Setup the testing environment feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): # Send tasks to the server test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_distributed_run") @@ -161,18 +146,16 @@ def test_distributed_run( for queue in queues_in_spec.values(): # Brackets are special chars in regex so we have to add \ to make them literal queue = queue.replace("[", "\\[").replace("]", "\\]") - matching_queues_on_server = redis_client.keys(pattern=f"{queue}*") + matching_queues_on_server = redis_broker_and_backend_function.client.keys(pattern=f"{queue}*") # Make sure any queues that exist on the server have tasks in them for matching_queue in matching_queues_on_server: - tasks = redis_client.lrange(matching_queue, 0, -1) + tasks = redis_broker_and_backend_function.client.lrange(matching_queue, 0, -1) assert len(tasks) > 0 def test_samplesfile_option( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -183,18 +166,8 @@ def test_samplesfile_option( in to the merlin_info subdirectory. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -217,28 +190,28 @@ def test_samplesfile_option( ] sample_filename = "test_samplesfile.csv" new_samples_file = os.path.join(run_command_testing_dir, sample_filename) - with open(new_samples_file, mode='w', newline='') as file: + with open(new_samples_file, mode="w", newline="") as file: writer = csv.writer(file) writer.writerows(data) - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): # Send tasks to the server - test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_samplesfile_option --samplesfile {new_samples_file}") + test_info = self.run_merlin_command( + f"merlin run {feature_demo} --vars NAME=run_command_test_samplesfile_option --samplesfile {new_samples_file}" + ) # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) conditions = [ HasReturnCode(), PathExists(expected_workspace_path), - PathExists(os.path.join(expected_workspace_path, "merlin_info", sample_filename)) + PathExists(os.path.join(expected_workspace_path, "merlin_info", sample_filename)), ] check_test_conditions(conditions, test_info) def test_pgen_and_pargs_options( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -251,18 +224,8 @@ def test_pgen_and_pargs_options( and `N_NEW_MAX`. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -277,10 +240,12 @@ def test_pgen_and_pargs_options( # Setup test vars and the testing environment new_x2_min, new_x2_max = 1, 2 new_n_new_min, new_n_new_max = 5, 15 - pgen_filepath = os.path.join(os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))), "pgen.py") + pgen_filepath = os.path.join( + os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))), "pgen.py" + ) feature_demo = self.setup_test_environment(path_to_merlin_codebase, merlin_server_dir, run_command_testing_dir) - - with CeleryTaskManager(celery_app, redis_client) as CTM: + + with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): # Send tasks to the server test_info = self.run_merlin_command( f'merlin run {feature_demo} --vars NAME=run_command_test_pgen_and_pargs_options --pgen {pgen_filepath} --parg "X2_MIN:{new_x2_min}" --parg "X2_MAX:{new_x2_max}" --parg "N_NAME_MIN:{new_n_new_min}" --parg "N_NAME_MAX:{new_n_new_max}"' @@ -289,11 +254,7 @@ def test_pgen_and_pargs_options( # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) expanded_yaml = os.path.join(expected_workspace_path, "merlin_info", "feature_demo.expanded.yaml") - conditions = [ - HasReturnCode(), - PathExists(expected_workspace_path), - PathExists(os.path.join(expanded_yaml)) - ] + conditions = [HasReturnCode(), PathExists(expected_workspace_path), PathExists(os.path.join(expanded_yaml))] check_test_conditions(conditions, test_info) # Read in the parameters from the expanded yaml and ensure they're within the new bounds we provided @@ -313,9 +274,7 @@ class TestRunCommandLocal(TestRunCommand): def test_dry_run( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -330,18 +289,8 @@ def test_dry_run( & stopping workers. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -356,7 +305,7 @@ def test_dry_run( # Run the test and grab the output workspace generated from it test_info = self.run_merlin_command(f"merlin run {feature_demo} --vars NAME=run_command_test_dry_run --local --dry") - + # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) conditions = [ @@ -371,23 +320,27 @@ def test_dry_run( step_directory = os.path.join(expected_workspace_path, step.name) assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" - allowed_dry_run_files = {"MERLIN_STATUS.json", "status.lock"} + allowed_dry_run_files = {"MERLIN_STATUS.json", "status.lock"} for dirpath, dirnames, filenames in os.walk(step_directory): # Check if the current directory has no subdirectories (leaf directory) if not dirnames: # Check for unexpected files - unexpected_files = [file for file in filenames if file not in allowed_dry_run_files and not file.endswith(".sh")] - assert not unexpected_files, f"Unexpected files found in {dirpath}: {unexpected_files}. Expected only .sh files or {allowed_dry_run_files}." + unexpected_files = [ + file for file in filenames if file not in allowed_dry_run_files and not file.endswith(".sh") + ] + assert ( + not unexpected_files + ), f"Unexpected files found in {dirpath}: {unexpected_files}. Expected only .sh files or {allowed_dry_run_files}." # Check that there is exactly one .sh file sh_file_count = sum(1 for file in filenames if file.endswith(".sh")) - assert sh_file_count == 1, f"Expected exactly one .sh file in {dirpath} but found {sh_file_count} .sh files." + assert ( + sh_file_count == 1 + ), f"Expected exactly one .sh file in {dirpath} but found {sh_file_count} .sh files." def test_local_run( self, - redis_client: FixtureRedis, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, merlin_server_dir: FixtureStr, run_command_testing_dir: FixtureStr, @@ -397,18 +350,8 @@ def test_local_run( the `merlin run` command with the `--local` flag. Args: - redis_client: - A fixture that connects us to a redis client that we can interact with. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_merlin_codebase: A fixture to provide the path to the directory containing Merlin's core functionality. @@ -429,7 +372,7 @@ def test_local_run( # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) - + conditions = [ HasReturnCode(), PathExists(expected_workspace_path), @@ -445,4 +388,6 @@ def test_local_run( # Check if the current directory has no subdirectories (leaf directory) if not dirnames: # Check for the existence of the MERLIN_FINISHED file - assert "MERLIN_FINISHED" in filenames, f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" + assert ( + "MERLIN_FINISHED" in filenames + ), f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py index 3cbbdc861..b167b64c4 100644 --- a/tests/integration/commands/test_stop_and_query_workers.py +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -11,13 +11,14 @@ import pytest +from tests.fixture_data_classes import RedisBrokerAndBackend from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.fixture_types import FixtureModification, FixtureStr from tests.integration.conditions import Condition, HasRegex from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec -# pylint: disable=unused-argument,import-outside-toplevel,too-many-arguments +# pylint: disable=unused-argument,import-outside-toplevel class WorkerMessages(Enum): @@ -138,9 +139,7 @@ def get_no_workers_msg(self, command_to_test: str) -> WorkerMessages: @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_no_workers( self, - redis_server: FixtureStr, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, merlin_server_dir: FixtureStr, command_to_test: str, ): @@ -160,19 +159,8 @@ def test_no_workers( passing. Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. merlin_server_dir: A fixture to provide the path to the merlin_server directory that will be created by the `redis_server` fixture. @@ -203,9 +191,7 @@ def test_no_workers( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_no_flags( self, - redis_server: FixtureStr, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -219,19 +205,8 @@ def test_no_flags( `run_test_with_workers()` method. Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_test_specs: A fixture to provide the path to the directory containing test specifications. merlin_server_dir: @@ -257,9 +232,7 @@ def test_no_flags( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_spec_flag( self, - redis_server: FixtureStr, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -274,19 +247,8 @@ def test_spec_flag( is doing, see the `run_test_with_workers()` method. Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_test_specs: A fixture to provide the path to the directory containing test specifications. merlin_server_dir: @@ -317,9 +279,7 @@ def test_spec_flag( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_workers_flag( self, - redis_server: FixtureStr, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -334,19 +294,8 @@ def test_workers_flag( test is doing, see the `run_test_with_workers()` method. Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_test_specs: A fixture to provide the path to the directory containing test specifications. merlin_server_dir: @@ -378,9 +327,7 @@ def test_workers_flag( @pytest.mark.parametrize("command_to_test", ["merlin stop-workers", "merlin query-workers"]) def test_queues_flag( self, - redis_server: FixtureStr, - redis_results_backend_config_function: FixtureModification, - redis_broker_config_function: FixtureModification, + redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, command_to_test: str, @@ -395,19 +342,8 @@ def test_queues_flag( test is doing, see the `run_test_with_workers()` method. Parameters: - redis_server: - A fixture that starts a containerized redis server instance that runs on - localhost:6379. - redis_results_backend_config_function: - A fixture that modifies the CONFIG object so that it points the results - backend configuration to the containerized redis server we start up with - the `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. - redis_broker_config_function: - A fixture that modifies the CONFIG object so that it points the broker - configuration to the containerized redis server we start up with the - `redis_server` fixture. The CONFIG object is what merlin uses to connect - to a server. + redis_broker_and_backend_function: Fixture for setting up Redis broker and + backend for function-scoped tests. path_to_test_specs: A fixture to provide the path to the directory containing test specifications. merlin_server_dir: @@ -441,4 +377,4 @@ def test_queues_flag( assert worker_name in active_queues -# pylint: enable=unused-argument,import-outside-toplevel,too-many-arguments +# pylint: enable=unused-argument,import-outside-toplevel diff --git a/tests/integration/workflows/test_chord_error.py b/tests/integration/workflows/test_chord_error.py index 46530a832..87abd71df 100644 --- a/tests/integration/workflows/test_chord_error.py +++ b/tests/integration/workflows/test_chord_error.py @@ -1,12 +1,15 @@ """ This module contains tests for the feature_demo workflow. """ + import subprocess +from tests.fixture_data_classes import ChordErrorSetup from tests.fixture_types import FixtureStr from tests.integration.conditions import HasRegex, StepFinishedFilesCount from tests.integration.helper_funcs import check_test_conditions + class TestChordError: """ Tests for the chord error workflow. @@ -14,17 +17,16 @@ class TestChordError: def test_chord_error_continues( self, - chord_err_testing_dir: FixtureStr, - chord_err_name: FixtureStr, - chord_err_run_workflow: subprocess.CompletedProcess + chord_err_setup: ChordErrorSetup, + chord_err_run_workflow: subprocess.CompletedProcess, ): """ Test that this workflow continues through to the end of its execution, even though a ChordError will be raised. Args: - chord_err_testing_dir: The directory containing the output of the chord error run. - chord_err_name: The name of the chord error study. + chord_err_setup: A fixture that returns a [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] + instance. chord_err_run_workflow: A fixture to run the chord error study. """ @@ -32,22 +34,22 @@ def test_chord_error_continues( HasRegex("Exception raised by request from the user"), StepFinishedFilesCount( # Check that the `process_samples` step has only 2 MERLIN_FINISHED files step="process_samples", - study_name=chord_err_name, - output_path=chord_err_testing_dir, + study_name=chord_err_setup.name, + output_path=chord_err_setup.testing_dir, expected_count=2, num_samples=3, ), StepFinishedFilesCount( # Check that the `samples_and_params` step has all of its MERLIN_FINISHED files step="samples_and_params", - study_name=chord_err_name, - output_path=chord_err_testing_dir, + study_name=chord_err_setup.name, + output_path=chord_err_setup.testing_dir, num_parameters=2, num_samples=3, ), StepFinishedFilesCount( # Check that the final step has a MERLIN_FINISHED file step="step_3", - study_name=chord_err_name, - output_path=chord_err_testing_dir, + study_name=chord_err_setup.name, + output_path=chord_err_setup.testing_dir, num_parameters=0, num_samples=0, ), diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py index 9d10b9ce6..88c8503ee 100644 --- a/tests/integration/workflows/test_feature_demo.py +++ b/tests/integration/workflows/test_feature_demo.py @@ -1,8 +1,10 @@ """ This module contains tests for the feature_demo workflow. """ + import subprocess +from tests.fixture_data_classes import FeatureDemoSetup from tests.fixture_types import FixtureInt, FixtureStr from tests.integration.conditions import ProvenanceYAMLFileHasRegex, StepFinishedFilesCount @@ -12,13 +14,7 @@ class TestFeatureDemo: Tests for the feature_demo workflow. """ - def test_end_to_end_run( - self, - feature_demo_testing_dir: FixtureStr, - feature_demo_num_samples: FixtureInt, - feature_demo_name: FixtureStr, - feature_demo_run_workflow: subprocess.CompletedProcess, - ): + def test_end_to_end_run(self, feature_demo_setup: FeatureDemoSetup, feature_demo_run_workflow: subprocess.CompletedProcess): """ Test that the workflow runs from start to finish with no problems. @@ -28,80 +24,78 @@ def test_end_to_end_run( fixture. Args: - feature_demo_testing_dir: The directory containing the output of the feature - demo run. - feature_demo_num_samples: The number of samples we give to the feature demo run. - feature_demo_name: The name of the feature demo study. + feature_demo_setup: A fixture that returns a + [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] instance. feature_demo_run_workflow: A fixture to run the feature demo study. """ conditions = [ ProvenanceYAMLFileHasRegex( # This condition will check that variable substitution worked - regex=f"N_SAMPLES: {feature_demo_num_samples}", + regex=f"N_SAMPLES: {feature_demo_setup.num_samples}", spec_file_name="feature_demo", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, provenance_type="expanded", ), StepFinishedFilesCount( # The rest of the conditions will ensure every step ran to completion step="hello", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, - num_samples=feature_demo_num_samples, + num_samples=feature_demo_setup.num_samples, ), StepFinishedFilesCount( step="python2_hello", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="python3_hello", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="collect", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="translate", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="learn", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="make_new_samples", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="predict", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), StepFinishedFilesCount( step="verify", - study_name=feature_demo_name, - output_path=feature_demo_testing_dir, + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, num_parameters=1, num_samples=0, ), From c1b39e449d7a14deeeb52a5cde0a3d193b400c2e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 08:58:41 -0800 Subject: [PATCH 189/201] fix lint issues --- merlin/managers/celerymanager.py | 1 - tests/conftest.py | 99 +++++++++++++++++-- tests/context_managers/celery_task_manager.py | 47 +++++++-- .../celery_workers_manager.py | 4 +- tests/fixture_data_classes.py | 10 +- tests/fixture_types.py | 11 ++- tests/fixtures/chord_err.py | 9 +- tests/fixtures/examples.py | 3 - tests/fixtures/feature_demo.py | 15 +-- tests/fixtures/run_command.py | 3 +- tests/integration/commands/test_purge.py | 12 ++- tests/integration/commands/test_run.py | 61 ++++++------ .../commands/test_stop_and_query_workers.py | 6 +- tests/integration/conditions.py | 28 ++++-- tests/integration/definitions.py | 3 - tests/integration/helper_funcs.py | 17 ++-- tests/integration/test_celeryadapter.py | 10 +- .../integration/workflows/test_chord_error.py | 1 - .../workflows/test_feature_demo.py | 5 +- tests/unit/common/test_encryption.py | 5 +- 20 files changed, 239 insertions(+), 111 deletions(-) diff --git a/merlin/managers/celerymanager.py b/merlin/managers/celerymanager.py index fe136d1ec..1d262ac51 100644 --- a/merlin/managers/celerymanager.py +++ b/merlin/managers/celerymanager.py @@ -130,7 +130,6 @@ def restart_celery_worker(self, worker: str) -> bool: # Start the worker again with the args saved in redis db with self.get_worker_args_redis_connection() as worker_args_connect, self.get_worker_status_redis_connection() as worker_status_connect: - # Get the args and remove the worker_cmd from the hash set args = worker_args_connect.hgetall(worker) worker_cmd = args["worker_cmd"] diff --git a/tests/conftest.py b/tests/conftest.py index d287e4e73..b80a4b47d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,24 +35,23 @@ from copy import copy from glob import glob from time import sleep -from typing import Callable, Dict import pytest import yaml from _pytest.tmpdir import TempPathFactory from celery import Celery -from celery.canvas import Signature from redis import Redis from merlin.config.configfile import CONFIG from tests.constants import CERT_FILES, SERVER_PASS from tests.context_managers.celery_workers_manager import CeleryWorkersManager from tests.context_managers.server_manager import RedisServerManager +from tests.fixture_data_classes import RedisBrokerAndBackend from tests.fixture_types import ( FixtureBytes, + FixtureCallable, FixtureCelery, FixtureDict, - FixtureInt, FixtureModification, FixtureRedis, FixtureSignature, @@ -70,7 +69,7 @@ fixture_glob = os.path.join("tests", "fixtures", "**", "*.py") pytest_plugins = [ - fixture_file.replace(os.sep, ".").replace(".py", "") + fixture_file.replace(os.sep, ".").replace(".py", "") for fixture_file in glob(fixture_glob, recursive=True) if not fixture_file.endswith("__init__.py") ] @@ -120,11 +119,11 @@ def setup_redis_config(config_type: str, merlin_server_dir: str): pass_file = os.path.join(merlin_server_dir, "redis.pass") create_pass_file(pass_file) - if config_type == 'broker': + if config_type == "broker": CONFIG.broker.password = pass_file CONFIG.broker.port = port CONFIG.broker.name = name - elif config_type == 'results_backend': + elif config_type == "results_backend": CONFIG.results_backend.password = pass_file CONFIG.results_backend.port = port CONFIG.results_backend.name = name @@ -170,7 +169,7 @@ def path_to_merlin_codebase() -> FixtureStr: @pytest.fixture(scope="session") -def create_testing_dir() -> Callable: +def create_testing_dir() -> FixtureCallable: """ Fixture to create a temporary testing directory. @@ -193,7 +192,7 @@ def _create_testing_dir(base_dir: str, sub_dir: str) -> str: if not os.path.exists(testing_dir): os.makedirs(testing_dir) # Use makedirs to create intermediate directories if needed return testing_dir - + return _create_testing_dir @@ -274,6 +273,7 @@ def redis_client(redis_server: FixtureStr) -> FixtureRedis: """ return Redis.from_url(url=redis_server) + @pytest.fixture(scope="session") def celery_app(redis_server: FixtureStr) -> FixtureCelery: """ @@ -372,7 +372,7 @@ def _config(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes): Args: merlin_server_dir: The directory to the merlin test server configuration test_encryption_key: An encryption key to be used for testing - + Yields: This function yields control back to the test function, allowing tests to run with the modified CONFIG settings. @@ -433,13 +433,14 @@ def config_function(merlin_server_dir: FixtureStr, test_encryption_key: FixtureB Args: merlin_server_dir: The directory to the merlin test server configuration test_encryption_key: An encryption key to be used for testing - + Yields: This function yields control back to the test function, allowing tests to run with the modified CONFIG settings. """ yield from _config(merlin_server_dir, test_encryption_key) + @pytest.fixture(scope="class") def config_class(merlin_server_dir: FixtureStr, test_encryption_key: FixtureBytes) -> FixtureModification: """ @@ -601,3 +602,81 @@ def mysql_results_backend_config( CONFIG.results_backend.ca_certs = CERT_FILES["ssl_ca"] yield + + +@pytest.fixture(scope="function") +def redis_broker_and_backend_function( + redis_client: FixtureRedis, + redis_server: FixtureStr, + redis_broker_config_function: FixtureModification, + redis_results_backend_config_function: FixtureModification, +): + """ + Fixture for setting up Redis broker and backend for function-scoped tests. + + This fixture creates an instance of `RedisBrokerAndBackend`, which + encapsulates all necessary Redis-related fixtures required for + establishing connections to Redis as both a broker and a backend + during function-scoped tests. + + Args: + redis_client: A fixture that provides a client for interacting with the + Redis server. + redis_server: A fixture providing the connection string to the Redis + server instance. + redis_broker_config_function: A fixture that modifies the configuration + to point to the Redis server used as the message broker for + function-scoped tests. + redis_results_backend_config_function: A fixture that modifies the + configuration to point to the Redis server used for storing results + in function-scoped tests. + + Returns: + An instance containing the Redis client, server connection string, and + configuration modifications for both the broker and backend. + """ + return RedisBrokerAndBackend( + client=redis_client, + server=redis_server, + broker_config=redis_broker_config_function, + results_backend_config=redis_results_backend_config_function, + ) + + +@pytest.fixture(scope="class") +def redis_broker_and_backend_class( + redis_client: FixtureRedis, + redis_server: FixtureStr, + redis_broker_config_class: FixtureModification, + redis_results_backend_config_class: FixtureModification, +) -> RedisBrokerAndBackend: + """ + Fixture for setting up Redis broker and backend for class-scoped tests. + + This fixture creates an instance of `RedisBrokerAndBackend`, which + encapsulates all necessary Redis-related fixtures required for + establishing connections to Redis as both a broker and a backend + during class-scoped tests. + + Args: + redis_client: A fixture that provides a client for interacting with the + Redis server. + redis_server: A fixture providing the connection string to the Redis + server instance. + redis_broker_config_function: A fixture that modifies the configuration + to point to the Redis server used as the message broker for + class-scoped tests. + redis_results_backend_config_function: A fixture that modifies the + configuration to point to the Redis server used for storing results + in class-scoped tests. + + Returns: + An instance containing the Redis client, server connection string, and + configuration modifications for both the broker and backend. + """ + return RedisBrokerAndBackend( + client=redis_client, + server=redis_server, + broker_config=redis_broker_config_class, + results_backend_config=redis_results_backend_config_class, + ) diff --git a/tests/context_managers/celery_task_manager.py b/tests/context_managers/celery_task_manager.py index bf6396070..b71dcd8a4 100644 --- a/tests/context_managers/celery_task_manager.py +++ b/tests/context_managers/celery_task_manager.py @@ -2,6 +2,7 @@ Module to define functionality for sending tasks to the server and ensuring they're cleared from the server when the test finishes. """ + from types import TracebackType from typing import List, Type @@ -67,14 +68,42 @@ def send_task(self, task_name: str, *args, **kwargs) -> AsyncResult: task that was sent to the server. """ valid_kwargs = [ - 'add_to_parent', 'chain', 'chord', 'compression', 'connection', - 'countdown', 'eta', 'exchange', 'expires', 'group_id', - 'group_index', 'headers', 'ignore_result', 'link', 'link_error', - 'parent_id', 'priority', 'producer', 'publisher', 'queue', - 'replaced_task_nesting', 'reply_to', 'result_cls', 'retries', - 'retry', 'retry_policy', 'root_id', 'route_name', 'router', - 'routing_key', 'serializer', 'shadow', 'soft_time_limit', 'task_id', - 'task_type', 'time_limit' + "add_to_parent", + "chain", + "chord", + "compression", + "connection", + "countdown", + "eta", + "exchange", + "expires", + "group_id", + "group_index", + "headers", + "ignore_result", + "link", + "link_error", + "parent_id", + "priority", + "producer", + "publisher", + "queue", + "replaced_task_nesting", + "reply_to", + "result_cls", + "retries", + "retry", + "retry_policy", + "root_id", + "route_name", + "router", + "routing_key", + "serializer", + "shadow", + "soft_time_limit", + "task_id", + "task_type", + "time_limit", ] send_task_kwargs = {key: kwargs.pop(key) for key in valid_kwargs if key in kwargs} @@ -117,7 +146,7 @@ def get_queue_list(self) -> List[str]: queues.extend(matching_queues) # Get any queues that start with '[merlin]' - cursor, matching_queues = self.redis_client.scan(cursor=cursor, match="\[merlin\]*") + cursor, matching_queues = self.redis_client.scan(cursor=cursor, match="\\[merlin\\]*") queues.extend(matching_queues) if cursor == 0: diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index c38173df9..e7335d12b 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -182,7 +182,7 @@ def launch_worker(self, worker_name: str, queues: List[str], concurrency: int = echo_process = subprocess.Popen( # pylint: disable=consider-using-with f"echo 'celery -A merlin_test_app {' '.join(worker_launch_cmd)}'; sleep inf", shell=True, - preexec_fn=os.setpgrp, # Make this the parent of the group so we can kill the 'sleep inf' that's spun up + start_new_session=True, # Make this the parent of the group so we can kill the 'sleep inf' that's spun up ) self.echo_processes[worker_name] = echo_process.pid @@ -215,7 +215,7 @@ def add_run_workers_process(self, pid: int): Add a process ID for a `merlin run-workers` process to the set that tracks all `merlin run-workers` processes that are currently running. - + Warning: The process that's added here must utilize the `start_new_session=True` setting of subprocess.Popen. This diff --git a/tests/fixture_data_classes.py b/tests/fixture_data_classes.py index 588a21aa2..2ff7bf877 100644 --- a/tests/fixture_data_classes.py +++ b/tests/fixture_data_classes.py @@ -1,6 +1,7 @@ """ This module houses dataclasses to be used with pytest fixtures. """ + from dataclasses import dataclass from tests.fixture_types import FixtureInt, FixtureModification, FixtureRedis, FixtureStr @@ -9,11 +10,11 @@ @dataclass class RedisBrokerAndBackend: """ - Data class to encapsulate all Redis-related fixtures required for + Data class to encapsulate all Redis-related fixtures required for establishing connections to Redis for both the broker and backend. - This class simplifies the management of Redis fixtures by grouping - them into a single object, reducing the number of individual fixture + This class simplifies the management of Redis fixtures by grouping + them into a single object, reducing the number of individual fixture imports needed in tests that require Redis functionality. Attributes: @@ -27,6 +28,7 @@ class RedisBrokerAndBackend: configuration to point to the Redis server used for storing results. """ + client: FixtureRedis server: FixtureStr results_backend_config: FixtureModification @@ -50,6 +52,7 @@ class FeatureDemoSetup: name: A string representing the name to use for the feature_demo workflow. path: The path to the feature demo YAML file. """ + testing_dir: FixtureStr num_samples: FixtureInt name: FixtureStr @@ -71,6 +74,7 @@ class ChordErrorSetup: name: A string representing the name to use for the chord_err workflow. path: The path to the chord error YAML file. """ + testing_dir: FixtureStr name: FixtureStr path: FixtureStr diff --git a/tests/fixture_types.py b/tests/fixture_types.py index 93229bdf0..62b827c57 100644 --- a/tests/fixture_types.py +++ b/tests/fixture_types.py @@ -15,18 +15,21 @@ - `FixtureSignature`: A fixture that returns a Celery Signature object - `FixtureStr`: A fixture that returns a string """ -import pytest + from argparse import Namespace +from typing import Annotated, Any, Callable, Dict, Tuple, TypeVar + +import pytest from celery import Celery from celery.canvas import Signature from redis import Redis -from typing import Any, Annotated, Callable, Dict, Tuple, TypeVar + # TODO convert unit test type hinting to use these # - likely will do this when I work on API docs for test library -K = TypeVar('K') -V = TypeVar('V') +K = TypeVar("K") +V = TypeVar("V") FixtureBytes = Annotated[bytes, pytest.fixture] FixtureCallable = Annotated[Callable, pytest.fixture] diff --git a/tests/fixtures/chord_err.py b/tests/fixtures/chord_err.py index 3e93f3f77..54c639072 100644 --- a/tests/fixtures/chord_err.py +++ b/tests/fixtures/chord_err.py @@ -8,10 +8,13 @@ import pytest from tests.fixture_data_classes import ChordErrorSetup, RedisBrokerAndBackend -from tests.fixture_types import FixtureCallable, FixtureModification, FixtureRedis, FixtureStr +from tests.fixture_types import FixtureCallable, FixtureStr from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow +# pylint: disable=redefined-outer-name + + @pytest.fixture(scope="session") def chord_err_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ @@ -51,7 +54,7 @@ def chord_err_setup( Fixture for setting up the environment required for testing the chord error workflow. This fixture prepares the necessary configuration and paths for executing tests related - to the chord error workflow. It aggregates the required parameters into a single + to the chord error workflow. It aggregates the required parameters into a single [`ChordErrorSetup`][fixture_data_classes.ChordErrorSetup] data class instance, which simplifies the management of these parameters in tests. @@ -59,7 +62,7 @@ def chord_err_setup( chord_err_testing_dir: The path to the temporary output directory where chord error workflow tests will store their results. chord_err_name: A string representing the name to use for the chord error workflow. - path_to_test_specs: The base path to the Merlin test specs directory, which is + path_to_test_specs: The base path to the Merlin test specs directory, which is used to locate the chord error YAML file. Returns: diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py index 3fe444c1d..4096b0e76 100644 --- a/tests/fixtures/examples.py +++ b/tests/fixtures/examples.py @@ -2,9 +2,6 @@ Fixtures specifically for help testing the modules in the examples/ directory. """ -import os -from typing import Callable - import pytest from tests.fixture_types import FixtureCallable, FixtureStr diff --git a/tests/fixtures/feature_demo.py b/tests/fixtures/feature_demo.py index 39ef2251f..ca77f23fb 100644 --- a/tests/fixtures/feature_demo.py +++ b/tests/fixtures/feature_demo.py @@ -8,10 +8,13 @@ import pytest from tests.fixture_data_classes import FeatureDemoSetup, RedisBrokerAndBackend -from tests.fixture_types import FixtureCallable, FixtureInt, FixtureModification, FixtureRedis, FixtureStr +from tests.fixture_types import FixtureCallable, FixtureInt, FixtureStr from tests.integration.helper_funcs import copy_app_yaml_to_cwd, run_workflow +# pylint: disable=redefined-outer-name + + @pytest.fixture(scope="session") def feature_demo_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir: FixtureStr) -> FixtureStr: """ @@ -65,18 +68,18 @@ def feature_demo_setup( Fixture for setting up the environment required for testing the feature demo workflow. This fixture prepares the necessary configuration and paths for executing tests related - to the feature demo workflow. It aggregates the required parameters into a single + to the feature demo workflow. It aggregates the required parameters into a single [`FeatureDemoSetup`][fixture_data_classes.FeatureDemoSetup] data class instance, which simplifies the management of these parameters in tests. Args: - feature_demo_testing_dir: The path to the temporary output directory where + feature_demo_testing_dir: The path to the temporary output directory where feature demo workflow tests will store their results. - feature_demo_num_samples: An integer representing the number of samples + feature_demo_num_samples: An integer representing the number of samples to use in the feature demo workflow. - feature_demo_name: A string representing the name to use for the feature + feature_demo_name: A string representing the name to use for the feature demo workflow. - path_to_merlin_codebase: The base path to the Merlin codebase, which is + path_to_merlin_codebase: The base path to the Merlin codebase, which is used to locate the feature demo YAML file. Returns: diff --git a/tests/fixtures/run_command.py b/tests/fixtures/run_command.py index 842f03fca..5a666209b 100644 --- a/tests/fixtures/run_command.py +++ b/tests/fixtures/run_command.py @@ -1,7 +1,6 @@ """ Fixtures specifically for help testing the `merlin run` command. """ -import os import pytest @@ -17,7 +16,7 @@ def run_command_testing_dir(create_testing_dir: FixtureCallable, temp_output_dir Args: create_testing_dir: A fixture which returns a function that creates the testing directory. temp_output_dir: The path to the temporary ouptut directory we'll be using for this test run. - + Returns: The path to the temporary testing directory for `merlin run` tests. """ diff --git a/tests/integration/commands/test_purge.py b/tests/integration/commands/test_purge.py index 69c04c0dd..91340e946 100644 --- a/tests/integration/commands/test_purge.py +++ b/tests/integration/commands/test_purge.py @@ -8,9 +8,9 @@ from typing import Dict, List, Tuple, Union from merlin.spec.expansion import get_spec_with_expansion -from tests.fixture_data_classes import RedisBrokerAndBackend from tests.context_managers.celery_task_manager import CeleryTaskManager -from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import FixtureRedis, FixtureStr from tests.integration.conditions import HasRegex, HasReturnCode from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd @@ -228,7 +228,7 @@ def test_no_options_no_tasks_y( # Check that the Redis server still has no tasks self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=0) - def test_no_options_N( + def test_no_options_n( self, redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, @@ -342,7 +342,7 @@ def test_steps_option( with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client) as celery_task_manager: # Send tasks to the server for every queue in the spec - queues_in_spec, num_queues = self.setup_tasks(celery_task_manager, feature_demo) + queues_in_spec, _ = self.setup_tasks(celery_task_manager, feature_demo) # Run the purge test steps_to_purge = ["hello", "collect"] @@ -358,4 +358,6 @@ def test_steps_option( check_test_conditions(conditions, test_info) # Check on the Redis queues to ensure they were not purged - self.check_queues(redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=1, steps_to_purge=steps_to_purge) + self.check_queues( + redis_broker_and_backend_function.client, queues_in_spec, expected_task_count=1, steps_to_purge=steps_to_purge + ) diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index 7f5f48f78..1207f49b6 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -10,13 +10,16 @@ from typing import Dict, Union from merlin.spec.expansion import get_spec_with_expansion -from tests.fixture_data_classes import RedisBrokerAndBackend from tests.context_managers.celery_task_manager import CeleryTaskManager -from tests.fixture_types import FixtureModification, FixtureRedis, FixtureStr +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import FixtureStr from tests.integration.conditions import HasReturnCode, PathExists from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd +# pylint: disable=import-outside-toplevel,unused-argument + + class TestRunCommand: """ Base class for testing the `merlin run` command. @@ -209,7 +212,7 @@ def test_samplesfile_option( ] check_test_conditions(conditions, test_info) - def test_pgen_and_pargs_options( + def test_pgen_and_pargs_options( # pylint: disable=too-many-locals self, redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, @@ -238,8 +241,7 @@ def test_pgen_and_pargs_options( from merlin.celery import app as celery_app # Setup test vars and the testing environment - new_x2_min, new_x2_max = 1, 2 - new_n_new_min, new_n_new_max = 5, 15 + bounds = {"X2": (1, 2), "N_NEW": (5, 15)} pgen_filepath = os.path.join( os.path.abspath(os.path.expandvars(os.path.expanduser(os.path.dirname(__file__)))), "pgen.py" ) @@ -248,7 +250,13 @@ def test_pgen_and_pargs_options( with CeleryTaskManager(celery_app, redis_broker_and_backend_function.client): # Send tasks to the server test_info = self.run_merlin_command( - f'merlin run {feature_demo} --vars NAME=run_command_test_pgen_and_pargs_options --pgen {pgen_filepath} --parg "X2_MIN:{new_x2_min}" --parg "X2_MAX:{new_x2_max}" --parg "N_NAME_MIN:{new_n_new_min}" --parg "N_NAME_MAX:{new_n_new_max}"' + f"merlin run {feature_demo} " + "--vars NAME=run_command_test_pgen_and_pargs_options " + f"--pgen {pgen_filepath} " + f'--parg "X2_MIN:{bounds["X2"][0]}" ' + f'--parg "X2_MAX:{bounds["X2"][1]}" ' + f'--parg "N_NAME_MIN:{bounds["N_NEW"][0]}" ' + f'--parg "N_NAME_MAX:{bounds["N_NEW"][1]}"' ) # Check that the test ran properly and created the correct directories/files @@ -258,12 +266,10 @@ def test_pgen_and_pargs_options( check_test_conditions(conditions, test_info) # Read in the parameters from the expanded yaml and ensure they're within the new bounds we provided - expanded_spec = get_spec_with_expansion(expanded_yaml) - params = expanded_spec.get_parameters() - for x2_param in params.parameters["X2"]: - assert new_x2_min <= x2_param <= new_x2_max - for n_new_param in params.parameters["N_NEW"]: - assert new_n_new_min <= n_new_param <= new_n_new_max + params = get_spec_with_expansion(expanded_yaml).get_parameters() + for param_name, (min_val, max_val) in bounds.items(): + for param in params.parameters[param_name]: + assert min_val <= param <= max_val class TestRunCommandLocal(TestRunCommand): @@ -272,7 +278,7 @@ class TestRunCommandLocal(TestRunCommand): than in a distributed manner. """ - def test_dry_run( + def test_dry_run( # pylint: disable=too-many-locals self, redis_broker_and_backend_function: RedisBrokerAndBackend, path_to_merlin_codebase: FixtureStr, @@ -308,15 +314,10 @@ def test_dry_run( # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) - conditions = [ - HasReturnCode(), - PathExists(expected_workspace_path), - ] - check_test_conditions(conditions, test_info) + check_test_conditions([HasReturnCode(), PathExists(expected_workspace_path)], test_info) # Check that every step was ran by looking for an existing output workspace - spec = get_spec_with_expansion(feature_demo) - for step in spec.get_study_steps(): + for step in get_spec_with_expansion(feature_demo).get_study_steps(): step_directory = os.path.join(expected_workspace_path, step.name) assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" @@ -328,9 +329,10 @@ def test_dry_run( unexpected_files = [ file for file in filenames if file not in allowed_dry_run_files and not file.endswith(".sh") ] - assert ( - not unexpected_files - ), f"Unexpected files found in {dirpath}: {unexpected_files}. Expected only .sh files or {allowed_dry_run_files}." + assert not unexpected_files, ( + f"Unexpected files found in {dirpath}: {unexpected_files}. " + f"Expected only .sh files or {allowed_dry_run_files}." + ) # Check that there is exactly one .sh file sh_file_count = sum(1 for file in filenames if file.endswith(".sh")) @@ -372,16 +374,10 @@ def test_local_run( # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) - - conditions = [ - HasReturnCode(), - PathExists(expected_workspace_path), - ] - check_test_conditions(conditions, test_info) + check_test_conditions([HasReturnCode(), PathExists(expected_workspace_path)], test_info) # Check that every step was ran by looking for an existing output workspace and MERLIN_FINISHED files - spec = get_spec_with_expansion(feature_demo) - for step in spec.get_study_steps(): + for step in get_spec_with_expansion(feature_demo).get_study_steps(): step_directory = os.path.join(expected_workspace_path, step.name) assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" for dirpath, dirnames, filenames in os.walk(step_directory): @@ -391,3 +387,6 @@ def test_local_run( assert ( "MERLIN_FINISHED" in filenames ), f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" + + +# pylint: enable=import-outside-toplevel,unused-argument diff --git a/tests/integration/commands/test_stop_and_query_workers.py b/tests/integration/commands/test_stop_and_query_workers.py index b167b64c4..beed6599b 100644 --- a/tests/integration/commands/test_stop_and_query_workers.py +++ b/tests/integration/commands/test_stop_and_query_workers.py @@ -11,9 +11,9 @@ import pytest -from tests.fixture_data_classes import RedisBrokerAndBackend from tests.context_managers.celery_workers_manager import CeleryWorkersManager -from tests.fixture_types import FixtureModification, FixtureStr +from tests.fixture_data_classes import RedisBrokerAndBackend +from tests.fixture_types import FixtureStr from tests.integration.conditions import Condition, HasRegex from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd, load_workers_from_spec @@ -46,7 +46,7 @@ class TestStopAndQueryWorkersCommands: """ @contextmanager - def run_test_with_workers( + def run_test_with_workers( # pylint: disable=too-many-arguments self, path_to_test_specs: FixtureStr, merlin_server_dir: FixtureStr, diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 04ec8fa28..5de093dde 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -228,7 +228,7 @@ def contains(self): with open(filename, "r") as textfile: filetext = textfile.read() return self.is_within(filetext) - except Exception: # pylint: disable=W0718 + except Exception: # pylint: disable=broad-except return False def is_within(self, text): @@ -267,7 +267,16 @@ class StepFinishedFilesCount(StudyOutputAware): passes: Checks if the count of `MERLIN_FINISHED` files matches the expected count. """ - def __init__(self, step: str, study_name: str, output_path: str, num_parameters: int = 0, num_samples: int = 0, expected_count: int = None): + # All of these parameters are necessary for this Condition so we'll ignore pylint + def __init__( + self, + step: str, + study_name: str, + output_path: str, + num_parameters: int = 0, + num_samples: int = 0, + expected_count: int = None, + ): # pylint: disable=too-many-arguments super().__init__(study_name, output_path) self.step = step self.num_parameters = num_parameters @@ -278,13 +287,13 @@ def __init__(self, step: str, study_name: str, output_path: str, num_parameters: def expected_count(self) -> int: """ Calculate the expected number of `MERLIN_FINISHED` files. - + Returns: The expected number of `MERLIN_FINISHED` files. """ # Return the explicitly set expected count if given if self._expected_count is not None: - return self._expected_count + return self._expected_count # Otherwise calculate the correct number of MERLIN_FINISHED files to expect if self.num_parameters > 0 and self.num_samples > 0: @@ -300,7 +309,7 @@ def expected_count(self) -> int: def glob_string(self) -> str: """ Glob pattern to find `MERLIN_FINISHED` files in the specified step's output directory. - + Returns: A glob pattern to find `MERLIN_FINISHED` files. """ @@ -322,14 +331,17 @@ def count_finished_files(self) -> int: def passes(self) -> bool: """ Check if the count of `MERLIN_FINISHED` files matches the expected count. - + Returns: True if the expected count matches the actual count. False otherwise. """ return self.count_finished_files() == self.expected_count def __str__(self) -> str: - return f"{__class__.__name__} expected {self.expected_count} `MERLIN_FINISHED` files, but found {self.count_finished_files()}" + return ( + f"{__class__.__name__} expected {self.expected_count} `MERLIN_FINISHED` " + f"files, but found {self.count_finished_files()}" + ) class ProvenanceYAMLFileHasRegex(HasRegex): @@ -428,7 +440,7 @@ def contains(self) -> bool: with open(self.filename, "r") as f: # pylint: disable=C0103 filetext = f.read() return self.is_within(filetext) - except Exception: # pylint: disable=W0718 + except Exception: # pylint: disable=broad-except return False def is_within(self, text): diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index a6f66020c..8d478a037 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -108,14 +108,12 @@ def define_tests(): # pylint: disable=R0914,R0915 workers_lsf = get_worker_by_cmd("jsrun", workers) run = f"merlin {err_lvl} run" restart = f"merlin {err_lvl} restart" - purge = "merlin purge" # Shortcuts for example workflow paths examples = "merlin/examples/workflows" dev_examples = "merlin/examples/dev_workflows" test_specs = "tests/integration/test_specs" demo = f"{examples}/feature_demo/feature_demo.yaml" - remote_demo = f"{examples}/remote_feature_demo/remote_feature_demo.yaml" demo_pgen = f"{examples}/feature_demo/scripts/pgen.py" simple = f"{examples}/simple_chain/simple_chain.yaml" slurm = f"{test_specs}/slurm_test.yaml" @@ -125,7 +123,6 @@ def define_tests(): # pylint: disable=R0914,R0915 flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" cli_substitution_wf = f"{test_specs}/cli_substitution_test.yaml" - chord_err_wf = f"{test_specs}/chord_err.yaml" # Other shortcuts black = "black --check --target-version py36" diff --git a/tests/integration/helper_funcs.py b/tests/integration/helper_funcs.py index 268687313..4837b516b 100644 --- a/tests/integration/helper_funcs.py +++ b/tests/integration/helper_funcs.py @@ -2,6 +2,7 @@ This module contains helper functions for the integration test suite. """ + import os import re import shutil @@ -9,8 +10,6 @@ from time import sleep from typing import Dict, List -import yaml - from merlin.spec.expansion import get_spec_with_expansion from tests.context_managers.celery_task_manager import CeleryTaskManager from tests.context_managers.celery_workers_manager import CeleryWorkersManager @@ -130,14 +129,14 @@ def run_workflow(redis_client: FixtureRedis, workflow_path: str, vars_to_substit The completed process object containing information about the execution of the workflow, including return code, stdout, and stderr. """ - from merlin.celery import app as celery_app + from merlin.celery import app as celery_app # pylint: disable=import-outside-toplevel run_workers_proc = None - with CeleryTaskManager(celery_app, redis_client) as CTM: + with CeleryTaskManager(celery_app, redis_client): # Send the tasks to the server try: - run_proc = subprocess.run( + subprocess.run( f"merlin run {workflow_path} --vars {' '.join(vars_to_substitute)}", shell=True, capture_output=True, @@ -148,16 +147,16 @@ def run_workflow(redis_client: FixtureRedis, workflow_path: str, vars_to_substit raise TimeoutError("Could not send tasks to the server within the allotted time.") from exc # We use a context manager to start workers so that they'll safely stop even if this test fails - with CeleryWorkersManager(celery_app) as CWM: + with CeleryWorkersManager(celery_app) as celery_worker_manager: # Start the workers then add them to the context manager so they can be stopped safely later - run_workers_proc = subprocess.Popen( + run_workers_proc = subprocess.Popen( # pylint: disable=consider-using-with f"merlin run-workers {workflow_path}".split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - start_new_session=True + start_new_session=True, ) - CWM.add_run_workers_process(run_workers_proc.pid) + celery_worker_manager.add_run_workers_process(run_workers_proc.pid) # Let the workflow try to run for 30 seconds sleep(30) diff --git a/tests/integration/test_celeryadapter.py b/tests/integration/test_celeryadapter.py index 60e24bb9a..448ca7777 100644 --- a/tests/integration/test_celeryadapter.py +++ b/tests/integration/test_celeryadapter.py @@ -250,7 +250,7 @@ def test_get_running_queues(self): This should return an empty list. """ result = celeryadapter.get_running_queues("merlin_test_app", test_mode=True) - assert result == [] + assert not result def test_get_active_celery_queues(self, celery_app: Celery): """ @@ -261,8 +261,8 @@ def test_get_active_celery_queues(self, celery_app: Celery): :param `celery_app`: A pytest fixture for the test Celery app """ queue_result, worker_result = celeryadapter.get_active_celery_queues(celery_app) - assert queue_result == {} - assert worker_result == [] + assert not queue_result + assert not worker_result def test_check_celery_workers_processing_tasks(self, celery_app: Celery, worker_queue_map: Dict[str, str]): """ @@ -476,7 +476,7 @@ def test_dump_celery_queue_info_csv(self, worker_queue_map: Dict[str, str]): # Make sure the rest of the csv file was created as expected dump_diff = DeepDiff(csv_dump_output, expected_output) - assert dump_diff == {} + assert not dump_diff finally: try: os.remove(outfile) @@ -513,7 +513,7 @@ def test_dump_celery_queue_info_json(self, worker_queue_map: Dict[str, str]): # There should only be one entry in the json dump file so this will only 'loop' once for dump_entry in json_df_contents.values(): json_dump_diff = DeepDiff(dump_entry, expected_output) - assert json_dump_diff == {} + assert not json_dump_diff finally: try: os.remove(outfile) diff --git a/tests/integration/workflows/test_chord_error.py b/tests/integration/workflows/test_chord_error.py index 87abd71df..e1f9fad0b 100644 --- a/tests/integration/workflows/test_chord_error.py +++ b/tests/integration/workflows/test_chord_error.py @@ -5,7 +5,6 @@ import subprocess from tests.fixture_data_classes import ChordErrorSetup -from tests.fixture_types import FixtureStr from tests.integration.conditions import HasRegex, StepFinishedFilesCount from tests.integration.helper_funcs import check_test_conditions diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py index 88c8503ee..6a0aa4411 100644 --- a/tests/integration/workflows/test_feature_demo.py +++ b/tests/integration/workflows/test_feature_demo.py @@ -5,7 +5,6 @@ import subprocess from tests.fixture_data_classes import FeatureDemoSetup -from tests.fixture_types import FixtureInt, FixtureStr from tests.integration.conditions import ProvenanceYAMLFileHasRegex, StepFinishedFilesCount @@ -14,7 +13,9 @@ class TestFeatureDemo: Tests for the feature_demo workflow. """ - def test_end_to_end_run(self, feature_demo_setup: FeatureDemoSetup, feature_demo_run_workflow: subprocess.CompletedProcess): + def test_end_to_end_run( + self, feature_demo_setup: FeatureDemoSetup, feature_demo_run_workflow: subprocess.CompletedProcess + ): """ Test that the workflow runs from start to finish with no problems. diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py index 4fb775dc0..3a06c0ab9 100644 --- a/tests/unit/common/test_encryption.py +++ b/tests/unit/common/test_encryption.py @@ -89,7 +89,10 @@ def test_gen_key(self, temp_output_dir: str): assert key_gen_contents != "" def test_get_key( - self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config_function: "fixture" # noqa: F821 + self, + merlin_server_dir: str, + test_encryption_key: bytes, + redis_results_backend_config_function: "fixture", # noqa: F821 ): """ Test the `_get_key` function. From c450cbd39c405c3100e0ecf0dff17bfd8a0f726c Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 10:10:41 -0800 Subject: [PATCH 190/201] remove hard requirement of Annotated type for python 3.7 and 3.8 --- tests/fixture_types.py | 50 ++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/tests/fixture_types.py b/tests/fixture_types.py index 62b827c57..a7dc964e3 100644 --- a/tests/fixture_types.py +++ b/tests/fixture_types.py @@ -16,8 +16,10 @@ - `FixtureStr`: A fixture that returns a string """ +import sys from argparse import Namespace -from typing import Annotated, Any, Callable, Dict, Tuple, TypeVar +from collections.abc import Callable +from typing import Any, Dict, Generic, Tuple, TypeVar import pytest from celery import Celery @@ -31,14 +33,38 @@ K = TypeVar("K") V = TypeVar("V") -FixtureBytes = Annotated[bytes, pytest.fixture] -FixtureCallable = Annotated[Callable, pytest.fixture] -FixtureCelery = Annotated[Celery, pytest.fixture] -FixtureDict = Annotated[Dict[K, V], pytest.fixture] -FixtureInt = Annotated[int, pytest.fixture] -FixtureModification = Annotated[Any, pytest.fixture] -FixtureNamespace = Annotated[Namespace, pytest.fixture] -FixtureRedis = Annotated[Redis, pytest.fixture] -FixtureSignature = Annotated[Signature, pytest.fixture] -FixtureStr = Annotated[str, pytest.fixture] -FixtureTuple = Annotated[Tuple[K, V], pytest.fixture] +# TODO when we drop support for Python 3.8, remove this if/else statement +# Check Python version +if sys.version_info >= (3, 9): + from typing import Annotated + + FixtureBytes = Annotated[bytes, pytest.fixture] + FixtureCallable = Annotated[Callable, pytest.fixture] + FixtureCelery = Annotated[Celery, pytest.fixture] + FixtureDict = Annotated[Dict[K, V], pytest.fixture] + FixtureInt = Annotated[int, pytest.fixture] + FixtureModification = Annotated[Any, pytest.fixture] + FixtureNamespace = Annotated[Namespace, pytest.fixture] + FixtureRedis = Annotated[Redis, pytest.fixture] + FixtureSignature = Annotated[Signature, pytest.fixture] + FixtureStr = Annotated[str, pytest.fixture] + FixtureTuple = Annotated[Tuple[K, V], pytest.fixture] +else: + # Fallback for Python 3.7 and 3.8 + class FixtureDict(Generic[K, V], Dict[K, V]): + pass + + class FixtureTuple(Generic[K, V], Tuple[K, V]): + pass + + FixtureBytes = pytest.fixture + FixtureCallable = pytest.fixture + FixtureCelery = pytest.fixture + FixtureDict = FixtureDict + FixtureInt = pytest.fixture + FixtureModification = pytest.fixture + FixtureNamespace = pytest.fixture + FixtureRedis = pytest.fixture + FixtureSignature = pytest.fixture + FixtureStr = Fixture + FixtureTuple = pytest.fixture From 6a9ede40e8df797e8ab5f2e8df47c1b422bbe383 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 10:11:07 -0800 Subject: [PATCH 191/201] remove distributed test CI and add unit test CI --- .github/workflows/push-pr_workflow.yml | 97 +++++++++++--------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index e03b939b1..abc2b2ac7 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -144,15 +144,11 @@ jobs: merlin example feature_demo pip3 install -r feature_demo/requirements.txt - - name: Run pytest over unit test suite - run: | - python3 -m pytest -v --order-scope=module tests/unit/ - - name: Run integration test suite for local tests run: | python3 tests/integration/run_tests.py --verbose --local - Integration-tests: + Unit-tests: runs-on: ubuntu-latest env: GO_VERSION: 1.18.1 @@ -183,11 +179,6 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Install merlin - run: | - pip3 install -e . - merlin config - - name: Install singularity run: | sudo apt-get update && sudo apt-get install -y \ @@ -209,50 +200,41 @@ jobs: make -C ./builddir && \ sudo make -C ./builddir install + - name: Install merlin to run unit tests + run: | + pip3 install -e . + merlin config + - name: Install CLI task dependencies generated from the 'feature demo' workflow run: | merlin example feature_demo pip3 install -r feature_demo/requirements.txt - # TODO remove the --ignore statement once those tests are fixed - - name: Run integration test suite for distributed tests + - name: Run pytest over unit test suite run: | - pytest --ignore tests/integration/test_celeryadapter.py tests/integration/ + python3 -m pytest -v --order-scope=module tests/unit/ - Distributed-test-suite: + Integration-tests: runs-on: ubuntu-latest - services: - # rabbitmq: - # image: rabbitmq:latest - # ports: - # - 5672:5672 - # options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5 - # Label used to access the service container - redis: - # Docker Hub image - image: redis - # Set health checks to wait until redis has started - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 + env: + GO_VERSION: 1.18.1 + SINGULARITY_VERSION: 3.9.9 + OS: linux + ARCH: amd64 strategy: matrix: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Check cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} @@ -263,33 +245,38 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Install merlin and setup redis as the broker + - name: Install merlin run: | pip3 install -e . - merlin config --broker redis + merlin config + + - name: Install singularity + run: | + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config + wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz + rm go$GO_VERSION.$OS-$ARCH.tar.gz + export PATH=$PATH:/usr/local/go/bin + wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz + tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz + cd singularity-ce-$SINGULARITY_VERSION + ./mconfig && \ + make -C ./builddir && \ + sudo make -C ./builddir install - name: Install CLI task dependencies generated from the 'feature demo' workflow run: | merlin example feature_demo pip3 install -r feature_demo/requirements.txt + # TODO remove the --ignore statement once those tests are fixed - name: Run integration test suite for distributed tests - env: - REDIS_HOST: redis - REDIS_PORT: 6379 run: | - python3 tests/integration/run_tests.py --verbose --distributed - - # - name: Setup rabbitmq config - # run: | - # merlin config --test rabbitmq - - # - name: Run integration test suite for rabbitmq - # env: - # AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }} - # RABBITMQ_USER: Jimmy_Space - # RABBITMQ_PASS: Alexander_Rules - # ports: - # - ${{ job.services.rabbitmq.ports['5672'] }} - # run: | - # python3 tests/integration/run_tests.py --verbose --ids 31 32 + python3 -m pytest -v --ignore tests/integration/test_celeryadapter.py tests/integration/ From 711ce06d729aaded18622dcd338bf7f425269c1b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 10:30:49 -0800 Subject: [PATCH 192/201] fix typo in fixture_types and fix lint issues --- tests/fixture_types.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/fixture_types.py b/tests/fixture_types.py index a7dc964e3..da1667353 100644 --- a/tests/fixture_types.py +++ b/tests/fixture_types.py @@ -52,19 +52,25 @@ else: # Fallback for Python 3.7 and 3.8 class FixtureDict(Generic[K, V], Dict[K, V]): + """ + This class is necessary to allow FixtureDict to be subscriptable + when using it to type hint. + """ pass class FixtureTuple(Generic[K, V], Tuple[K, V]): + """ + This class is necessary to allow FixtureTuple to be subscriptable + when using it to type hint. + """ pass FixtureBytes = pytest.fixture FixtureCallable = pytest.fixture FixtureCelery = pytest.fixture - FixtureDict = FixtureDict FixtureInt = pytest.fixture FixtureModification = pytest.fixture FixtureNamespace = pytest.fixture FixtureRedis = pytest.fixture FixtureSignature = pytest.fixture - FixtureStr = Fixture - FixtureTuple = pytest.fixture + FixtureStr = pytest.fixture From 088ecc1c08a6a3588b1ed5ee36c14f41ec114a92 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 10:39:36 -0800 Subject: [PATCH 193/201] run fix-style --- tests/fixture_types.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/fixture_types.py b/tests/fixture_types.py index da1667353..ac633d4dd 100644 --- a/tests/fixture_types.py +++ b/tests/fixture_types.py @@ -56,14 +56,12 @@ class FixtureDict(Generic[K, V], Dict[K, V]): This class is necessary to allow FixtureDict to be subscriptable when using it to type hint. """ - pass class FixtureTuple(Generic[K, V], Tuple[K, V]): """ This class is necessary to allow FixtureTuple to be subscriptable when using it to type hint. """ - pass FixtureBytes = pytest.fixture FixtureCallable = pytest.fixture From f5446443f04386201a67a1bab529d5511ce090ca Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 11:20:41 -0800 Subject: [PATCH 194/201] add check for python2 before adding that condition check --- .../workflows/test_feature_demo.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/integration/workflows/test_feature_demo.py b/tests/integration/workflows/test_feature_demo.py index 6a0aa4411..7974ebecc 100644 --- a/tests/integration/workflows/test_feature_demo.py +++ b/tests/integration/workflows/test_feature_demo.py @@ -2,6 +2,7 @@ This module contains tests for the feature_demo workflow. """ +import shutil import subprocess from tests.fixture_data_classes import FeatureDemoSetup @@ -44,13 +45,6 @@ def test_end_to_end_run( num_parameters=1, num_samples=feature_demo_setup.num_samples, ), - StepFinishedFilesCount( - step="python2_hello", - study_name=feature_demo_setup.name, - output_path=feature_demo_setup.testing_dir, - num_parameters=1, - num_samples=0, - ), StepFinishedFilesCount( step="python3_hello", study_name=feature_demo_setup.name, @@ -101,6 +95,19 @@ def test_end_to_end_run( num_samples=0, ), ] + + # GitHub actions doesn't have a python2 path so we'll conditionally add this check + if shutil.which("python2"): + conditions.append( + StepFinishedFilesCount( + step="python2_hello", + study_name=feature_demo_setup.name, + output_path=feature_demo_setup.testing_dir, + num_parameters=1, + num_samples=0, + ) + ) + for condition in conditions: assert condition.passes From 28cae9103d20cca2fa854af42f88cf59e73022d6 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 11:51:16 -0800 Subject: [PATCH 195/201] convert local run test to use StepFinishedFilesCount condition --- tests/integration/commands/test_run.py | 103 +++++++++++++++++++++---- 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index 1207f49b6..26ef8cda6 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -6,6 +6,7 @@ import csv import os import re +import shutil import subprocess from typing import Dict, Union @@ -13,7 +14,7 @@ from tests.context_managers.celery_task_manager import CeleryTaskManager from tests.fixture_data_classes import RedisBrokerAndBackend from tests.fixture_types import FixtureStr -from tests.integration.conditions import HasReturnCode, PathExists +from tests.integration.conditions import HasReturnCode, PathExists, StepFinishedFilesCount from tests.integration.helper_funcs import check_test_conditions, copy_app_yaml_to_cwd @@ -368,25 +369,99 @@ def test_local_run( # Run the test and grab the output workspace generated from it study_name = "run_command_test_local_run" + num_samples = 8 test_info = self.run_merlin_command( - f"merlin run {feature_demo} --vars NAME={study_name} OUTPUT_PATH={run_command_testing_dir} --local" + f"merlin run {feature_demo} --vars NAME={study_name} OUTPUT_PATH={run_command_testing_dir} N_SAMPLES={num_samples} --local" ) # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) - check_test_conditions([HasReturnCode(), PathExists(expected_workspace_path)], test_info) + conditions = [ + HasReturnCode(), + PathExists(expected_workspace_path), + StepFinishedFilesCount( # The rest of the conditions will ensure every step ran to completion + step="hello", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=num_samples, + ), + StepFinishedFilesCount( + step="python3_hello", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="collect", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="translate", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="learn", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="make_new_samples", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="predict", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + StepFinishedFilesCount( + step="verify", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ), + ] - # Check that every step was ran by looking for an existing output workspace and MERLIN_FINISHED files - for step in get_spec_with_expansion(feature_demo).get_study_steps(): - step_directory = os.path.join(expected_workspace_path, step.name) - assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" - for dirpath, dirnames, filenames in os.walk(step_directory): - # Check if the current directory has no subdirectories (leaf directory) - if not dirnames: - # Check for the existence of the MERLIN_FINISHED file - assert ( - "MERLIN_FINISHED" in filenames - ), f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" + # GitHub actions doesn't have a python2 path so we'll conditionally add this check + if shutil.which("python2"): + conditions.append( + StepFinishedFilesCount( + step="python2_hello", + study_name=study_name, + output_path=run_command_testing_dir, + num_parameters=1, + num_samples=0, + ) + ) + + check_test_conditions(conditions, test_info) + + # # Check that every step was ran by looking for an existing output workspace and MERLIN_FINISHED files + # for step in get_spec_with_expansion(feature_demo).get_study_steps(): + # step_directory = os.path.join(expected_workspace_path, step.name) + # assert os.path.exists(step_directory), f"Output directory for step '{step.name}' not found: {step_directory}" + # for dirpath, dirnames, filenames in os.walk(step_directory): + # # Check if the current directory has no subdirectories (leaf directory) + # if not dirnames: + # # Check for the existence of the MERLIN_FINISHED file + # assert ( + # "MERLIN_FINISHED" in filenames + # ), f"Expected a MERLIN_FINISHED file in list of files for {dirpath} but did not find one" # pylint: enable=import-outside-toplevel,unused-argument From 916b5f822bf1f056e1f7bf9ab64b749f9a69df0d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 11:55:25 -0800 Subject: [PATCH 196/201] update CHANGELOG.md --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8fd49fa3..3f94dfe0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py` - Added additional tests for the `merlin run` and `merlin purge` commands - Aliased types to represent different types of pytest fixtures +- New test condition `StepFinishedFilesCount` to help search for `MERLIN_FINISHED` files in output workspaces +- Added "Unit-tests" GitHub action to run the unit test suite ### Changed - Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier - Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier - Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot` -- Moved stop-workers and query-workers integration tests to pytest tests -- Ported `run and purge feature demo` test to pytest +- Ported all distributed tests of the integration test suite to pytest + - There is now a `commands/` directory and a `workflows/` directory under the integration suite to house these tests + - Removed the "Distributed-tests" GitHub action as these tests will now be run under "Integration-tests" ## [1.12.2b1] ### Added From 3bc1fa67956e9706adaeff224db46750fca1197f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Fri, 8 Nov 2024 13:33:53 -0800 Subject: [PATCH 197/201] fix problem created by merge conflict when mergin develop --- tests/conftest.py | 18 ------------------ tests/integration/commands/test_run.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fe70a2ecb..46fad196b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -176,24 +176,6 @@ def create_testing_dir() -> FixtureCallable: Returns: A function that creates the testing directory. """ - if not os.path.exists(key_filepath): - with open(key_filepath, "w") as key_file: - key_file.write(encryption_key.decode("utf-8")) - - if app_yaml_filepath is not None: - # Load up the app.yaml that was created by starting the server - with open(app_yaml_filepath, "r") as app_yaml_file: - app_yaml = yaml.load(app_yaml_file, yaml.Loader) - - # Modify the path to the encryption key and then save it - app_yaml["results_backend"]["encryption_key"] = key_filepath - with open(app_yaml_filepath, "w") as app_yaml_file: - yaml.dump(app_yaml, app_yaml_file) - - -####################################### -######### Fixture Definitions ######### -####################################### def _create_testing_dir(base_dir: str, sub_dir: str) -> str: """ diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index 26ef8cda6..1b2bfe257 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -370,9 +370,14 @@ def test_local_run( # Run the test and grab the output workspace generated from it study_name = "run_command_test_local_run" num_samples = 8 - test_info = self.run_merlin_command( - f"merlin run {feature_demo} --vars NAME={study_name} OUTPUT_PATH={run_command_testing_dir} N_SAMPLES={num_samples} --local" - ) + vars_dict = { + "NAME": study_name, + "OUTPUT_PATH": run_command_testing_dir, + "N_SAMPLES": num_samples + } + vars_str = " ".join(f"{key}={value}" for key, value in vars_dict.items()) + command = f"merlin run {feature_demo} --vars {vars_str} --local" + test_info = self.run_merlin_command(command) # Check that the test ran properly and created the correct directories/files expected_workspace_path = self.get_output_workspace_from_logs(test_info) From 3be8963f295691ef15df02b8c48b8a03f7298438 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 11 Nov 2024 16:02:16 -0800 Subject: [PATCH 198/201] remove manager functionality from this PR --- CHANGELOG.md | 1 - merlin/main.py | 101 +----------- merlin/managers/__init__.py | 0 merlin/managers/celerymanager.py | 214 ------------------------- merlin/managers/redis_connection.py | 102 ------------ merlin/router.py | 5 +- merlin/study/celeryadapter.py | 57 ++----- merlin/study/celerymanageradapter.py | 145 ----------------- tests/integration/commands/test_run.py | 6 +- 9 files changed, 13 insertions(+), 618 deletions(-) delete mode 100644 merlin/managers/__init__.py delete mode 100644 merlin/managers/celerymanager.py delete mode 100644 merlin/managers/redis_connection.py delete mode 100644 merlin/study/celerymanageradapter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 83f3e54ea..0821f2f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Merlin manager capability to monitor celery workers. - Added additional tests for the `merlin run` and `merlin purge` commands - Aliased types to represent different types of pytest fixtures - New test condition `StepFinishedFilesCount` to help search for `MERLIN_FINISHED` files in output workspaces diff --git a/merlin/main.py b/merlin/main.py index 2e033e6d4..6ca79e86d 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -57,7 +57,6 @@ from merlin.server.server_commands import config_server, init_server, restart_server, start_server, status_server, stop_server from merlin.spec.expansion import RESERVED, get_spec_with_expansion from merlin.spec.specification import MerlinSpec -from merlin.study.celerymanageradapter import run_manager, start_manager, stop_manager from merlin.study.status import DetailedStatus, Status from merlin.study.status_constants import VALID_RETURN_CODES, VALID_STATUS_FILTERS from merlin.study.status_renderers import status_renderer_factory @@ -360,7 +359,7 @@ def stop_workers(args): LOG.warning(f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?") # Send stop command to router - router.stop_workers(args.task_server, worker_names, args.queues, args.workers, args.level.upper()) + router.stop_workers(args.task_server, worker_names, args.queues, args.workers) def print_info(args): @@ -401,35 +400,6 @@ def process_example(args: Namespace) -> None: setup_example(args.workflow, args.path) -def process_manager(args: Namespace): - """ - Process the command for managing the workers. - - This function interprets the command provided in the `args` namespace and - executes the corresponding manager function. It supports three commands: - "run", "start", and "stop". - - :param args: parsed CLI arguments - """ - if args.command == "run": - run_manager(query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout) - elif args.command == "start": - try: - start_manager( - query_frequency=args.query_frequency, query_timeout=args.query_timeout, worker_timeout=args.worker_timeout - ) - LOG.info("Manager started successfully.") - except Exception as e: - LOG.error(f"Unable to start manager.\n{e}") - elif args.command == "stop": - if stop_manager(): - LOG.info("Manager stopped successfully.") - else: - LOG.error("Unable to stop manager.") - else: - print("Run manager with a command. Try 'merlin manager -h' for more details") - - def process_monitor(args): """ CLI command to monitor merlin workers and queues to keep @@ -935,75 +905,6 @@ def generate_worker_touching_parsers(subparsers: ArgumentParser) -> None: help="regex match for specific workers to stop", ) - # merlin manager - manager: ArgumentParser = subparsers.add_parser( - "manager", - help="Watchdog application to manage workers", - description="A daemon process that helps to restart and communicate with workers while running.", - formatter_class=ArgumentDefaultsHelpFormatter, - ) - manager.set_defaults(func=process_manager) - - def add_manager_options(manager_parser: ArgumentParser): - """ - Add shared options for manager subcommands. - - The `manager run` and `manager start` subcommands have the same options. - Rather than writing duplicate code for these we'll use this function - to add the arguments to these subcommands. - - :param manager_parser: The ArgumentParser object to add these options to - """ - manager_parser.add_argument( - "-qf", - "--query_frequency", - action="store", - type=int, - default=60, - help="The frequency at which workers will be queried for response.", - ) - manager_parser.add_argument( - "-qt", - "--query_timeout", - action="store", - type=float, - default=0.5, - help="The timeout for the query response that are sent to workers.", - ) - manager_parser.add_argument( - "-wt", - "--worker_timeout", - action="store", - type=int, - default=180, - help="The sum total (query_frequency*tries) time before an attempt is made to restart worker.", - ) - - manager_commands: ArgumentParser = manager.add_subparsers(dest="command") - manager_run = manager_commands.add_parser( - "run", - help="Run the daemon process", - description="Run manager", - formatter_class=ArgumentDefaultsHelpFormatter, - ) - add_manager_options(manager_run) - manager_run.set_defaults(func=process_manager) - manager_start = manager_commands.add_parser( - "start", - help="Start the daemon process", - description="Start manager", - formatter_class=ArgumentDefaultsHelpFormatter, - ) - add_manager_options(manager_start) - manager_start.set_defaults(func=process_manager) - manager_stop = manager_commands.add_parser( - "stop", - help="Stop the daemon process", - description="Stop manager", - formatter_class=ArgumentDefaultsHelpFormatter, - ) - manager_stop.set_defaults(func=process_manager) - # merlin monitor monitor: ArgumentParser = subparsers.add_parser( "monitor", diff --git a/merlin/managers/__init__.py b/merlin/managers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/merlin/managers/celerymanager.py b/merlin/managers/celerymanager.py deleted file mode 100644 index 1d262ac51..000000000 --- a/merlin/managers/celerymanager.py +++ /dev/null @@ -1,214 +0,0 @@ -############################################################################### -# Copyright (c) 2023, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory -# Written by the Merlin dev team, listed in the CONTRIBUTORS file. -# -# -# LLNL-CODE-797170 -# All rights reserved. -# This file is part of Merlin, Version: 1.12.1. -# -# For details, see https://github.com/LLNL/merlin. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -############################################################################### -import logging -import os -import subprocess -import time - -import psutil - -from merlin.managers.redis_connection import RedisConnectionManager - - -LOG = logging.getLogger(__name__) - - -class WorkerStatus: - running = "Running" - stalled = "Stalled" - stopped = "Stopped" - rebooting = "Rebooting" - - -WORKER_INFO = { - "status": WorkerStatus.running, - "pid": -1, - "monitored": 1, # This setting is for debug mode - "num_unresponsive": 0, - "processing_work": 1, -} - - -class CeleryManager: - def __init__(self, query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180): - """ - Initializer for Celery Manager - - :param query_frequency: The frequency at which workers will be queried with ping commands - :param query_timeout: The timeout for the query pings that are sent to workers - :param worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. - """ - self.query_frequency = query_frequency - self.query_timeout = query_timeout - self.worker_timeout = worker_timeout - - @staticmethod - def get_worker_status_redis_connection() -> RedisConnectionManager: - """Get the redis connection for info regarding the worker and manager status.""" - return RedisConnectionManager(1) - - @staticmethod - def get_worker_args_redis_connection() -> RedisConnectionManager: - """Get the redis connection for info regarding the args used to generate each worker.""" - return RedisConnectionManager(2) - - def get_celery_workers_status(self, workers: list) -> dict: - """ - Get the worker status of a current worker that is being managed - - :param workers: Workers that are checked. - :return: The result dictionary for each worker and the response. - """ - from merlin.celery import app - - celery_app = app.control - ping_result = celery_app.ping(workers, timeout=self.query_timeout) - worker_results = {worker: status for d in ping_result for worker, status in d.items()} - return worker_results - - def stop_celery_worker(self, worker: str) -> bool: - """ - Stop a celery worker by kill the worker with pid - - :param worker: Worker that is being stopped. - :return: The result of whether a worker was stopped. - """ - - # Get the PID associated with the worker - with self.get_worker_status_redis_connection() as worker_status_connect: - worker_pid = int(worker_status_connect.hget(worker, "pid")) - worker_status = worker_status_connect.hget(worker, "status") - - # TODO be wary of stalled state workers (should not happen since we use psutil.Process.kill()) - # Check to see if the pid exists and worker is set as running - if worker_status == WorkerStatus.running and psutil.pid_exists(worker_pid): - # Check to see if the pid is associated with celery - worker_process = psutil.Process(worker_pid) - if "celery" in worker_process.name(): - # Kill the pid if both conditions are right - worker_process.kill() - return True - return False - - def restart_celery_worker(self, worker: str) -> bool: - """ - Restart a celery worker with the same arguements and parameters during its creation - - :param worker: Worker that is being restarted. - :return: The result of whether a worker was restarted. - """ - - # Stop the worker that is currently running (if possible) - self.stop_celery_worker(worker) - - # Start the worker again with the args saved in redis db - with self.get_worker_args_redis_connection() as worker_args_connect, self.get_worker_status_redis_connection() as worker_status_connect: - # Get the args and remove the worker_cmd from the hash set - args = worker_args_connect.hgetall(worker) - worker_cmd = args["worker_cmd"] - del args["worker_cmd"] - kwargs = args - for key in args: - if args[key].startswith("link:"): - kwargs[key] = worker_args_connect.hgetall(args[key].split(":", 1)[1]) - elif args[key] == "True": - kwargs[key] = True - elif args[key] == "False": - kwargs[key] = False - - # Run the subprocess for the worker and save the PID - process = subprocess.Popen(worker_cmd, **kwargs) - worker_status_connect.hset(worker, "pid", process.pid) - - return True - - def run(self): - """ - Main manager loop for monitoring and managing Celery workers. - - This method continuously monitors the status of Celery workers by - checking their health and attempting to restart any that are - unresponsive. It updates the Redis database with the current - status of the manager and the workers. - """ - manager_info = { - "status": "Running", - "pid": os.getpid(), - } - - with self.get_worker_status_redis_connection() as redis_connection: - LOG.debug(f"MANAGER: setting manager key in redis to hold the following info {manager_info}") - redis_connection.hset("manager", mapping=manager_info) - - # TODO figure out what to do with "processing_work" entry for the merlin monitor - while True: # TODO Make it so that it will stop after a list of workers is stopped - # Get the list of running workers - workers = redis_connection.keys() - LOG.debug(f"MANAGER: workers: {workers}") - workers.remove("manager") - workers = [worker for worker in workers if int(redis_connection.hget(worker, "monitored"))] - LOG.info(f"MANAGER: Monitoring {workers} workers") - - # Check/ Ping each worker to see if they are still running - if workers: - worker_results = self.get_celery_workers_status(workers) - - # If running set the status on redis that it is running - LOG.info(f"MANAGER: Responsive workers: {worker_results.keys()}") - for worker in list(worker_results.keys()): - redis_connection.hset(worker, "status", WorkerStatus.running) - - # If not running attempt to restart it - for worker in workers: - if worker not in worker_results: - LOG.info(f"MANAGER: Worker '{worker}' is unresponsive.") - # If time where the worker is unresponsive is less than the worker time out then just increment - num_unresponsive = int(redis_connection.hget(worker, "num_unresponsive")) + 1 - if num_unresponsive * self.query_frequency < self.worker_timeout: - # Attempt to restart worker - LOG.info(f"MANAGER: Attempting to restart worker '{worker}'...") - if self.restart_celery_worker(worker): - # If successful set the status to running and reset num_unresponsive - redis_connection.hset(worker, "status", WorkerStatus.running) - redis_connection.hset(worker, "num_unresponsive", 0) - LOG.info(f"MANAGER: Worker '{worker}' restarted.") - else: - # If failed set the status to stalled - redis_connection.hset(worker, "status", WorkerStatus.stalled) - LOG.error(f"MANAGER: Could not restart worker '{worker}'.") - else: - redis_connection.hset(worker, "num_unresponsive", num_unresponsive) - # Sleep for the query_frequency for the next iteration - time.sleep(self.query_frequency) - - -if __name__ == "__main__": - cm = CeleryManager() - cm.run() diff --git a/merlin/managers/redis_connection.py b/merlin/managers/redis_connection.py deleted file mode 100644 index 50a63492c..000000000 --- a/merlin/managers/redis_connection.py +++ /dev/null @@ -1,102 +0,0 @@ -############################################################################### -# Copyright (c) 2023, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory -# Written by the Merlin dev team, listed in the CONTRIBUTORS file. -# -# -# LLNL-CODE-797170 -# All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. -# -# For details, see https://github.com/LLNL/merlin. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -############################################################################### -""" -This module stores a manager for redis connections. -""" -import logging - -import redis - - -LOG = logging.getLogger(__name__) - - -class RedisConnectionManager: - """ - A context manager for handling redis connections. - This will ensure safe opening and closing of Redis connections. - """ - - def __init__(self, db_num: int): - self.db_num = db_num - self.connection = None - - def __enter__(self): - self.connection = self.get_redis_connection() - return self.connection - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.connection: - LOG.debug(f"MANAGER: Closing connection at db_num: {self.db_num}") - self.connection.close() - - def get_redis_connection(self) -> redis.Redis: - """ - Generic redis connection function to get the results backend redis server with a given db number increment. - - :return: Redis connection object that can be used to access values for the manager. - """ - from merlin.config.configfile import CONFIG # pylint: disable=import-outside-toplevel - from merlin.config.results_backend import get_backend_password # pylint: disable=import-outside-toplevel - - password_file = CONFIG.results_backend.password if hasattr(CONFIG.results_backend, "password") else None - server = CONFIG.results_backend.server if hasattr(CONFIG.results_backend, "server") else None - port = CONFIG.results_backend.port if hasattr(CONFIG.results_backend, "port") else None - results_db_num = CONFIG.results_backend.db_num if hasattr(CONFIG.results_backend, "db_num") else None - username = CONFIG.results_backend.username if hasattr(CONFIG.results_backend, "username") else None - - password = None - if password_file is not None: - try: - password = get_backend_password(password_file) - except IOError: - if hasattr(CONFIG.results_backend, "password"): - password = CONFIG.results_backend.password - - # Base configuration for Redis connection (this does not have ssl) - redis_config = { - "host": server, - "port": port, - "db": results_db_num + self.db_num, # Increment db_num to avoid conflicts - "username": username, - "password": password, - "decode_responses": True, - } - - # Add ssl settings if necessary - if CONFIG.results_backend.name == "rediss": - redis_config.update( - { - "ssl": True, - "ssl_cert_reqs": getattr(CONFIG.results_backend, "cert_reqs", "required"), - } - ) - - return redis.Redis(**redis_config) diff --git a/merlin/router.py b/merlin/router.py index 435100a08..552323260 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -190,7 +190,7 @@ def get_workers(task_server): return [] -def stop_workers(task_server, spec_worker_names, queues, workers_regex, debug_lvl): +def stop_workers(task_server, spec_worker_names, queues, workers_regex): """ Stops workers. @@ -198,13 +198,12 @@ def stop_workers(task_server, spec_worker_names, queues, workers_regex, debug_lv :param `spec_worker_names`: Worker names to stop, drawn from a spec. :param `queues` : The queues to stop :param `workers_regex` : Regex for workers to stop - :param debug_lvl: The debug level to use (INFO, DEBUG, ERROR, etc.) """ LOG.info("Stopping workers...") if task_server == "celery": # pylint: disable=R1705 # Stop workers - stop_celery_workers(queues, spec_worker_names, workers_regex, debug_lvl) + stop_celery_workers(queues, spec_worker_names, workers_regex) else: LOG.error("Celery is not specified as the task server!") diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 5f95bf18a..60bded1b2 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -46,9 +46,7 @@ from merlin.common.dumper import dump_handler from merlin.config import Config -from merlin.managers.celerymanager import CeleryManager, WorkerStatus from merlin.study.batch import batch_check_parallel, batch_worker_launch -from merlin.study.celerymanageradapter import add_monitor_workers, remove_monitor_workers from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running @@ -502,22 +500,15 @@ def check_celery_workers_processing(queues_in_spec: List[str], app: Celery) -> b """ # Query celery for active tasks active_tasks = app.control.inspect().active() - result = False - with CeleryManager.get_worker_status_redis_connection() as redis_connection: - # Search for the queues we provided if necessary - if active_tasks is not None: - for worker, tasks in active_tasks.items(): - for task in tasks: - if task["delivery_info"]["routing_key"] in queues_in_spec: - result = True + # Search for the queues we provided if necessary + if active_tasks is not None: + for tasks in active_tasks.values(): + for task in tasks: + if task["delivery_info"]["routing_key"] in queues_in_spec: + return True - # Set the entry in the Redis DB for the manager to signify if the worker - # is still doing work - worker_still_processing = 1 if result else 0 - redis_connection.hset(worker, "processing_work", worker_still_processing) - - return result + return False def _get_workers_to_start(spec, steps): @@ -770,36 +761,8 @@ def launch_celery_worker(worker_cmd, worker_list, kwargs): :side effect: Launches a celery worker via a subprocess """ try: - process = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 - # Get the worker name from worker_cmd and add to be monitored by celery manager - worker_cmd_list = worker_cmd.split() - worker_name = worker_cmd_list[worker_cmd_list.index("-n") + 1].replace("%h", kwargs["env"]["HOSTNAME"]) - worker_name = "celery@" + worker_name + _ = subprocess.Popen(worker_cmd, **kwargs) # pylint: disable=R1732 worker_list.append(worker_cmd) - - # Adding the worker args to redis db - with CeleryManager.get_worker_args_redis_connection() as redis_connection: - args = kwargs.copy() - # Save worker command with the arguements - args["worker_cmd"] = worker_cmd - # Store the nested dictionaries into a separate key with a link. - # Note: This only support single nested dicts(for simplicity) and - # further nesting can be accomplished by making this recursive. - for key in kwargs: - if type(kwargs[key]) is dict: - key_name = worker_name + "_" + key - redis_connection.hmset(name=key_name, mapping=kwargs[key]) - args[key] = "link:" + key_name - if type(kwargs[key]) is bool: - if kwargs[key]: - args[key] = "True" - else: - args[key] = "False" - redis_connection.hmset(name=worker_name, mapping=args) - - # Adding the worker to redis db to be monitored - add_monitor_workers(workers=((worker_name, process.pid),)) - LOG.info(f"Added {worker_name} to be monitored") except Exception as e: # pylint: disable=C0103 LOG.error(f"Cannot start celery workers, {e}") raise @@ -838,7 +801,7 @@ def purge_celery_tasks(queues, force): return subprocess.run(purge_command, shell=True).returncode -def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None, debug_lvl="INFO"): # pylint: disable=R0912 +def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None): # pylint: disable=R0912 """Send a stop command to celery workers. Default behavior is to stop all connected workers. @@ -903,8 +866,6 @@ def stop_celery_workers(queues=None, spec_worker_names=None, worker_regex=None, if workers_to_stop: LOG.info(f"Sending stop to these workers: {workers_to_stop}") app.control.broadcast("shutdown", destination=workers_to_stop) - remove_entry = False if debug_lvl == "DEBUG" else True - remove_monitor_workers(workers=workers_to_stop, worker_status=WorkerStatus.stopped, remove_entry=remove_entry) else: LOG.warning("No workers found to stop") diff --git a/merlin/study/celerymanageradapter.py b/merlin/study/celerymanageradapter.py deleted file mode 100644 index 6dc07bab2..000000000 --- a/merlin/study/celerymanageradapter.py +++ /dev/null @@ -1,145 +0,0 @@ -############################################################################### -# Copyright (c) 2023, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory -# Written by the Merlin dev team, listed in the CONTRIBUTORS file. -# -# -# LLNL-CODE-797170 -# All rights reserved. -# This file is part of Merlin, Version: 1.12.1. -# -# For details, see https://github.com/LLNL/merlin. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -############################################################################### -import logging -import subprocess - -import psutil - -from merlin.managers.celerymanager import WORKER_INFO, CeleryManager, WorkerStatus - - -LOG = logging.getLogger(__name__) - - -def add_monitor_workers(workers: list): - """ - Adds workers to be monitored by the celery manager. - :param list workers: A list of tuples which includes (worker_name, pid) - """ - if workers is None or len(workers) <= 0: - return - - LOG.info( - f"MANAGER: Attempting to have the manager monitor the following workers {[worker_name for worker_name, _ in workers]}." - ) - monitored_workers = [] - - with CeleryManager.get_worker_status_redis_connection() as redis_connection: - for worker in workers: - LOG.debug(f"MANAGER: Checking if connection for worker '{worker}' exists...") - if redis_connection.exists(worker[0]): - LOG.debug(f"MANAGER: Connection for worker '{worker}' exists. Setting this worker to be monitored") - redis_connection.hset(worker[0], "monitored", 1) - redis_connection.hset(worker[0], "pid", worker[1]) - monitored_workers.append(worker[0]) - else: - LOG.debug(f"MANAGER: Connection for worker '{worker}' does not exist. Not monitoring this worker.") - worker_info = WORKER_INFO - worker_info["pid"] = worker[1] - redis_connection.hmset(name=worker[0], mapping=worker_info) - LOG.info(f"MANAGER: Manager is monitoring the following workers {monitored_workers}.") - - -def remove_monitor_workers(workers: list, worker_status: WorkerStatus = None, remove_entry: bool = True): - """ - Remove workers from being monitored by the celery manager. - :param list workers: A worker names - """ - if workers is None or len(workers) <= 0: - return - with CeleryManager.get_worker_status_redis_connection() as redis_connection: - for worker in workers: - if redis_connection.exists(worker): - redis_connection.hset(worker, "monitored", 0) - if worker_status is not None: - redis_connection.hset(worker, "status", worker_status) - if remove_entry: - redis_connection.delete(worker) - - -def is_manager_runnning() -> bool: - """ - Check to see if the manager is running - - :return: True if manager is running and False if not. - """ - with CeleryManager.get_worker_args_redis_connection() as redis_connection: - manager_status = redis_connection.hgetall("manager") - return manager_status["status"] == WorkerStatus.running and psutil.pid_exists(manager_status["pid"]) - - -def run_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180) -> bool: - """ - A process locking function that calls the celery manager with proper arguments. - - :param query_frequency: The frequency at which workers will be queried with ping commands - :param query_timeout: The timeout for the query pings that are sent to workers - :param worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. - """ - celerymanager = CeleryManager(query_frequency=query_frequency, query_timeout=query_timeout, worker_timeout=worker_timeout) - celerymanager.run() - - -def start_manager(query_frequency: int = 60, query_timeout: float = 0.5, worker_timeout: int = 180) -> bool: - """ - A Non-locking function that calls the celery manager with proper arguments. - - :param query_frequency: The frequency at which workers will be queried with ping commands - :param query_timeout: The timeout for the query pings that are sent to workers - :param worker_timeout: The sum total(query_frequency*tries) time before an attempt is made to restart worker. - :return bool: True if the manager was started successfully. - """ - subprocess.Popen( - f"merlin manager run -qf {query_frequency} -qt {query_timeout} -wt {worker_timeout}", - shell=True, - close_fds=True, - stdout=subprocess.PIPE, - ) - return True - - -def stop_manager() -> bool: - """ - Stop the manager process using it's pid. - - :return bool: True if the manager was stopped successfully and False otherwise. - """ - with CeleryManager.get_worker_status_redis_connection() as redis_connection: - LOG.debug(f"MANAGER: manager keys: {redis_connection.hgetall('manager')}") - manager_pid = int(redis_connection.hget("manager", "pid")) - manager_status = redis_connection.hget("manager", "status") - LOG.debug(f"MANAGER: manager_status: {manager_status}") - LOG.debug(f"MANAGER: pid exists: {psutil.pid_exists(manager_pid)}") - - # Check to make sure that the manager is running and the pid exists - if manager_status == WorkerStatus.running and psutil.pid_exists(manager_pid): - psutil.Process(manager_pid).terminate() - return True - return False diff --git a/tests/integration/commands/test_run.py b/tests/integration/commands/test_run.py index 1b2bfe257..c7b27b3bb 100644 --- a/tests/integration/commands/test_run.py +++ b/tests/integration/commands/test_run.py @@ -370,11 +370,7 @@ def test_local_run( # Run the test and grab the output workspace generated from it study_name = "run_command_test_local_run" num_samples = 8 - vars_dict = { - "NAME": study_name, - "OUTPUT_PATH": run_command_testing_dir, - "N_SAMPLES": num_samples - } + vars_dict = {"NAME": study_name, "OUTPUT_PATH": run_command_testing_dir, "N_SAMPLES": num_samples} vars_str = " ".join(f"{key}={value}" for key, value in vars_dict.items()) command = f"merlin run {feature_demo} --vars {vars_str} --local" test_info = self.run_merlin_command(command) From 29b39d328036e013d24f6cfbce8741877abd01bf Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Mon, 11 Nov 2024 16:57:00 -0800 Subject: [PATCH 199/201] update README for test suite --- tests/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 9b2f7ba1f..e2fa22cbf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,16 +4,20 @@ This directory utilizes pytest to create and run our test suite. This directory is organized like so: - `conftest.py` - The script containing common fixtures for our tests +- `constants.py` - Constant values to be used throughout the test suite. +- `fixture_data_classes.py` - Dataclasses to help group pytest fixtures together, reducing the required number of imports. +- `fixture_types.py` - Aliases for type hinting fixtures. - `context_managers/` - The directory containing context managers used for testing - `celery_workers_manager.py` - A context manager used to manage celery workers for integration testing - `server_manager.py` - A context manager used to manage the redis server used for integration testing - `fixtures/` - The directory containing specific test module fixtures - `.py` - Fixtures for specific test modules - `integration/` - The directory containing integration tests - - `definitions.py` - The test definitions - `run_tests.py` - The script to run the tests defined in `definitions.py` - `conditions.py` - The conditions to test against + - `commands/` - The directory containing tests for commands of the Merlin library. + - `workflows/` The directory containing tests for entire workflow runs. - `unit/` - The directory containing unit tests - `test_*.py` - The actual test scripts to run From 6dab66bc0e69b783c4893b10558207e03f07ae26 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Nov 2024 12:30:56 -0800 Subject: [PATCH 200/201] change SIGTERM to SIGKILL --- tests/context_managers/celery_workers_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/context_managers/celery_workers_manager.py b/tests/context_managers/celery_workers_manager.py index 50ae75169..279eab325 100644 --- a/tests/context_managers/celery_workers_manager.py +++ b/tests/context_managers/celery_workers_manager.py @@ -248,7 +248,7 @@ def stop_worker(self, worker_name: str): # Terminate the echo process and its sleep inf subprocess if self.echo_processes[worker_name] is not None: - os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGTERM) + os.killpg(os.getpgid(self.echo_processes[worker_name]), signal.SIGKILL) sleep(2) def stop_all_workers(self): @@ -256,7 +256,7 @@ def stop_all_workers(self): Stop all of the running workers and the processes associated with them. """ for run_worker_pid in self.run_worker_processes: - os.killpg(os.getpgid(run_worker_pid), signal.SIGTERM) + os.killpg(os.getpgid(run_worker_pid), signal.SIGKILL) for worker_name in self.running_workers: self.stop_worker(worker_name) From dcefd22788e0dbccb80d0d962fe57864025b70a8 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Tue, 12 Nov 2024 12:51:19 -0800 Subject: [PATCH 201/201] update Makefile to include new changes to test suite --- CHANGELOG.md | 2 ++ Makefile | 29 ++++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0821f2f4d..d2ef4eeaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New test condition `StepFinishedFilesCount` to help search for `MERLIN_FINISHED` files in output workspaces - Added "Unit-tests" GitHub action to run the unit test suite - Added `CeleryTaskManager` context manager to the test suite to ensure tasks are safely purged from queues if tests fail +- Added `command-tests`, `workflow-tests`, and `integration-tests` to the Makefile ### Changed - Ported all distributed tests of the integration test suite to pytest - There is now a `commands/` directory and a `workflows/` directory under the integration suite to house these tests - Removed the "Distributed-tests" GitHub action as these tests will now be run under "Integration-tests" +- Removed `e2e-distributed*` definitions from the Makefile ## [1.12.2] ### Added diff --git a/Makefile b/Makefile index a261b1d99..9e1592925 100644 --- a/Makefile +++ b/Makefile @@ -34,12 +34,13 @@ include config.mk .PHONY : install-workflow-deps .PHONY : install-dev .PHONY : unit-tests +.PHONY : command-tests +.PHONY : workflow-tests +.PHONY : integration-tests .PHONY : e2e-tests .PHONY : e2e-tests-diagnostic .PHONY : e2e-tests-local .PHONY : e2e-tests-local-diagnostic -.PHONY : e2e-tests-distributed -.PHONY : e2e-tests-distributed-diagnostic .PHONY : tests .PHONY : check-flake8 .PHONY : check-black @@ -89,6 +90,18 @@ unit-tests: . $(VENV)/bin/activate; \ $(PYTHON) -m pytest -v --order-scope=module $(UNIT); \ +command-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) -m pytest -v $(TEST)/integration/commands/; \ + + +workflow-tests: + . $(VENV)/bin/activate; \ + $(PYTHON) -m pytest -v $(TEST)/integration/workflows/; \ + + +integration-tests: command-tests workflow-tests + # run CLI tests - these require an active install of merlin in a venv e2e-tests: @@ -111,18 +124,8 @@ e2e-tests-local-diagnostic: $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose -e2e-tests-distributed: - . $(VENV)/bin/activate; \ - $(PYTHON) $(TEST)/integration/run_tests.py --distributed; \ - - -e2e-tests-distributed-diagnostic: - . $(VENV)/bin/activate; \ - $(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose - - # run unit and CLI tests -tests: unit-tests e2e-tests +tests: unit-tests integration-tests e2e-tests check-flake8: