From ff28d415134e592ee04fa7f08ea5f5a2a5a47f73 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 23 Jan 2025 23:40:13 +0000 Subject: [PATCH] fix(plugins/filler): Move session-wide actions to pytest_sessionfinish --- src/ethereum_test_fixtures/file.py | 11 ++- src/pytest_plugins/filler/filler.py | 100 ++++++++++++---------------- stubs/xdist/__init__.pyi | 3 + stubs/xdist/methods.pyi | 3 + 4 files changed, 53 insertions(+), 64 deletions(-) create mode 100644 stubs/xdist/__init__.pyi create mode 100644 stubs/xdist/methods.pyi diff --git a/src/ethereum_test_fixtures/file.py b/src/ethereum_test_fixtures/file.py index 4c178fdb18a..32bc778fcf1 100644 --- a/src/ethereum_test_fixtures/file.py +++ b/src/ethereum_test_fixtures/file.py @@ -66,17 +66,16 @@ def collect_into_file(self, file_path: Path): add the hash to the info field on per-fixture basis. """ json_fixtures: Dict[str, Dict[str, Any]] = {} - for name, fixture in self.items(): - json_fixtures[name] = fixture.json_dict_with_info() lock_file_path = file_path.with_suffix(".lock") with FileLock(lock_file_path): - initial_data = {} if file_path.exists(): with open(file_path, "r") as f: - initial_data = json.load(f) - initial_data.update(json_fixtures) + json_fixtures = json.load(f) + for name, fixture in self.items(): + json_fixtures[name] = fixture.json_dict_with_info() + with open(file_path, "w") as f: - json.dump(initial_data, f, indent=4) + json.dump(dict(sorted(json_fixtures.items())), f, indent=4) @classmethod def from_file( diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index 7ad50c3a244..03543cedf52 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -15,8 +15,8 @@ from typing import Any, Dict, Generator, List, Type import pytest +import xdist from _pytest.terminal import TerminalReporter -from filelock import FileLock from pytest_metadata.plugin import metadata_key # type: ignore from cli.gen_index import generate_fixtures_index @@ -532,27 +532,6 @@ def create_properties_file( config.write(f) -@pytest.fixture(scope="session", autouse=True) -def create_tarball( - request: pytest.FixtureRequest, output_dir: Path, is_output_tarball: bool -) -> Generator[None, None, None]: - """ - Create a tarball of json files the output directory if the configured - output ends with '.tar.gz'. - - Only include .json and .ini files in the archive. - """ - yield - if is_output_tarball: - source_dir = output_dir - tarball_filename = request.config.getoption("output") - with tarfile.open(tarball_filename, "w:gz") as tar: - for file in source_dir.rglob("*"): - if file.suffix in {".json", ".ini"}: - arcname = Path("fixtures") / file.relative_to(source_dir) - tar.add(file, arcname=arcname) - - @pytest.fixture(scope="function") def dump_dir_parameter_level( request: pytest.FixtureRequest, base_dump_dir: Path | None, filler_path: Path @@ -590,11 +569,6 @@ def get_fixture_collection_scope(fixture_name, config): return "module" -@pytest.fixture(scope="session") -def generate_index(request) -> bool: # noqa: D103 - return request.config.option.generate_index - - @pytest.fixture(scope=get_fixture_collection_scope) def fixture_collector( request: pytest.FixtureRequest, @@ -603,29 +577,11 @@ def fixture_collector( filler_path: Path, base_dump_dir: Path | None, output_dir: Path, - session_temp_folder: Path | None, - generate_index: bool, ) -> Generator[FixtureCollector, None, None]: """ Return configured fixture collector instance used for all tests in one test module. """ - if session_temp_folder is not None: - fixture_collector_count_file_name = "fixture_collector_count" - fixture_collector_count_file = session_temp_folder / fixture_collector_count_file_name - fixture_collector_count_file_lock = ( - session_temp_folder / f"{fixture_collector_count_file_name}.lock" - ) - with FileLock(fixture_collector_count_file_lock): - if fixture_collector_count_file.exists(): - with open(fixture_collector_count_file, "r") as f: - fixture_collector_count = int(f.read()) - else: - fixture_collector_count = 0 - fixture_collector_count += 1 - with open(fixture_collector_count_file, "w") as f: - f.write(str(fixture_collector_count)) - fixture_collector = FixtureCollector( output_dir=output_dir, flat_output=request.config.getoption("flat_output"), @@ -638,19 +594,6 @@ def fixture_collector( if do_fixture_verification: fixture_collector.verify_fixture_files(evm_fixture_verification) - fixture_collector_count = 0 - if session_temp_folder is not None: - with FileLock(fixture_collector_count_file_lock): - with open(fixture_collector_count_file, "r") as f: - fixture_collector_count = int(f.read()) - fixture_collector_count -= 1 - with open(fixture_collector_count_file, "w") as f: - f.write(str(fixture_collector_count)) - if generate_index and fixture_collector_count == 0: - generate_fixtures_index( - output_dir, quiet_mode=True, force_flag=False, disable_infer_format=False - ) - @pytest.fixture(autouse=True, scope="session") def filler_path(request: pytest.FixtureRequest) -> Path: @@ -809,3 +752,44 @@ def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item item.add_marker(mark) if "yul" in item.fixturenames: # type: ignore item.add_marker(pytest.mark.yul_test) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int): + """ + Perform session finish tasks. + + - Remove any lock files that may have been created. + - Generate index file for all produced fixtures. + - Create tarball of the output directory if the output is a tarball. + """ + if xdist.is_xdist_worker(session): + return + + output: Path = session.config.getoption("output") + is_output_tarball = output.suffix == ".gz" and output.with_suffix("").suffix == ".tar" + output_dir = strip_output_tarball_suffix(output) + + if is_output_stdout(output): + # Don't perform any further actions if the output is stdout. + return + + # Remove any lock files that may have been created. + for file in output_dir.rglob("*.lock"): + file.unlink() + + # Generate index file for all produced fixtures. + generate_index = session.config.getoption("generate_index") + if generate_index: + generate_fixtures_index( + output_dir, quiet_mode=True, force_flag=False, disable_infer_format=False + ) + + # Create tarball of the output directory if the output is a tarball. + if is_output_tarball: + source_dir = output_dir + tarball_filename = output + with tarfile.open(tarball_filename, "w:gz") as tar: + for file in source_dir.rglob("*"): + if file.suffix in {".json", ".ini"}: + arcname = Path("fixtures") / file.relative_to(source_dir) + tar.add(file, arcname=arcname) diff --git a/stubs/xdist/__init__.pyi b/stubs/xdist/__init__.pyi new file mode 100644 index 00000000000..1ec85655d44 --- /dev/null +++ b/stubs/xdist/__init__.pyi @@ -0,0 +1,3 @@ +from .methods import is_xdist_worker + +__all__ = ("is_xdist_worker",) diff --git a/stubs/xdist/methods.pyi b/stubs/xdist/methods.pyi new file mode 100644 index 00000000000..c3e9d5bbada --- /dev/null +++ b/stubs/xdist/methods.pyi @@ -0,0 +1,3 @@ +import pytest + +def is_xdist_worker(session: pytest.Session) -> bool: ...