From 2561d07695cdc72c06d0440869510a4b42ec54dd Mon Sep 17 00:00:00 2001 From: "Samuel Hierholzer (Adfinis AG)" Date: Thu, 27 Jun 2024 15:16:44 +0200 Subject: [PATCH 1/6] fix: Don't fail if a to-be archived publish has wrong snapshots --- pyaptly/publish.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pyaptly/publish.py b/pyaptly/publish.py index 8f914af..f64f9f6 100644 --- a/pyaptly/publish.py +++ b/pyaptly/publish.py @@ -85,6 +85,9 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) return cmd publish_fullname = "%s %s" % (publish_name, publish_config["distribution"]) + # This line might fail if no publish has been created yet + # TODO: add clag --create to create publishes when they haven't been created yet + # TODO: Fail gracefully and show an error when there is no existing publish current_snapshots = state_reader.state_reader().publish_map()[publish_fullname] if "snapshots" in publish_config: snapshots_config = publish_config["snapshots"] @@ -129,11 +132,13 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) continue prefix_to_search = re.sub("%T$", "", snap["name"]) - current_snapshot = [ - snap_name - for snap_name in sorted(current_snapshots, key=lambda x: -len(x)) - if snap_name.startswith(prefix_to_search) - ][0] + current_snapshot = None + for snap_name in sorted(current_snapshots, key=lambda x: -len(x)): + if snap_name.startswith(prefix_to_search): + current_snapshot = snap_name + break + if current_snapshot is None: + lg.warn("Snapshot %s doesn't exist on to-be archived publish %s." % (snap["name"], publish_fullname)) snapshot.clone_snapshot(current_snapshot, archive).execute() From 0bd799fb1da0d037df3b329c8ca1a9d5d88e678e Mon Sep 17 00:00:00 2001 From: "Samuel Hierholzer (Adfinis AG)" Date: Wed, 24 Jul 2024 10:23:01 +0200 Subject: [PATCH 2/6] fix: only archive snapshot if possible --- pyaptly/publish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaptly/publish.py b/pyaptly/publish.py index f64f9f6..c79a2cf 100644 --- a/pyaptly/publish.py +++ b/pyaptly/publish.py @@ -139,8 +139,8 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) break if current_snapshot is None: lg.warn("Snapshot %s doesn't exist on to-be archived publish %s." % (snap["name"], publish_fullname)) - - snapshot.clone_snapshot(current_snapshot, archive).execute() + else: + snapshot.clone_snapshot(current_snapshot, archive).execute() publish_cmd.append("switch") options.append("-component=%s" % ",".join(components)) From 8045a6643db03cead38540d1c060ff647fdd9eb6 Mon Sep 17 00:00:00 2001 From: "Samuel Hierholzer (Adfinis AG)" Date: Wed, 24 Jul 2024 10:23:23 +0200 Subject: [PATCH 3/6] fix: Properly fail if publish hasn't been created yet --- pyaptly/publish.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyaptly/publish.py b/pyaptly/publish.py index c79a2cf..bb18a83 100644 --- a/pyaptly/publish.py +++ b/pyaptly/publish.py @@ -85,10 +85,12 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) return cmd publish_fullname = "%s %s" % (publish_name, publish_config["distribution"]) - # This line might fail if no publish has been created yet - # TODO: add clag --create to create publishes when they haven't been created yet + # TODO: add flag --create to create publishes when they haven't been created yet # TODO: Fail gracefully and show an error when there is no existing publish - current_snapshots = state_reader.state_reader().publish_map()[publish_fullname] + try: + current_snapshots = state_reader.state_reader().publish_map()[publish_fullname] + except KeyError as e: + util.exit_with_error("The publish hasn't been created yet: " + e) if "snapshots" in publish_config: snapshots_config = publish_config["snapshots"] new_snapshots = [ From 8ec1cabd3ef38717b3b8439b9a892327ff4a98e8 Mon Sep 17 00:00:00 2001 From: "Samuel Hierholzer (Adfinis AG)" Date: Thu, 7 Nov 2024 14:54:10 +0100 Subject: [PATCH 4/6] style: fix naming & lg.warn -> lg.warning --- pyaptly/conftest.py | 2 +- pyaptly/main.py | 2 +- pyaptly/publish.py | 2 +- pyaptly/tests/test_publish.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyaptly/conftest.py b/pyaptly/conftest.py index 2e383b5..b6b55ed 100644 --- a/pyaptly/conftest.py +++ b/pyaptly/conftest.py @@ -147,7 +147,7 @@ def mirror_update(environment, config): @pytest.fixture() def snapshot_create(config, mirror_update, freeze): - """Test if createing snapshots works.""" + """Test if creating snapshots works.""" args = ["-c", config, "snapshot", "create"] main.main(args) state = state_reader.SystemStateReader() diff --git a/pyaptly/main.py b/pyaptly/main.py index ce3ada6..4b476fb 100755 --- a/pyaptly/main.py +++ b/pyaptly/main.py @@ -51,7 +51,7 @@ def prepare(args): import yaml cfg = yaml.safe_load(f) - lg.warn( + lg.warning( "NOTE: yaml has beed deprecated and will be remove on the next major release." ) else: diff --git a/pyaptly/publish.py b/pyaptly/publish.py index bb18a83..a0f56bc 100644 --- a/pyaptly/publish.py +++ b/pyaptly/publish.py @@ -140,7 +140,7 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) current_snapshot = snap_name break if current_snapshot is None: - lg.warn("Snapshot %s doesn't exist on to-be archived publish %s." % (snap["name"], publish_fullname)) + lg.warning("Snapshot %s doesn't exist on to-be archived publish %s." % (snap["name"], publish_fullname)) else: snapshot.clone_snapshot(current_snapshot, archive).execute() diff --git a/pyaptly/tests/test_publish.py b/pyaptly/tests/test_publish.py index d48c5c5..700cf41 100644 --- a/pyaptly/tests/test_publish.py +++ b/pyaptly/tests/test_publish.py @@ -155,7 +155,7 @@ def test_publish_update_republish(config, publish_create_republish, freeze): @pytest.mark.parametrize("config", ["publish.toml"], indirect=True) -def test_publish_updating_basic(config, publish_create, freeze): +def test_publish_update_basic(config, publish_create, freeze): """Test if updating publishes works.""" freeze.move_to("2012-10-11 10:10:10") args = ["-c", config, "snapshot", "create"] From 7340ac190d264b3d963d660248062878832702f3 Mon Sep 17 00:00:00 2001 From: "Samuel Hierholzer (Adfinis AG)" Date: Thu, 7 Nov 2024 14:55:15 +0100 Subject: [PATCH 5/6] fix: Proper message when updating a missing publish --- pyaptly/publish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyaptly/publish.py b/pyaptly/publish.py index a0f56bc..8a9eaa4 100644 --- a/pyaptly/publish.py +++ b/pyaptly/publish.py @@ -89,8 +89,8 @@ def publish_cmd_update(cfg, publish_name, publish_config, ignore_existing=False) # TODO: Fail gracefully and show an error when there is no existing publish try: current_snapshots = state_reader.state_reader().publish_map()[publish_fullname] - except KeyError as e: - util.exit_with_error("The publish hasn't been created yet: " + e) + except KeyError: # pragma: no cover + util.exit_with_error(f"The publish {publish_fullname} hasn't been created yet.") if "snapshots" in publish_config: snapshots_config = publish_config["snapshots"] new_snapshots = [ From f8a3654b2d854cc1358e2b8cfa43e30032141b7c Mon Sep 17 00:00:00 2001 From: "Samuel Hierholzer (Adfinis AG)" Date: Thu, 7 Nov 2024 14:55:48 +0100 Subject: [PATCH 6/6] test: Add a test updating a publish with bad history --- pyaptly/tests/publish-bad.toml | 43 ++++++++++++++++++++++++++++++++++ pyaptly/tests/test_publish.py | 25 ++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 pyaptly/tests/publish-bad.toml diff --git a/pyaptly/tests/publish-bad.toml b/pyaptly/tests/publish-bad.toml new file mode 100644 index 0000000..b023d17 --- /dev/null +++ b/pyaptly/tests/publish-bad.toml @@ -0,0 +1,43 @@ +[[publish.fakerepo01]] +gpg-key = "6D79A810B9B7ABAE" +skip-contents = true +automatic-update = true +components = "main" +distribution = "main" +snapshots = [ + { name = "fakerepo02-%T", timestamp = "current", archive-on-update = "archived-fakerepo01-%T" }, +] + +[[publish.fakerepo02]] +gpg-key = "6D79A810B9B7ABAE" +automatic-update = true +components = "main" +distribution = "main" +snapshots = [ + { name = "fakerepo01-%T", timestamp = "current", archive-on-update = "archived-fakerepo02-%T" }, +] + +[mirror.fakerepo01] +max-tries = 2 +archive = "http://localhost:3123/fakerepo01" +gpg-keys = [ + "2841988729C7F3FF", +] +components = "main" +distribution = "main" + +[mirror.fakerepo02] +archive = "http://localhost:3123/fakerepo02" +gpg-keys = [ + "2841988729C7F3FF", +] +components = "main" +distribution = "main" + +[snapshot."fakerepo01-%T"] +mirror = "fakerepo01" +timestamp = { time = "00:00" } + +[snapshot."fakerepo02-%T"] +mirror = "fakerepo02" +timestamp = { time = "00:00", repeat-weekly = "sat" } diff --git a/pyaptly/tests/test_publish.py b/pyaptly/tests/test_publish.py index 700cf41..4e80f73 100644 --- a/pyaptly/tests/test_publish.py +++ b/pyaptly/tests/test_publish.py @@ -191,3 +191,28 @@ def test_repo_create_single(config, repo, test_key_03): main.main(args) state = state_reader.SystemStateReader() assert set(["centrify"]) == state.repos() + + +@pytest.mark.parametrize("config", ["publish.toml"], indirect=True) +def test_publish_update_wrong_snapshots(config, publish_create, freeze): + """Test if updating publishes works when the snapshot history is wrong.""" + freeze.move_to("2012-10-11 10:10:10") + args = ["-c", config, "snapshot", "create"] + main.main(args) + args = ["-c", config.replace('.toml','-bad.toml'), "publish", "update"] + main.main(args) + state = state_reader.SystemStateReader() + expect = set( + [ + "fakerepo01-20121011T0000Z", + "fakerepo02-20121006T0000Z", + "fakerepo01-20121010T0000Z", + ] + ) + assert expect == state.snapshots() + # Reversed! + expect2 = { + "fakerepo01 main": set(["fakerepo02-20121006T0000Z"]), + "fakerepo02 main": set(["fakerepo01-20121011T0000Z"]), + } + assert expect2 == state.publish_map()