From 667832c397ff2f4490db8d98455e220380e3d978 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 9 Jan 2025 10:47:38 -0500 Subject: [PATCH 1/2] [reconfigurator-cli] Fix blueprint-edit failing if there are multiple expunged zones with the same external IP (#7307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I tried to reproduce #7305 using just reconfigurator-cli's simulated example system. This required one change (adding external DNS zones, db90b4a6528edb84e7d6095170c5a3646956fad6), and while I was here I smoothed over a couple of minor ergonomic annoyances (275e24e115059eb5ae9ca5dd5635d133596bf173 is uncontroversial, I hope, and a147954743afc15ff0f73c306c0ff2c4c62ad7ae which is a little less clearly good - happy to back this one out if folks don't like it). With these changes, I can get to a reproduction: ``` generated RNG seed: discernibly-adaptive-grison 〉load-example loaded example system with: - collection: ed006741-bed9-4fd1-87e7-26c95a6f4dd9 - blueprint: 72183372-6429-4ab7-af83-5fea1a40bbcf 〉blueprint-show 72183372-6429-4ab7-af83-5fea1a40bbcf ... snip, finding an external DNS zone to expunge ... 〉blueprint-edit 72183372-6429-4ab7-af83-5fea1a40bbcf expunge-zone 34feb176-1129-4ca8-8df8-6e6cb1be2173 blueprint 9a5f95b4-7eb7-4b8d-b953-5fc9f403f330 created from blueprint 72183372-6429-4ab7-af83-5fea1a40bbcf: expunged zone 34feb176-1129-4ca8-8df8-6e6cb1be2173 from sled e78b0535-959d-4ba9-8d8c-ac3d591fc490 〉blueprint-plan 9a5f95b4-7eb7-4b8d-b953-5fc9f403f330 ... snip, finding the ID of the newly-generated blueprint ... 〉blueprint-diff 9a5f95b4-7eb7-4b8d-b953-5fc9f403f330 0a054c71-82cb-4589-afa3-09ced1e425b9 ... snip, finding the ID of the newly-added external DNS zone ... 〉blueprint-edit 0a054c71-82cb-4589-afa3-09ced1e425b9 expunge-zone 900df298-9c72-4a64-a4a7-7a92b722f68c error: adding omicron zone external IP: associating Omicron zone 900df298-9c72-4a64-a4a7-7a92b722f68c with Floating(OmicronZoneExternalFloatingIp { id: 1c962069-7ced-4162-a11f-acb40fd954f4 (external_ip), ip: 198.51.100.3 }) failed due to duplicates: duplicate entry: OmicronZoneExternalIpEntry { zone_id: 900df298-9c72-4a64-a4a7-7a92b722f68c (service), ip: Floating(OmicronZoneExternalFloatingIp { id: 1c962069-7ced-4162-a11f-acb40fd954f4 (external_ip), ip: 198.51.100.3 }) } conflicts with existing: [OmicronZoneExternalIpEntry { zone_id: 34feb176-1129-4ca8-8df8-6e6cb1be2173 (service), ip: Floating(OmicronZoneExternalFloatingIp { id: 1b31bc81-c2ef-4915-be62-c97bd7881b07 (external_ip), ip: 198.51.100.3 }) }] ``` Fixes #7305 --- Cargo.lock | 3 +- dev-tools/reconfigurator-cli/Cargo.toml | 1 + dev-tools/reconfigurator-cli/src/main.rs | 52 +- .../tests/input/cmds-expunge-newly-added.txt | 10 + .../tests/output/cmd-example-stdout | 15 +- .../output/cmd-expunge-newly-added-stderr | 0 .../output/cmd-expunge-newly-added-stdout | 444 ++++++++++++++++++ .../reconfigurator-cli/tests/test_basic.rs | 24 + 8 files changed, 534 insertions(+), 15 deletions(-) create mode 100644 dev-tools/reconfigurator-cli/tests/input/cmds-expunge-newly-added.txt create mode 100644 dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stderr create mode 100644 dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stdout diff --git a/Cargo.lock b/Cargo.lock index 7c4f36da46..3bde26679f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9447,6 +9447,7 @@ dependencies = [ "humantime", "indent_write", "internal-dns-types", + "itertools 0.13.0", "nexus-client", "nexus-db-queries", "nexus-inventory", @@ -13298,7 +13299,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/dev-tools/reconfigurator-cli/Cargo.toml b/dev-tools/reconfigurator-cli/Cargo.toml index 0f8afd7b76..881484aa08 100644 --- a/dev-tools/reconfigurator-cli/Cargo.toml +++ b/dev-tools/reconfigurator-cli/Cargo.toml @@ -20,6 +20,7 @@ colored.workspace = true humantime.workspace = true indent_write.workspace = true internal-dns-types.workspace = true +itertools.workspace = true nexus-inventory.workspace = true nexus-reconfigurator-planning.workspace = true nexus-reconfigurator-simulation.workspace = true diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs index 7842535c7f..f1ce603965 100644 --- a/dev-tools/reconfigurator-cli/src/main.rs +++ b/dev-tools/reconfigurator-cli/src/main.rs @@ -12,6 +12,7 @@ use clap::ValueEnum; use clap::{Args, Parser, Subcommand}; use indent_write::fmt::IndentWriter; use internal_dns_types::diff::DnsDiff; +use itertools::Itertools; use log_capture::LogCapture; use nexus_inventory::CollectionBuilder; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; @@ -120,8 +121,8 @@ impl ReconfiguratorSim { builder.set_internal_dns_version(parent_blueprint.internal_dns_version); builder.set_external_dns_version(parent_blueprint.external_dns_version); - for (_, zone) in - parent_blueprint.all_omicron_zones(BlueprintZoneFilter::All) + for (_, zone) in parent_blueprint + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) { if let Some((external_ip, nic)) = zone.zone_type.external_networking() @@ -404,7 +405,10 @@ struct BlueprintPlanArgs { /// id of the blueprint on which this one will be based parent_blueprint_id: Uuid, /// id of the inventory collection to use in planning - collection_id: CollectionUuid, + /// + /// Must be provided unless there is only one collection in the loaded + /// state. + collection_id: Option, } #[derive(Debug, Args)] @@ -431,7 +435,7 @@ enum BlueprintEditCommands { /// add a CockroachDB instance to a particular sled AddCockroach { sled_id: SledUuid }, /// expunge a particular zone from a particular sled - ExpungeZone { sled_id: SledUuid, zone_id: OmicronZoneUuid }, + ExpungeZone { zone_id: OmicronZoneUuid }, } #[derive(Debug, Args)] @@ -762,10 +766,25 @@ fn cmd_blueprint_plan( let parent_blueprint_id = args.parent_blueprint_id; let collection_id = args.collection_id; let parent_blueprint = system.get_blueprint(parent_blueprint_id)?; - let collection = system.get_collection(collection_id)?; + let collection = match collection_id { + Some(collection_id) => system.get_collection(collection_id)?, + None => { + let mut all_collections_iter = system.all_collections(); + match all_collections_iter.len() { + 0 => bail!("cannot plan blueprint with no loaded collections"), + 1 => all_collections_iter.next().expect("iter length is 1"), + _ => bail!( + "blueprint-plan: must specify collection ID (one of {:?})", + all_collections_iter.map(|c| c.id).join(", ") + ), + } + } + }; let creator = "reconfigurator-sim"; - let planning_input = sim.planning_input(parent_blueprint)?; + let planning_input = sim + .planning_input(parent_blueprint) + .context("failed to construct planning input")?; let planner = Planner::new_based_on( sim.log.clone(), parent_blueprint, @@ -799,7 +818,9 @@ fn cmd_blueprint_edit( let blueprint_id = args.blueprint_id; let blueprint = system.get_blueprint(blueprint_id)?; let creator = args.creator.as_deref().unwrap_or("reconfigurator-cli"); - let planning_input = sim.planning_input(blueprint)?; + let planning_input = sim + .planning_input(blueprint) + .context("failed to create planning input")?; // TODO: We may want to do something other than just using the latest // collection -- add a way to specify which collection to use. @@ -836,7 +857,20 @@ fn cmd_blueprint_edit( .context("failed to add CockroachDB zone")?; format!("added CockroachDB zone to sled {}", sled_id) } - BlueprintEditCommands::ExpungeZone { sled_id, zone_id } => { + BlueprintEditCommands::ExpungeZone { zone_id } => { + let mut parent_sled_id = None; + for sled_id in builder.sled_ids_with_zones() { + if builder + .current_sled_zones(sled_id, BlueprintZoneFilter::All) + .any(|z| z.id == zone_id) + { + parent_sled_id = Some(sled_id); + break; + } + } + let Some(sled_id) = parent_sled_id else { + bail!("could not find parent sled for zone {zone_id}"); + }; builder .sled_expunge_zone(sled_id, zone_id) .context("failed to expunge zone")?; @@ -1279,6 +1313,8 @@ fn cmd_load_example( .num_nexus() .map_or(NEXUS_REDUNDANCY, |n| n.into()), ) + .external_dns_count(3) + .context("invalid external DNS zone count")? .create_zones(!args.no_zones) .create_disks_in_blueprint(!args.no_disks_in_blueprint) .build(); diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds-expunge-newly-added.txt b/dev-tools/reconfigurator-cli/tests/input/cmds-expunge-newly-added.txt new file mode 100644 index 0000000000..3cee31ba7a --- /dev/null +++ b/dev-tools/reconfigurator-cli/tests/input/cmds-expunge-newly-added.txt @@ -0,0 +1,10 @@ +load-example + +blueprint-show 3f00b694-1b16-4aaa-8f78-e6b3a527b434 +blueprint-edit 3f00b694-1b16-4aaa-8f78-e6b3a527b434 expunge-zone 9995de32-dd52-4eb1-b0eb-141eb84bc739 + +blueprint-show 366b0b68-d80e-4bc1-abd3-dc69837847e0 +blueprint-plan 366b0b68-d80e-4bc1-abd3-dc69837847e0 + +blueprint-show 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 +blueprint-edit 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 expunge-zone d786ef4a-5acb-4f5d-a732-a00addf986b5 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout index ac6587781f..00636a8380 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout @@ -91,15 +91,16 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 clickhouse 3a3f243b-7e9e-4818-bb7c-fe30966b2949 in service fd00:1122:3344:102::23 crucible 279b230f-5e77-4960-b08e-594c6f2f57c0 in service fd00:1122:3344:102::2a crucible 4328425e-f5ba-436a-9e46-3f337f07671e in service fd00:1122:3344:102::2d + crucible 4e60ff64-155b-44e8-9d39-e6de8c5d5fd3 in service fd00:1122:3344:102::2f crucible 61282e88-43b3-4011-9314-b0929880895a in service fd00:1122:3344:102::2b crucible 7537db8e-11c9-4a84-9dc7-b3ae7b657cc4 in service fd00:1122:3344:102::2e crucible b37ebcb3-533b-4fd7-9960-bb1ac511bea2 in service fd00:1122:3344:102::27 - crucible b4c3734e-b6d8-47d8-a695-5dad2c21622e in service fd00:1122:3344:102::25 crucible b8aba012-d4b3-48e1-af2d-cf6265e02bd7 in service fd00:1122:3344:102::2c crucible e1b405aa-a32c-4410-8335-59237a7bc9ad in service fd00:1122:3344:102::28 crucible e696d6f8-c706-4ca7-8846-561f0323ccbf in service fd00:1122:3344:102::29 crucible f69e36ff-ea67-4d1f-bc73-3d2a0315c77f in service fd00:1122:3344:102::26 - crucible_pantry c5fefafb-d65c-44e0-b45e-d2097dca02d1 in service fd00:1122:3344:102::24 + crucible_pantry b4c3734e-b6d8-47d8-a695-5dad2c21622e in service fd00:1122:3344:102::25 + external_dns c5fefafb-d65c-44e0-b45e-d2097dca02d1 in service fd00:1122:3344:102::24 internal_dns 426face8-6cc4-4ba0-b3a3-8492876ecd37 in service fd00:1122:3344:1::1 internal_ntp eaa48c21-f17c-41f7-9e85-fc6a10913b31 in service fd00:1122:3344:102::21 nexus db1aa26e-7608-4ce0-933e-9968489f8a46 in service fd00:1122:3344:102::22 @@ -132,13 +133,14 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 crucible 1a07a7f2-76ae-4670-8491-3383bb3e2d19 in service fd00:1122:3344:103::2d crucible 1a7ddc8f-90c7-4842-81a9-2abfc76e3cb4 in service fd00:1122:3344:103::26 crucible 2f5dec78-6071-41c1-8f3f-2f4e98fdad0a in service fd00:1122:3344:103::25 - crucible 41f7b32f-d85f-4cce-853c-144342cc8361 in service fd00:1122:3344:103::24 crucible 52960cc6-af73-4ae6-b776-b4bcc371fd68 in service fd00:1122:3344:103::2c crucible 6914b9aa-5712-405c-817a-77b2e6c6a824 in service fd00:1122:3344:103::27 crucible 8e92b0f0-77b7-4b95-905f-653ee962b932 in service fd00:1122:3344:103::2b + crucible a2a98ae0-ee42-4933-9c4b-660123bc693d in service fd00:1122:3344:103::2e crucible d9ea5125-d6f0-4bfd-9ebd-497569d91adf in service fd00:1122:3344:103::2a crucible f7ced707-a517-4529-91fa-03dc7683f413 in service fd00:1122:3344:103::29 - crucible_pantry 25087c5b-58b9-46f2-9e4c-e9440c081111 in service fd00:1122:3344:103::23 + crucible_pantry 41f7b32f-d85f-4cce-853c-144342cc8361 in service fd00:1122:3344:103::24 + external_dns 25087c5b-58b9-46f2-9e4c-e9440c081111 in service fd00:1122:3344:103::23 internal_dns 5c1386b0-ed6b-4e09-8a65-7d9f47c41839 in service fd00:1122:3344:2::1 internal_ntp 9ec70cc1-a22d-40df-9697-8a4db3c72d74 in service fd00:1122:3344:103::21 nexus 3bfd90d6-0640-4f63-a578-76277ce9c7c6 in service fd00:1122:3344:103::22 @@ -170,14 +172,15 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 crucible 157d5b03-6897-4e80-9357-3cf733efe4b5 in service fd00:1122:3344:101::28 crucible 7341456c-4c6c-4bb7-8be4-2acac834886f in service fd00:1122:3344:101::2a crucible 793a6315-a07b-4fcf-a0b4-633d5c53b8cf in service fd00:1122:3344:101::27 + crucible 7ce8eb07-58a7-4f1d-ba61-16db33b6fedd in service fd00:1122:3344:101::2e crucible 9915de3b-8104-40ca-a6b5-46132d26bb15 in service fd00:1122:3344:101::29 crucible a975d276-7434-4def-8f5b-f250657d1040 in service fd00:1122:3344:101::2c crucible b41461de-6b60-4d35-ad90-336eb1fa9874 in service fd00:1122:3344:101::26 crucible d1374f2f-e9ba-4046-ba0b-83da927ba0d3 in service fd00:1122:3344:101::2b crucible dc3c9584-44d8-4be6-b215-2df289f5763d in service fd00:1122:3344:101::25 crucible e70d6f37-d0a6-4309-93b5-4f2f72b779a7 in service fd00:1122:3344:101::2d - crucible efa9fb1c-9431-4072-877d-ff33d9d926ba in service fd00:1122:3344:101::24 - crucible_pantry 761999e7-cf90-412c-91d8-f3247507edbc in service fd00:1122:3344:101::23 + crucible_pantry efa9fb1c-9431-4072-877d-ff33d9d926ba in service fd00:1122:3344:101::24 + external_dns 761999e7-cf90-412c-91d8-f3247507edbc in service fd00:1122:3344:101::23 internal_dns a2708dbc-a751-4c26-a1ed-6eaadf3402cf in service fd00:1122:3344:3::1 internal_ntp 4bb07cd6-dc94-4601-ac22-c7ad972735b3 in service fd00:1122:3344:101::21 nexus ba910747-f596-4088-a2d4-4372ee883dfd in service fd00:1122:3344:101::22 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stderr b/dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stdout new file mode 100644 index 0000000000..3b722317b1 --- /dev/null +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-expunge-newly-added-stdout @@ -0,0 +1,444 @@ +using provided RNG seed: test_expunge_newly_added_external_dns +> load-example +loaded example system with: +- collection: 8bc58dd6-dd6e-44e2-9671-e35a2a941c98 +- blueprint: 3f00b694-1b16-4aaa-8f78-e6b3a527b434 + +> + +> blueprint-show 3f00b694-1b16-4aaa-8f78-e6b3a527b434 +blueprint 3f00b694-1b16-4aaa-8f78-e6b3a527b434 +parent: 06c88262-f435-410e-ba98-101bed41ec27 + + sled: 711ac7f8-d19e-4572-bdb9-e9b50f6e362a (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-3b46c403-ad14-435c-a1a8-e8f940bf814f + fake-vendor fake-model serial-3f8c9484-06e8-4662-9a90-aa7e92c43405 + fake-vendor fake-model serial-4a62a827-4bf3-45d5-a7f5-d080f25c61ef + fake-vendor fake-model serial-5f774f00-52b7-41b5-a57f-6f38037196f5 + fake-vendor fake-model serial-68ae41d4-99ed-4612-99e1-fecf795ca694 + fake-vendor fake-model serial-6a66241b-b595-423d-84ef-a81b5d8430e8 + fake-vendor fake-model serial-7c45c3f6-6369-40d9-a73f-2f7ed0afe96b + fake-vendor fake-model serial-a216d334-4a9a-49dd-8b13-20548839306c + fake-vendor fake-model serial-c9ff8eb0-807c-40ad-a5c4-0d534947c9ad + fake-vendor fake-model serial-fb29d469-7d3f-47b9-944c-ce817fc70370 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse e9d3a6d6-6e95-4ec8-a857-a3ab1ce6e62d in service fd00:1122:3344:103::23 + crucible 53491964-307a-45f6-b277-79a5e90f20b7 in service fd00:1122:3344:103::2b + crucible 777087c0-4fb9-43ed-846b-16aa6431a272 in service fd00:1122:3344:103::2f + crucible 78a45052-138b-47ee-877d-e73143467c8a in service fd00:1122:3344:103::2a + crucible 8ec567c1-6776-4573-bfe5-f3041165fdca in service fd00:1122:3344:103::28 + crucible 8ef7ff46-81f7-4f16-aa8f-0c1957589a5f in service fd00:1122:3344:103::2e + crucible af3e8274-151f-4cb1-990c-b0a7680ff210 in service fd00:1122:3344:103::27 + crucible c3114d7e-07cd-4d80-b82b-49ccbe856af2 in service fd00:1122:3344:103::29 + crucible d23b5191-04de-4b48-8599-cf866ffc06e8 in service fd00:1122:3344:103::2d + crucible efdbf061-321c-47d6-bf0f-4530ec4c287e in service fd00:1122:3344:103::26 + crucible f31ab1c5-0f23-44dc-8504-c67c65afc11e in service fd00:1122:3344:103::2c + crucible_pantry 07738018-1b2a-4efc-8c5c-bd6b7a5d7b40 in service fd00:1122:3344:103::25 + external_dns 1b37911d-c427-4bf1-90d3-b2d0c2e98825 in service fd00:1122:3344:103::24 + internal_dns 7a1bd482-74f8-4562-b1db-c8bad16afb44 in service fd00:1122:3344:1::1 + internal_ntp 8c368ed6-6c91-4437-b8ff-0d789193db38 in service fd00:1122:3344:103::21 + nexus ed2d3be3-fcee-4f6b-bb99-a1a9130c4eeb in service fd00:1122:3344:103::22 + + + + sled: 9dc50690-f9bf-4520-bf80-051d0f465c2c (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-1f9589d8-0a68-47e8-b977-d0fb17bd3fdb + fake-vendor fake-model serial-44882b6c-5e19-418b-b6c3-065f2af5a557 + fake-vendor fake-model serial-6de47efc-8a6d-4108-bf82-0146eab3be06 + fake-vendor fake-model serial-80e2c62f-052c-4580-8252-7af238fbbe9c + fake-vendor fake-model serial-81d326ae-5f8a-4ffd-9d5e-a9e8246ac014 + fake-vendor fake-model serial-878af5a0-7810-43e5-bdd5-a3215242459a + fake-vendor fake-model serial-af59fef5-8258-4852-be1d-ce55ae7dc822 + fake-vendor fake-model serial-b16aa11f-6e49-44c1-abcb-2e7584bffa12 + fake-vendor fake-model serial-f173c79b-a3b4-4f4a-a983-bc94b6b1a616 + fake-vendor fake-model serial-f1a041cc-85c7-4d14-8fc0-8d0e417f7e24 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 426cd521-e6ad-4a67-ad8d-a1acef5eab5e in service fd00:1122:3344:102::27 + crucible 4a392139-7423-459b-b1c9-5903421be40c in service fd00:1122:3344:102::2d + crucible 51e403e5-a703-4cfb-b044-c72c52885e82 in service fd00:1122:3344:102::26 + crucible 55a2094a-9590-442f-8203-dffe5a76ae66 in service fd00:1122:3344:102::25 + crucible 758d9a91-7070-43e3-8c60-ea1d0f818a12 in service fd00:1122:3344:102::29 + crucible 8e4d80f8-80ea-48ac-ad47-95501c0c3fbe in service fd00:1122:3344:102::2a + crucible cbf109e0-cdd5-48f9-8998-e9f55c9be980 in service fd00:1122:3344:102::2e + crucible d60cc034-46c4-4233-8066-4008e947d904 in service fd00:1122:3344:102::2c + crucible f24b63e6-2382-4862-90bf-c39e42aad988 in service fd00:1122:3344:102::2b + crucible fddd2bd6-a535-4e2c-9a36-cb90d048c0b7 in service fd00:1122:3344:102::28 + crucible_pantry eb4f0a57-7386-4216-bcb2-874d39aae836 in service fd00:1122:3344:102::24 + external_dns 47387358-e16d-4c34-b8f7-1f3f12c662eb in service fd00:1122:3344:102::23 + internal_dns edb6573c-3621-4d95-ad3b-93a088caeea5 in service fd00:1122:3344:2::1 + internal_ntp 6e1cdf48-6846-4bb3-8a72-8bc1e8d5b21d in service fd00:1122:3344:102::21 + nexus 2cea0275-1fbc-410c-819e-2ec54ffeeffe in service fd00:1122:3344:102::22 + + + + sled: a88790de-5962-4871-8686-61c1fd5b7094 (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-2f26e76b-6c91-4ee9-87b3-f424b942091b + fake-vendor fake-model serial-6f8fa855-4f34-42db-a6e4-9d0090d6c828 + fake-vendor fake-model serial-71450d62-791e-4068-9882-8a206a465fd9 + fake-vendor fake-model serial-7b4ad242-8330-4c08-9588-c66782742678 + fake-vendor fake-model serial-9bf23b52-565e-4439-9728-edb603fa6c4e + fake-vendor fake-model serial-c9476e3d-7745-4fa9-b336-b54ac5b08f56 + fake-vendor fake-model serial-d2cd1e65-b63d-4748-895f-aafecc81e440 + fake-vendor fake-model serial-d4ad3cc1-956a-4444-81a6-da6a025f6df2 + fake-vendor fake-model serial-e1298a43-fa1a-4e6f-bcfa-b26996f69c50 + fake-vendor fake-model serial-fa9ce87c-fa7c-4854-95bd-69b8f01c46f9 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0903299a-6296-44f0-8ab9-7c70b5766a05 in service fd00:1122:3344:101::25 + crucible 2b6b3cfd-4524-465b-ac6e-10be9ab6d4d4 in service fd00:1122:3344:101::27 + crucible 3e0ff677-2ca8-4124-9fef-7c23ac2da6fa in service fd00:1122:3344:101::2d + crucible 512758d0-335f-4c94-9afe-c82f6f127421 in service fd00:1122:3344:101::2a + crucible 8ae67b12-ca74-40d3-a55e-90456cd623ea in service fd00:1122:3344:101::26 + crucible 918c8cbc-1c62-4de5-9cf2-c4153a3d1a7e in service fd00:1122:3344:101::2e + crucible be422e7a-16d0-426e-8567-da0aed7200d4 in service fd00:1122:3344:101::28 + crucible ea50f438-1b5e-4d61-bd65-46d360f590ee in service fd00:1122:3344:101::2b + crucible f055a042-9d1b-4931-8268-d94cac801d7e in service fd00:1122:3344:101::29 + crucible f33e09d1-e3d7-4341-baf2-5f079f3679e5 in service fd00:1122:3344:101::2c + crucible_pantry 100e20aa-1816-4195-8d76-08beace10332 in service fd00:1122:3344:101::24 + external_dns 9995de32-dd52-4eb1-b0eb-141eb84bc739 in service fd00:1122:3344:101::23 + internal_dns 171cabf0-8635-42e7-8ea1-2a1d2a2daf55 in service fd00:1122:3344:3::1 + internal_ntp 7b4bccad-9869-48da-8320-2dc82752deea in service fd00:1122:3344:101::21 + nexus 5b01b0e0-5cdf-4b34-b0a6-c947cbbdb8be in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) + cluster.preserve_downgrade_option: (do not modify) + + METADATA: + created by::::::::::: test suite + created at::::::::::: + comment:::::::::::::: (none) + internal DNS version: 1 + external DNS version: 1 + + + +> blueprint-edit 3f00b694-1b16-4aaa-8f78-e6b3a527b434 expunge-zone 9995de32-dd52-4eb1-b0eb-141eb84bc739 +blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0 created from blueprint 3f00b694-1b16-4aaa-8f78-e6b3a527b434: expunged zone 9995de32-dd52-4eb1-b0eb-141eb84bc739 from sled a88790de-5962-4871-8686-61c1fd5b7094 + +> + +> blueprint-show 366b0b68-d80e-4bc1-abd3-dc69837847e0 +blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0 +parent: 3f00b694-1b16-4aaa-8f78-e6b3a527b434 + + sled: 711ac7f8-d19e-4572-bdb9-e9b50f6e362a (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-3b46c403-ad14-435c-a1a8-e8f940bf814f + fake-vendor fake-model serial-3f8c9484-06e8-4662-9a90-aa7e92c43405 + fake-vendor fake-model serial-4a62a827-4bf3-45d5-a7f5-d080f25c61ef + fake-vendor fake-model serial-5f774f00-52b7-41b5-a57f-6f38037196f5 + fake-vendor fake-model serial-68ae41d4-99ed-4612-99e1-fecf795ca694 + fake-vendor fake-model serial-6a66241b-b595-423d-84ef-a81b5d8430e8 + fake-vendor fake-model serial-7c45c3f6-6369-40d9-a73f-2f7ed0afe96b + fake-vendor fake-model serial-a216d334-4a9a-49dd-8b13-20548839306c + fake-vendor fake-model serial-c9ff8eb0-807c-40ad-a5c4-0d534947c9ad + fake-vendor fake-model serial-fb29d469-7d3f-47b9-944c-ce817fc70370 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse e9d3a6d6-6e95-4ec8-a857-a3ab1ce6e62d in service fd00:1122:3344:103::23 + crucible 53491964-307a-45f6-b277-79a5e90f20b7 in service fd00:1122:3344:103::2b + crucible 777087c0-4fb9-43ed-846b-16aa6431a272 in service fd00:1122:3344:103::2f + crucible 78a45052-138b-47ee-877d-e73143467c8a in service fd00:1122:3344:103::2a + crucible 8ec567c1-6776-4573-bfe5-f3041165fdca in service fd00:1122:3344:103::28 + crucible 8ef7ff46-81f7-4f16-aa8f-0c1957589a5f in service fd00:1122:3344:103::2e + crucible af3e8274-151f-4cb1-990c-b0a7680ff210 in service fd00:1122:3344:103::27 + crucible c3114d7e-07cd-4d80-b82b-49ccbe856af2 in service fd00:1122:3344:103::29 + crucible d23b5191-04de-4b48-8599-cf866ffc06e8 in service fd00:1122:3344:103::2d + crucible efdbf061-321c-47d6-bf0f-4530ec4c287e in service fd00:1122:3344:103::26 + crucible f31ab1c5-0f23-44dc-8504-c67c65afc11e in service fd00:1122:3344:103::2c + crucible_pantry 07738018-1b2a-4efc-8c5c-bd6b7a5d7b40 in service fd00:1122:3344:103::25 + external_dns 1b37911d-c427-4bf1-90d3-b2d0c2e98825 in service fd00:1122:3344:103::24 + internal_dns 7a1bd482-74f8-4562-b1db-c8bad16afb44 in service fd00:1122:3344:1::1 + internal_ntp 8c368ed6-6c91-4437-b8ff-0d789193db38 in service fd00:1122:3344:103::21 + nexus ed2d3be3-fcee-4f6b-bb99-a1a9130c4eeb in service fd00:1122:3344:103::22 + + + + sled: 9dc50690-f9bf-4520-bf80-051d0f465c2c (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-1f9589d8-0a68-47e8-b977-d0fb17bd3fdb + fake-vendor fake-model serial-44882b6c-5e19-418b-b6c3-065f2af5a557 + fake-vendor fake-model serial-6de47efc-8a6d-4108-bf82-0146eab3be06 + fake-vendor fake-model serial-80e2c62f-052c-4580-8252-7af238fbbe9c + fake-vendor fake-model serial-81d326ae-5f8a-4ffd-9d5e-a9e8246ac014 + fake-vendor fake-model serial-878af5a0-7810-43e5-bdd5-a3215242459a + fake-vendor fake-model serial-af59fef5-8258-4852-be1d-ce55ae7dc822 + fake-vendor fake-model serial-b16aa11f-6e49-44c1-abcb-2e7584bffa12 + fake-vendor fake-model serial-f173c79b-a3b4-4f4a-a983-bc94b6b1a616 + fake-vendor fake-model serial-f1a041cc-85c7-4d14-8fc0-8d0e417f7e24 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 426cd521-e6ad-4a67-ad8d-a1acef5eab5e in service fd00:1122:3344:102::27 + crucible 4a392139-7423-459b-b1c9-5903421be40c in service fd00:1122:3344:102::2d + crucible 51e403e5-a703-4cfb-b044-c72c52885e82 in service fd00:1122:3344:102::26 + crucible 55a2094a-9590-442f-8203-dffe5a76ae66 in service fd00:1122:3344:102::25 + crucible 758d9a91-7070-43e3-8c60-ea1d0f818a12 in service fd00:1122:3344:102::29 + crucible 8e4d80f8-80ea-48ac-ad47-95501c0c3fbe in service fd00:1122:3344:102::2a + crucible cbf109e0-cdd5-48f9-8998-e9f55c9be980 in service fd00:1122:3344:102::2e + crucible d60cc034-46c4-4233-8066-4008e947d904 in service fd00:1122:3344:102::2c + crucible f24b63e6-2382-4862-90bf-c39e42aad988 in service fd00:1122:3344:102::2b + crucible fddd2bd6-a535-4e2c-9a36-cb90d048c0b7 in service fd00:1122:3344:102::28 + crucible_pantry eb4f0a57-7386-4216-bcb2-874d39aae836 in service fd00:1122:3344:102::24 + external_dns 47387358-e16d-4c34-b8f7-1f3f12c662eb in service fd00:1122:3344:102::23 + internal_dns edb6573c-3621-4d95-ad3b-93a088caeea5 in service fd00:1122:3344:2::1 + internal_ntp 6e1cdf48-6846-4bb3-8a72-8bc1e8d5b21d in service fd00:1122:3344:102::21 + nexus 2cea0275-1fbc-410c-819e-2ec54ffeeffe in service fd00:1122:3344:102::22 + + + + sled: a88790de-5962-4871-8686-61c1fd5b7094 (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-2f26e76b-6c91-4ee9-87b3-f424b942091b + fake-vendor fake-model serial-6f8fa855-4f34-42db-a6e4-9d0090d6c828 + fake-vendor fake-model serial-71450d62-791e-4068-9882-8a206a465fd9 + fake-vendor fake-model serial-7b4ad242-8330-4c08-9588-c66782742678 + fake-vendor fake-model serial-9bf23b52-565e-4439-9728-edb603fa6c4e + fake-vendor fake-model serial-c9476e3d-7745-4fa9-b336-b54ac5b08f56 + fake-vendor fake-model serial-d2cd1e65-b63d-4748-895f-aafecc81e440 + fake-vendor fake-model serial-d4ad3cc1-956a-4444-81a6-da6a025f6df2 + fake-vendor fake-model serial-e1298a43-fa1a-4e6f-bcfa-b26996f69c50 + fake-vendor fake-model serial-fa9ce87c-fa7c-4854-95bd-69b8f01c46f9 + + + omicron zones at generation 3: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0903299a-6296-44f0-8ab9-7c70b5766a05 in service fd00:1122:3344:101::25 + crucible 2b6b3cfd-4524-465b-ac6e-10be9ab6d4d4 in service fd00:1122:3344:101::27 + crucible 3e0ff677-2ca8-4124-9fef-7c23ac2da6fa in service fd00:1122:3344:101::2d + crucible 512758d0-335f-4c94-9afe-c82f6f127421 in service fd00:1122:3344:101::2a + crucible 8ae67b12-ca74-40d3-a55e-90456cd623ea in service fd00:1122:3344:101::26 + crucible 918c8cbc-1c62-4de5-9cf2-c4153a3d1a7e in service fd00:1122:3344:101::2e + crucible be422e7a-16d0-426e-8567-da0aed7200d4 in service fd00:1122:3344:101::28 + crucible ea50f438-1b5e-4d61-bd65-46d360f590ee in service fd00:1122:3344:101::2b + crucible f055a042-9d1b-4931-8268-d94cac801d7e in service fd00:1122:3344:101::29 + crucible f33e09d1-e3d7-4341-baf2-5f079f3679e5 in service fd00:1122:3344:101::2c + crucible_pantry 100e20aa-1816-4195-8d76-08beace10332 in service fd00:1122:3344:101::24 + external_dns 9995de32-dd52-4eb1-b0eb-141eb84bc739 expunged fd00:1122:3344:101::23 + internal_dns 171cabf0-8635-42e7-8ea1-2a1d2a2daf55 in service fd00:1122:3344:3::1 + internal_ntp 7b4bccad-9869-48da-8320-2dc82752deea in service fd00:1122:3344:101::21 + nexus 5b01b0e0-5cdf-4b34-b0a6-c947cbbdb8be in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) + cluster.preserve_downgrade_option: (do not modify) + + METADATA: + created by::::::::::: reconfigurator-cli + created at::::::::::: + comment:::::::::::::: (none) + internal DNS version: 1 + external DNS version: 1 + + + +> blueprint-plan 366b0b68-d80e-4bc1-abd3-dc69837847e0 +INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 +INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CruciblePantry zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO added zone to sled, sled_id: a88790de-5962-4871-8686-61c1fd5b7094, kind: ExternalDns +INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 +INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify +generated blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 based on parent blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0 + +> + +> blueprint-show 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 +blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 +parent: 366b0b68-d80e-4bc1-abd3-dc69837847e0 + + sled: 711ac7f8-d19e-4572-bdb9-e9b50f6e362a (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-3b46c403-ad14-435c-a1a8-e8f940bf814f + fake-vendor fake-model serial-3f8c9484-06e8-4662-9a90-aa7e92c43405 + fake-vendor fake-model serial-4a62a827-4bf3-45d5-a7f5-d080f25c61ef + fake-vendor fake-model serial-5f774f00-52b7-41b5-a57f-6f38037196f5 + fake-vendor fake-model serial-68ae41d4-99ed-4612-99e1-fecf795ca694 + fake-vendor fake-model serial-6a66241b-b595-423d-84ef-a81b5d8430e8 + fake-vendor fake-model serial-7c45c3f6-6369-40d9-a73f-2f7ed0afe96b + fake-vendor fake-model serial-a216d334-4a9a-49dd-8b13-20548839306c + fake-vendor fake-model serial-c9ff8eb0-807c-40ad-a5c4-0d534947c9ad + fake-vendor fake-model serial-fb29d469-7d3f-47b9-944c-ce817fc70370 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse e9d3a6d6-6e95-4ec8-a857-a3ab1ce6e62d in service fd00:1122:3344:103::23 + crucible 53491964-307a-45f6-b277-79a5e90f20b7 in service fd00:1122:3344:103::2b + crucible 777087c0-4fb9-43ed-846b-16aa6431a272 in service fd00:1122:3344:103::2f + crucible 78a45052-138b-47ee-877d-e73143467c8a in service fd00:1122:3344:103::2a + crucible 8ec567c1-6776-4573-bfe5-f3041165fdca in service fd00:1122:3344:103::28 + crucible 8ef7ff46-81f7-4f16-aa8f-0c1957589a5f in service fd00:1122:3344:103::2e + crucible af3e8274-151f-4cb1-990c-b0a7680ff210 in service fd00:1122:3344:103::27 + crucible c3114d7e-07cd-4d80-b82b-49ccbe856af2 in service fd00:1122:3344:103::29 + crucible d23b5191-04de-4b48-8599-cf866ffc06e8 in service fd00:1122:3344:103::2d + crucible efdbf061-321c-47d6-bf0f-4530ec4c287e in service fd00:1122:3344:103::26 + crucible f31ab1c5-0f23-44dc-8504-c67c65afc11e in service fd00:1122:3344:103::2c + crucible_pantry 07738018-1b2a-4efc-8c5c-bd6b7a5d7b40 in service fd00:1122:3344:103::25 + external_dns 1b37911d-c427-4bf1-90d3-b2d0c2e98825 in service fd00:1122:3344:103::24 + internal_dns 7a1bd482-74f8-4562-b1db-c8bad16afb44 in service fd00:1122:3344:1::1 + internal_ntp 8c368ed6-6c91-4437-b8ff-0d789193db38 in service fd00:1122:3344:103::21 + nexus ed2d3be3-fcee-4f6b-bb99-a1a9130c4eeb in service fd00:1122:3344:103::22 + + + + sled: 9dc50690-f9bf-4520-bf80-051d0f465c2c (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-1f9589d8-0a68-47e8-b977-d0fb17bd3fdb + fake-vendor fake-model serial-44882b6c-5e19-418b-b6c3-065f2af5a557 + fake-vendor fake-model serial-6de47efc-8a6d-4108-bf82-0146eab3be06 + fake-vendor fake-model serial-80e2c62f-052c-4580-8252-7af238fbbe9c + fake-vendor fake-model serial-81d326ae-5f8a-4ffd-9d5e-a9e8246ac014 + fake-vendor fake-model serial-878af5a0-7810-43e5-bdd5-a3215242459a + fake-vendor fake-model serial-af59fef5-8258-4852-be1d-ce55ae7dc822 + fake-vendor fake-model serial-b16aa11f-6e49-44c1-abcb-2e7584bffa12 + fake-vendor fake-model serial-f173c79b-a3b4-4f4a-a983-bc94b6b1a616 + fake-vendor fake-model serial-f1a041cc-85c7-4d14-8fc0-8d0e417f7e24 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 426cd521-e6ad-4a67-ad8d-a1acef5eab5e in service fd00:1122:3344:102::27 + crucible 4a392139-7423-459b-b1c9-5903421be40c in service fd00:1122:3344:102::2d + crucible 51e403e5-a703-4cfb-b044-c72c52885e82 in service fd00:1122:3344:102::26 + crucible 55a2094a-9590-442f-8203-dffe5a76ae66 in service fd00:1122:3344:102::25 + crucible 758d9a91-7070-43e3-8c60-ea1d0f818a12 in service fd00:1122:3344:102::29 + crucible 8e4d80f8-80ea-48ac-ad47-95501c0c3fbe in service fd00:1122:3344:102::2a + crucible cbf109e0-cdd5-48f9-8998-e9f55c9be980 in service fd00:1122:3344:102::2e + crucible d60cc034-46c4-4233-8066-4008e947d904 in service fd00:1122:3344:102::2c + crucible f24b63e6-2382-4862-90bf-c39e42aad988 in service fd00:1122:3344:102::2b + crucible fddd2bd6-a535-4e2c-9a36-cb90d048c0b7 in service fd00:1122:3344:102::28 + crucible_pantry eb4f0a57-7386-4216-bcb2-874d39aae836 in service fd00:1122:3344:102::24 + external_dns 47387358-e16d-4c34-b8f7-1f3f12c662eb in service fd00:1122:3344:102::23 + internal_dns edb6573c-3621-4d95-ad3b-93a088caeea5 in service fd00:1122:3344:2::1 + internal_ntp 6e1cdf48-6846-4bb3-8a72-8bc1e8d5b21d in service fd00:1122:3344:102::21 + nexus 2cea0275-1fbc-410c-819e-2ec54ffeeffe in service fd00:1122:3344:102::22 + + + + sled: a88790de-5962-4871-8686-61c1fd5b7094 (active) + + physical disks at generation 2: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-2f26e76b-6c91-4ee9-87b3-f424b942091b + fake-vendor fake-model serial-6f8fa855-4f34-42db-a6e4-9d0090d6c828 + fake-vendor fake-model serial-71450d62-791e-4068-9882-8a206a465fd9 + fake-vendor fake-model serial-7b4ad242-8330-4c08-9588-c66782742678 + fake-vendor fake-model serial-9bf23b52-565e-4439-9728-edb603fa6c4e + fake-vendor fake-model serial-c9476e3d-7745-4fa9-b336-b54ac5b08f56 + fake-vendor fake-model serial-d2cd1e65-b63d-4748-895f-aafecc81e440 + fake-vendor fake-model serial-d4ad3cc1-956a-4444-81a6-da6a025f6df2 + fake-vendor fake-model serial-e1298a43-fa1a-4e6f-bcfa-b26996f69c50 + fake-vendor fake-model serial-fa9ce87c-fa7c-4854-95bd-69b8f01c46f9 + + + omicron zones at generation 4: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0903299a-6296-44f0-8ab9-7c70b5766a05 in service fd00:1122:3344:101::25 + crucible 2b6b3cfd-4524-465b-ac6e-10be9ab6d4d4 in service fd00:1122:3344:101::27 + crucible 3e0ff677-2ca8-4124-9fef-7c23ac2da6fa in service fd00:1122:3344:101::2d + crucible 512758d0-335f-4c94-9afe-c82f6f127421 in service fd00:1122:3344:101::2a + crucible 8ae67b12-ca74-40d3-a55e-90456cd623ea in service fd00:1122:3344:101::26 + crucible 918c8cbc-1c62-4de5-9cf2-c4153a3d1a7e in service fd00:1122:3344:101::2e + crucible be422e7a-16d0-426e-8567-da0aed7200d4 in service fd00:1122:3344:101::28 + crucible ea50f438-1b5e-4d61-bd65-46d360f590ee in service fd00:1122:3344:101::2b + crucible f055a042-9d1b-4931-8268-d94cac801d7e in service fd00:1122:3344:101::29 + crucible f33e09d1-e3d7-4341-baf2-5f079f3679e5 in service fd00:1122:3344:101::2c + crucible_pantry 100e20aa-1816-4195-8d76-08beace10332 in service fd00:1122:3344:101::24 + external_dns 9995de32-dd52-4eb1-b0eb-141eb84bc739 expunged fd00:1122:3344:101::23 + external_dns d786ef4a-5acb-4f5d-a732-a00addf986b5 in service fd00:1122:3344:101::2f + internal_dns 171cabf0-8635-42e7-8ea1-2a1d2a2daf55 in service fd00:1122:3344:3::1 + internal_ntp 7b4bccad-9869-48da-8320-2dc82752deea in service fd00:1122:3344:101::21 + nexus 5b01b0e0-5cdf-4b34-b0a6-c947cbbdb8be in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) + cluster.preserve_downgrade_option: (do not modify) + + METADATA: + created by::::::::::: reconfigurator-sim + created at::::::::::: + comment:::::::::::::: (none) + internal DNS version: 1 + external DNS version: 1 + + + +> blueprint-edit 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 expunge-zone d786ef4a-5acb-4f5d-a732-a00addf986b5 +blueprint 2ac8c740-444d-42ff-8d66-9812a7e51288 created from blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2: expunged zone d786ef4a-5acb-4f5d-a732-a00addf986b5 from sled a88790de-5962-4871-8686-61c1fd5b7094 + diff --git a/dev-tools/reconfigurator-cli/tests/test_basic.rs b/dev-tools/reconfigurator-cli/tests/test_basic.rs index 749f090e06..14573fa461 100644 --- a/dev-tools/reconfigurator-cli/tests/test_basic.rs +++ b/dev-tools/reconfigurator-cli/tests/test_basic.rs @@ -84,6 +84,30 @@ fn test_example() { assert_contents("tests/output/cmd-example-stderr", &stderr_text); } +// Run tests to expunge an external DNS zone, plan (which should add it again), +// then expunge the newly-added zone. +#[test] +fn test_expunge_newly_added_external_dns() { + let (exit_status, stdout_text, stderr_text) = run_cli( + "tests/input/cmds-expunge-newly-added.txt", + &["--seed", "test_expunge_newly_added_external_dns"], + ); + assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); + + // The example system uses a fixed seed, which means that UUIDs are + // deterministic. Some of the test commands also use those UUIDs, and it's + // convenient for everyone if they aren't redacted. + let stdout_text = Redactor::default().uuids(false).do_redact(&stdout_text); + assert_contents( + "tests/output/cmd-expunge-newly-added-stdout", + &stdout_text, + ); + assert_contents( + "tests/output/cmd-expunge-newly-added-stderr", + &stderr_text, + ); +} + type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; From d9a6a8bf063e5948ff2ed13a59a98759b55e0e7c Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Thu, 9 Jan 2025 09:45:37 -0800 Subject: [PATCH 2/2] add end-to-end test that new blueprint on a fresh system is a noop (#7323) --- Cargo.lock | 6 + end-to-end-tests/Cargo.toml | 6 + end-to-end-tests/src/helpers/ctx.rs | 2 +- end-to-end-tests/src/lib.rs | 1 + end-to-end-tests/src/noop_blueprint.rs | 124 +++++++++++++++++++ nexus/types/src/deployment/blueprint_diff.rs | 36 ++++++ 6 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 end-to-end-tests/src/noop_blueprint.rs diff --git a/Cargo.lock b/Cargo.lock index 3bde26679f..40bd736257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2884,9 +2884,12 @@ dependencies = [ "hickory-resolver", "http", "humantime", + "internal-dns-resolver", + "internal-dns-types", "internet-checksum", "ispf", "macaddr", + "nexus-client", "omicron-sled-agent", "omicron-test-utils", "omicron-workspace-hack", @@ -2898,7 +2901,10 @@ dependencies = [ "serde", "serde_json", "sled-agent-types", + "slog", + "slog-error-chain", "socket2", + "thiserror 1.0.69", "tokio", "toml 0.8.19", "uuid", diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index b5e63179da..bbf5eb20f6 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -15,6 +15,9 @@ bytes.workspace = true chrono.workspace = true http.workspace = true futures.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true +nexus-client.workspace = true omicron-sled-agent.workspace = true omicron-test-utils.workspace = true oxide-client.workspace = true @@ -25,6 +28,9 @@ russh-keys = "0.45.0" serde.workspace = true serde_json.workspace = true sled-agent-types.workspace = true +slog.workspace = true +slog-error-chain.workspace = true +thiserror.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } toml.workspace = true hickory-resolver.workspace = true diff --git a/end-to-end-tests/src/helpers/ctx.rs b/end-to-end-tests/src/helpers/ctx.rs index 5363557502..3301988b8a 100644 --- a/end-to-end-tests/src/helpers/ctx.rs +++ b/end-to-end-tests/src/helpers/ctx.rs @@ -73,7 +73,7 @@ impl Context { } } -fn rss_config() -> Result { +pub fn rss_config() -> Result { let path = "/opt/oxide/sled-agent/pkg/config-rss.toml"; let content = std::fs::read_to_string(&path).unwrap_or(RSS_CONFIG_STR.to_string()); diff --git a/end-to-end-tests/src/lib.rs b/end-to-end-tests/src/lib.rs index 330dc446b4..5994da687a 100644 --- a/end-to-end-tests/src/lib.rs +++ b/end-to-end-tests/src/lib.rs @@ -2,3 +2,4 @@ pub mod helpers; mod instance_launch; mod no_spoof; +mod noop_blueprint; diff --git a/end-to-end-tests/src/noop_blueprint.rs b/end-to-end-tests/src/noop_blueprint.rs new file mode 100644 index 0000000000..66fdf33772 --- /dev/null +++ b/end-to-end-tests/src/noop_blueprint.rs @@ -0,0 +1,124 @@ +//! Test that generating a new blueprint on a freshly-installed system will not +//! change anything. + +#![cfg(test)] + +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; +use nexus_client::Client as NexusClient; +use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; +use omicron_test_utils::dev::test_setup_log; +use slog::{debug, info}; +use slog_error_chain::InlineErrorChain; +use std::time::Duration; +use thiserror::Error; + +/// Test that generating a new blueprint on a freshly-installed system will not +/// change anything. +/// +/// If this test fails, there's probably a bug somewhere. Maybe the initial +/// system blueprint is incorrect or incomplete or maybe the planner is doing +/// the wrong thing after initial setup. +#[tokio::test] +async fn new_blueprint_noop() { + // In order to check anything with blueprints, we need to reach the Nexus + // internal API. This in turn requires finding the internal DNS servers. + let rss_config = + crate::helpers::ctx::rss_config().expect("loading RSS config"); + let rack_subnet = &rss_config.rack_network_config.rack_subnet; + println!("rack subnet: {}", rack_subnet); + let logctx = test_setup_log("new_blueprint_noop"); + let resolver = + Resolver::new_from_ip(logctx.log.clone(), rack_subnet.addr()) + .expect("creating internal DNS resolver"); + + // Wait up to 5 minutes to get a working Nexus client. + let nexus_client = wait_for_condition( + || async { + match make_nexus_client(&resolver, &logctx.log).await { + Ok(nexus_client) => Ok(nexus_client), + Err(e) => { + debug!( + &logctx.log, + "obtaining a working Nexus client failed"; + InlineErrorChain::new(&e), + ); + Err(CondCheckError::<()>::NotYet) + } + } + }, + &Duration::from_millis(500), + &Duration::from_secs(300), + ) + .await + .expect("timed out waiting to obtain a working Nexus client"); + println!("Nexus is running and has a target blueprint"); + + // Now generate a new blueprint. + let new_blueprint = nexus_client + .blueprint_regenerate() + .await + .expect("failed to generate new blueprint") + .into_inner(); + println!("new blueprint generated: {}", new_blueprint.id); + let parent_blueprint_id = new_blueprint + .parent_blueprint_id + .expect("generated blueprint always has a parent"); + println!("parent blueprint id: {}", parent_blueprint_id); + + // Fetch its parent. + let parent_blueprint = nexus_client + .blueprint_view(&parent_blueprint_id) + .await + .expect("failed to fetch parent blueprint") + .into_inner(); + + let diff = new_blueprint.diff_since_blueprint(&parent_blueprint); + println!("new blueprint: {}", new_blueprint.id); + println!("differences:"); + println!("{}", diff.display()); + + if diff.has_changes() { + panic!( + "unexpected changes between initial blueprint and \ + newly-generated one (see above)" + ); + } + + logctx.cleanup_successful(); +} + +/// Error returned by [`make_nexus_client()`]. +#[derive(Debug, Error)] +enum MakeNexusError { + #[error("looking up Nexus IP in internal DNS")] + Resolve(#[from] internal_dns_resolver::ResolveError), + #[error("making request to Nexus")] + Request(#[from] nexus_client::Error), +} + +/// Make one attempt to look up the IP of Nexus in internal DNS and make an HTTP +/// request to its internal API to fetch its current target blueprint. +/// +/// If this succeeds, Nexus is ready for the rest of this test to proceed. +/// +/// Returns a client for this Nexus. +async fn make_nexus_client( + resolver: &Resolver, + log: &slog::Logger, +) -> Result { + debug!(log, "doing DNS lookup for Nexus"); + let nexus_ip = resolver.lookup_socket_v6(ServiceName::Nexus).await?; + let url = format!("http://{}", nexus_ip); + debug!(log, "found Nexus IP"; "nexus_ip" => %nexus_ip, "url" => &url); + + let client = NexusClient::new(&url, log.clone()); + + // Once this call succeeds, Nexus is ready for us to proceed. + let blueprint_response = client.blueprint_target_view().await?.into_inner(); + info!(log, "found target blueprint (Nexus is ready)"; + "target_blueprint" => ?blueprint_response + ); + + Ok(client) +} diff --git a/nexus/types/src/deployment/blueprint_diff.rs b/nexus/types/src/deployment/blueprint_diff.rs index 3c5f7a120d..ebe61c5876 100644 --- a/nexus/types/src/deployment/blueprint_diff.rs +++ b/nexus/types/src/deployment/blueprint_diff.rs @@ -14,6 +14,7 @@ use super::{ zone_sort_key, Blueprint, ClickhouseClusterConfig, CockroachDbPreserveDowngrade, DiffBeforeClickhouseClusterConfig, }; +use diffus::Diffable; use nexus_sled_agent_shared::inventory::ZoneKind; use omicron_common::api::external::Generation; use omicron_common::disk::DiskIdentity; @@ -946,6 +947,41 @@ impl BlueprintDiff { pub fn display(&self) -> BlueprintDiffDisplay<'_> { BlueprintDiffDisplay::new(self) } + + /// Returns whether the diff reflects any changes or if the blueprints are + /// equivalent. + pub fn has_changes(&self) -> bool { + // Any changes to physical disks, datasets, or zones would be reflected + // in `self.sleds_modified`, `self.sleds_added`, or + // `self.sleds_removed`. + if !self.sleds_modified.is_empty() + || !self.sleds_added.is_empty() + || !self.sleds_removed.is_empty() + { + return true; + } + + // The clickhouse cluster config has changed if: + // - there was one before and now there isn't + // - there wasn't one before and now there is + // - there's one both before and after and their generation has changed + match ( + &self.before_clickhouse_cluster_config, + &self.after_clickhouse_cluster_config, + ) { + (DiffBeforeClickhouseClusterConfig::Blueprint(None), None) => false, + (DiffBeforeClickhouseClusterConfig::Blueprint(None), Some(_)) => { + true + } + (DiffBeforeClickhouseClusterConfig::Blueprint(Some(_)), None) => { + true + } + ( + DiffBeforeClickhouseClusterConfig::Blueprint(Some(before)), + Some(after), + ) => before.diff(&after).is_change(), + } + } } /// A printable representation of `ClickhouseClusterConfig` diff tables where