Skip to content

Commit

Permalink
[Reconfigurator] Use per-sled RNGs (#7321)
Browse files Browse the repository at this point in the history 
Suggested by @sunshowers

(#7307 (comment)).
This should prevent changes to one sled (e.g., adding a new zone) from
perturbing IDs on all subsequent sleds in otherwise-stable tests. (I
took a stab at combining this with #7307 and confirmed the diff output
is much more reasonable; will update #7307 after this lands.)

The actual code changes here are small and mostly-mechanical; the
changes in `rng.rs` are the driver (moving the sled-specific RNGs into a
per-sled RNG container and dealing with handing those out keyed by sled
ID), and everything else fell out from that. Unfortunately this changes
_all_ zone/dataset/NIC/IP UUIDs in expectorate tests, so the diff volume
is... quite large.
  • Loading branch information
@jgallagher
jgallagher authored Jan 8, 2025
1 parent 6efb34e commit f63ed09
Show file tree
Hide file tree
Showing 22 changed files with 2,944 additions and 2,906 deletions.
86 changes: 43 additions & 43 deletions dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,21 +88,21 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9
---------------------------------------------------------------------------------------------
zone type zone id disposition underlay IP
---------------------------------------------------------------------------------------------
clickhouse fe79023f-c5d5-4be5-ad2c-da4e9e9237e4 in service fd00:1122:3344:102::23
crucible 054f64a5-182c-4c28-8994-d2e082550201 in service fd00:1122:3344:102::26
crucible 3b5bffea-e5ed-44df-8468-fd4fa69757d8 in service fd00:1122:3344:102::27
crucible 53dd7fa4-899e-49ed-9fc2-48222db3e20d in service fd00:1122:3344:102::2a
crucible 7db307d4-a6ed-4c47-bddf-6759161bf64a in service fd00:1122:3344:102::2c
crucible 95ad9a1d-4063-4874-974c-2fc92830be27 in service fd00:1122:3344:102::29
crucible bc095417-e2f0-4e95-b390-9cc3fc6e3c6d in service fd00:1122:3344:102::28
crucible d90401f1-fbc2-42cb-bf17-309ee0f922fe in service fd00:1122:3344:102::2b
crucible e8f994c0-0a1b-40e6-8db1-40a8ca89e503 in service fd00:1122:3344:102::2d
crucible e9bf481e-323e-466e-842f-8107078c7137 in service fd00:1122:3344:102::2e
crucible f97aa057-6485-45d0-9cb4-4af5b0831d48 in service fd00:1122:3344:102::25
crucible_pantry eaec16c0-0d44-4847-b2d6-31a5151bae52 in service fd00:1122:3344:102::24
internal_dns 8b8f7c02-7a18-4268-b045-2e286b464c5d in service fd00:1122:3344:1::1
internal_ntp c67dd9a4-0d6c-4e9f-b28d-20003f211f7d in service fd00:1122:3344:102::21
nexus 94b45ce9-d3d8-413a-a76b-865da1f67930 in service fd00:1122:3344:102::22
clickhouse 3a3f243b-7e9e-4818-bb7c-fe30966b2949 in service fd00:1122:3344:102::23
crucible 279b230f-5e77-4960-b08e-594c6f2f57c0 in service fd00:1122:3344:102::2a
crucible 4328425e-f5ba-436a-9e46-3f337f07671e in service fd00:1122:3344:102::2d
crucible 61282e88-43b3-4011-9314-b0929880895a in service fd00:1122:3344:102::2b
crucible 7537db8e-11c9-4a84-9dc7-b3ae7b657cc4 in service fd00:1122:3344:102::2e
crucible b37ebcb3-533b-4fd7-9960-bb1ac511bea2 in service fd00:1122:3344:102::27
crucible b4c3734e-b6d8-47d8-a695-5dad2c21622e in service fd00:1122:3344:102::25
crucible b8aba012-d4b3-48e1-af2d-cf6265e02bd7 in service fd00:1122:3344:102::2c
crucible e1b405aa-a32c-4410-8335-59237a7bc9ad in service fd00:1122:3344:102::28
crucible e696d6f8-c706-4ca7-8846-561f0323ccbf in service fd00:1122:3344:102::29
crucible f69e36ff-ea67-4d1f-bc73-3d2a0315c77f in service fd00:1122:3344:102::26
crucible_pantry c5fefafb-d65c-44e0-b45e-d2097dca02d1 in service fd00:1122:3344:102::24
internal_dns 426face8-6cc4-4ba0-b3a3-8492876ecd37 in service fd00:1122:3344:1::1
internal_ntp eaa48c21-f17c-41f7-9e85-fc6a10913b31 in service fd00:1122:3344:102::21
nexus db1aa26e-7608-4ce0-933e-9968489f8a46 in service fd00:1122:3344:102::22



Expand All @@ -128,20 +128,20 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9
---------------------------------------------------------------------------------------------
zone type zone id disposition underlay IP
---------------------------------------------------------------------------------------------
crucible 09937ebb-bb6a-495b-bc97-b58076b70a78 in service fd00:1122:3344:103::2c
crucible a999e5fa-3edc-4dac-919a-d7b554cdae58 in service fd00:1122:3344:103::26
crucible b416f299-c23c-46c8-9820-be2b66ffea0a in service fd00:1122:3344:103::27
crucible b5d5491d-b3aa-4727-8b55-f66e0581ea4f in service fd00:1122:3344:103::2b
crucible cc1dc86d-bd6f-4929-aa4a-9619012e9393 in service fd00:1122:3344:103::24
crucible cd3bb540-e605-465f-8c62-177ac482d850 in service fd00:1122:3344:103::29
crucible e8971ab3-fb7d-4ad8-aae3-7f2fe87c51f3 in service fd00:1122:3344:103::25
crucible f3628f0a-2301-4fc8-bcbf-961199771731 in service fd00:1122:3344:103::2d
crucible f52aa245-7e1b-46c0-8a31-e09725f02caf in service fd00:1122:3344:103::2a
crucible fae49024-6cec-444d-a6c4-83658ab015a4 in service fd00:1122:3344:103::28
crucible_pantry 728db429-8621-4e1e-9915-282aadfa27d1 in service fd00:1122:3344:103::23
internal_dns e7dd3e98-7fe7-4827-be7f-395ff9a5f542 in service fd00:1122:3344:2::1
internal_ntp 4f2eb088-7d28-4c4e-a27c-746400ec65ba in service fd00:1122:3344:103::21
nexus c8aa84a5-a802-46c9-adcd-d61e9c8393c9 in service fd00:1122:3344:103::22
crucible 096964a1-a60b-4de9-b4b5-dada560870ca in service fd00:1122:3344:103::28
crucible 1a07a7f2-76ae-4670-8491-3383bb3e2d19 in service fd00:1122:3344:103::2d
crucible 1a7ddc8f-90c7-4842-81a9-2abfc76e3cb4 in service fd00:1122:3344:103::26
crucible 2f5dec78-6071-41c1-8f3f-2f4e98fdad0a in service fd00:1122:3344:103::25
crucible 41f7b32f-d85f-4cce-853c-144342cc8361 in service fd00:1122:3344:103::24
crucible 52960cc6-af73-4ae6-b776-b4bcc371fd68 in service fd00:1122:3344:103::2c
crucible 6914b9aa-5712-405c-817a-77b2e6c6a824 in service fd00:1122:3344:103::27
crucible 8e92b0f0-77b7-4b95-905f-653ee962b932 in service fd00:1122:3344:103::2b
crucible d9ea5125-d6f0-4bfd-9ebd-497569d91adf in service fd00:1122:3344:103::2a
crucible f7ced707-a517-4529-91fa-03dc7683f413 in service fd00:1122:3344:103::29
crucible_pantry 25087c5b-58b9-46f2-9e4c-e9440c081111 in service fd00:1122:3344:103::23
internal_dns 5c1386b0-ed6b-4e09-8a65-7d9f47c41839 in service fd00:1122:3344:2::1
internal_ntp 9ec70cc1-a22d-40df-9697-8a4db3c72d74 in service fd00:1122:3344:103::21
nexus 3bfd90d6-0640-4f63-a578-76277ce9c7c6 in service fd00:1122:3344:103::22



Expand All @@ -167,20 +167,20 @@ parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9
---------------------------------------------------------------------------------------------
zone type zone id disposition underlay IP
---------------------------------------------------------------------------------------------
crucible 413d3e02-e19f-400a-9718-a662347538f0 in service fd00:1122:3344:101::24
crucible 6cb330f9-4609-4d6c-98ad-b5cc34245813 in service fd00:1122:3344:101::29
crucible 6d725df0-0189-4429-b270-3eeb891d39c8 in service fd00:1122:3344:101::28
crucible b5443ebd-1f5b-448c-8edc-b4ca25c25db1 in service fd00:1122:3344:101::25
crucible bb55534c-1042-4af4-ad2f-9590803695ac in service fd00:1122:3344:101::27
crucible c4296f9f-f902-4fc7-b896-178e56e60732 in service fd00:1122:3344:101::2d
crucible d14c165f-6370-4cce-9dba-3c6deb762cfc in service fd00:1122:3344:101::2c
crucible de65f128-30f7-422b-a234-d1fc8dd6ef78 in service fd00:1122:3344:101::2b
crucible e135441d-637e-4de9-8023-5ea0096347f3 in service fd00:1122:3344:101::26
crucible fee71ee6-da42-4a7f-a00e-f56b6a3327ce in service fd00:1122:3344:101::2a
crucible_pantry 315a3670-d019-425c-b7a6-c9429428b671 in service fd00:1122:3344:101::23
internal_dns 8b47e1e8-0396-4e44-a4a5-ea891405c9f2 in service fd00:1122:3344:3::1
internal_ntp cbe91cdc-cbb6-4760-aece-6ce08b67e85a in service fd00:1122:3344:101::21
nexus b43ce109-90d6-46f9-9df0-8c68bfe6d4a0 in service fd00:1122:3344:101::22
crucible 157d5b03-6897-4e80-9357-3cf733efe4b5 in service fd00:1122:3344:101::28
crucible 7341456c-4c6c-4bb7-8be4-2acac834886f in service fd00:1122:3344:101::2a
crucible 793a6315-a07b-4fcf-a0b4-633d5c53b8cf in service fd00:1122:3344:101::27
crucible 9915de3b-8104-40ca-a6b5-46132d26bb15 in service fd00:1122:3344:101::29
crucible a975d276-7434-4def-8f5b-f250657d1040 in service fd00:1122:3344:101::2c
crucible b41461de-6b60-4d35-ad90-336eb1fa9874 in service fd00:1122:3344:101::26
crucible d1374f2f-e9ba-4046-ba0b-83da927ba0d3 in service fd00:1122:3344:101::2b
crucible dc3c9584-44d8-4be6-b215-2df289f5763d in service fd00:1122:3344:101::25
crucible e70d6f37-d0a6-4309-93b5-4f2f72b779a7 in service fd00:1122:3344:101::2d
crucible efa9fb1c-9431-4072-877d-ff33d9d926ba in service fd00:1122:3344:101::24
crucible_pantry 761999e7-cf90-412c-91d8-f3247507edbc in service fd00:1122:3344:101::23
internal_dns a2708dbc-a751-4c26-a1ed-6eaadf3402cf in service fd00:1122:3344:3::1
internal_ntp 4bb07cd6-dc94-4601-ac22-c7ad972735b3 in service fd00:1122:3344:101::21
nexus ba910747-f596-4088-a2d4-4372ee883dfd in service fd00:1122:3344:101::22


COCKROACHDB SETTINGS:
Expand Down
42 changes: 21 additions & 21 deletions nexus/reconfigurator/planning/src/blueprint_builder/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1062,7 +1062,7 @@ impl<'a> BlueprintBuilder<'a> {
id: disk_id,
pool_id: *zpool,
},
&mut self.rng,
self.rng.sled_rng(sled_id),
)
.map_err(|err| Error::SledEditError { sled_id, err })?;
}
Expand Down Expand Up @@ -1106,7 +1106,7 @@ impl<'a> BlueprintBuilder<'a> {

let initial_counts = editor.edit_counts();
editor
.ensure_datasets_for_running_zones(&mut self.rng)
.ensure_datasets_for_running_zones(self.rng.sled_rng(sled_id))
.map_err(|err| Error::SledEditError { sled_id, err })?;
let final_counts = editor.edit_counts();

Expand Down Expand Up @@ -1170,7 +1170,7 @@ impl<'a> BlueprintBuilder<'a> {

let zone = BlueprintZoneConfig {
disposition: BlueprintZoneDisposition::InService,
id: self.rng.next_zone(),
id: self.rng.sled_rng(sled_id).next_zone(),
filesystem_pool: Some(zpool),
zone_type,
};
Expand All @@ -1182,15 +1182,15 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let id = self.rng.next_zone();
let id = self.rng.sled_rng(sled_id).next_zone();
let ExternalNetworkingChoice {
external_ip,
nic_ip,
nic_subnet,
nic_mac,
} = self.external_networking()?.for_new_external_dns()?;
let nic = NetworkInterface {
id: self.rng.next_network_interface(),
id: self.rng.sled_rng(sled_id).next_network_interface(),
kind: NetworkInterfaceKind::Service { id: id.into_untyped_uuid() },
name: format!("external-dns-{id}").parse().unwrap(),
ip: nic_ip,
Expand All @@ -1206,7 +1206,7 @@ impl<'a> BlueprintBuilder<'a> {
let http_address =
SocketAddrV6::new(underlay_address, DNS_HTTP_PORT, 0, 0);
let dns_address = OmicronZoneExternalFloatingAddr {
id: self.rng.next_external_ip(),
id: self.rng.sled_rng(sled_id).next_external_ip(),
addr: SocketAddr::new(external_ip, DNS_PORT),
};
let pool_name =
Expand Down Expand Up @@ -1259,7 +1259,7 @@ impl<'a> BlueprintBuilder<'a> {

let zone = BlueprintZoneConfig {
disposition: BlueprintZoneDisposition::InService,
id: self.rng.next_zone(),
id: self.rng.sled_rng(sled_id).next_zone(),
filesystem_pool: Some(filesystem_pool),
zone_type,
};
Expand Down Expand Up @@ -1319,7 +1319,7 @@ impl<'a> BlueprintBuilder<'a> {

let zone = BlueprintZoneConfig {
disposition: BlueprintZoneDisposition::InService,
id: self.rng.next_zone(),
id: self.rng.sled_rng(sled_id).next_zone(),
filesystem_pool: Some(filesystem_pool),
zone_type,
};
Expand Down Expand Up @@ -1386,20 +1386,20 @@ impl<'a> BlueprintBuilder<'a> {
external_tls: bool,
external_dns_servers: Vec<IpAddr>,
) -> Result<(), Error> {
let nexus_id = self.rng.next_zone();
let nexus_id = self.rng.sled_rng(sled_id).next_zone();
let ExternalNetworkingChoice {
external_ip,
nic_ip,
nic_subnet,
nic_mac,
} = self.external_networking()?.for_new_nexus()?;
let external_ip = OmicronZoneExternalFloatingIp {
id: self.rng.next_external_ip(),
id: self.rng.sled_rng(sled_id).next_external_ip(),
ip: external_ip,
};

let nic = NetworkInterface {
id: self.rng.next_network_interface(),
id: self.rng.sled_rng(sled_id).next_network_interface(),
kind: NetworkInterfaceKind::Service {
id: nexus_id.into_untyped_uuid(),
},
Expand Down Expand Up @@ -1439,7 +1439,7 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let oximeter_id = self.rng.next_zone();
let oximeter_id = self.rng.sled_rng(sled_id).next_zone();
let ip = self.sled_alloc_ip(sled_id)?;
let port = omicron_common::address::OXIMETER_PORT;
let address = SocketAddrV6::new(ip, port, 0, 0);
Expand All @@ -1463,7 +1463,7 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let pantry_id = self.rng.next_zone();
let pantry_id = self.rng.sled_rng(sled_id).next_zone();
let ip = self.sled_alloc_ip(sled_id)?;
let port = omicron_common::address::CRUCIBLE_PANTRY_PORT;
let address = SocketAddrV6::new(ip, port, 0, 0);
Expand Down Expand Up @@ -1493,7 +1493,7 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let zone_id = self.rng.next_zone();
let zone_id = self.rng.sled_rng(sled_id).next_zone();
let underlay_ip = self.sled_alloc_ip(sled_id)?;
let pool_name =
self.sled_select_zpool(sled_id, ZoneKind::CockroachDb)?;
Expand All @@ -1519,7 +1519,7 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let id = self.rng.next_zone();
let id = self.rng.sled_rng(sled_id).next_zone();
let underlay_address = self.sled_alloc_ip(sled_id)?;
let address =
SocketAddrV6::new(underlay_address, CLICKHOUSE_HTTP_PORT, 0, 0);
Expand All @@ -1544,7 +1544,7 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let zone_id = self.rng.next_zone();
let zone_id = self.rng.sled_rng(sled_id).next_zone();
let underlay_ip = self.sled_alloc_ip(sled_id)?;
let pool_name =
self.sled_select_zpool(sled_id, ZoneKind::ClickhouseServer)?;
Expand All @@ -1571,7 +1571,7 @@ impl<'a> BlueprintBuilder<'a> {
&mut self,
sled_id: SledUuid,
) -> Result<(), Error> {
let zone_id = self.rng.next_zone();
let zone_id = self.rng.sled_rng(sled_id).next_zone();
let underlay_ip = self.sled_alloc_ip(sled_id)?;
let pool_name =
self.sled_select_zpool(sled_id, ZoneKind::ClickhouseKeeper)?;
Expand Down Expand Up @@ -1670,19 +1670,19 @@ impl<'a> BlueprintBuilder<'a> {
})?;

// Add the new boundary NTP zone.
let new_zone_id = self.rng.next_zone();
let new_zone_id = self.rng.sled_rng(sled_id).next_zone();
let ExternalSnatNetworkingChoice {
snat_cfg,
nic_ip,
nic_subnet,
nic_mac,
} = self.external_networking()?.for_new_boundary_ntp()?;
let external_ip = OmicronZoneExternalSnatIp {
id: self.rng.next_external_ip(),
id: self.rng.sled_rng(sled_id).next_external_ip(),
snat_cfg,
};
let nic = NetworkInterface {
id: self.rng.next_network_interface(),
id: self.rng.sled_rng(sled_id).next_network_interface(),
kind: NetworkInterfaceKind::Service {
id: new_zone_id.into_untyped_uuid(),
},
Expand Down Expand Up @@ -1751,7 +1751,7 @@ impl<'a> BlueprintBuilder<'a> {
))
})?;
editor
.add_zone(zone, &mut self.rng)
.add_zone(zone, self.rng.sled_rng(sled_id))
.map_err(|err| Error::SledEditError { sled_id, err })
}

Expand Down
16 changes: 8 additions & 8 deletions nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
//! Support for editing the blueprint details of a single sled.
use crate::blueprint_builder::SledEditCounts;
use crate::planner::PlannerRng;
use crate::planner::SledPlannerRng;
use illumos_utils::zpool::ZpoolName;
use itertools::Either;
use nexus_sled_agent_shared::inventory::ZoneKind;
Expand Down Expand Up @@ -256,7 +256,7 @@ impl SledEditor {
pub fn ensure_disk(
&mut self,
disk: BlueprintPhysicalDiskConfig,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) -> Result<(), SledEditError> {
self.as_active_mut()?.ensure_disk(disk, rng);
Ok(())
Expand All @@ -272,7 +272,7 @@ impl SledEditor {
pub fn add_zone(
&mut self,
zone: BlueprintZoneConfig,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) -> Result<(), SledEditError> {
self.as_active_mut()?.add_zone(zone, rng)
}
Expand All @@ -289,7 +289,7 @@ impl SledEditor {
/// datasets for all its zones. This method backfills them.
pub fn ensure_datasets_for_running_zones(
&mut self,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) -> Result<(), SledEditError> {
self.as_active_mut()?.ensure_datasets_for_running_zones(rng)
}
Expand Down Expand Up @@ -395,7 +395,7 @@ impl ActiveSledEditor {
pub fn ensure_disk(
&mut self,
disk: BlueprintPhysicalDiskConfig,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) {
let zpool = ZpoolName::new_external(disk.pool_id);

Expand Down Expand Up @@ -427,7 +427,7 @@ impl ActiveSledEditor {
pub fn add_zone(
&mut self,
zone: BlueprintZoneConfig,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) -> Result<(), SledEditError> {
// Ensure we can construct the configs for the datasets for this zone.
let datasets = ZoneDatasetConfigs::new(&self.disks, &zone)?;
Expand Down Expand Up @@ -475,7 +475,7 @@ impl ActiveSledEditor {
/// datasets for all its zones. This method backfills them.
pub fn ensure_datasets_for_running_zones(
&mut self,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) -> Result<(), SledEditError> {
for zone in self.zones.zones(BlueprintZoneFilter::ShouldBeRunning) {
ZoneDatasetConfigs::new(&self.disks, zone)?
Expand Down Expand Up @@ -548,7 +548,7 @@ impl ZoneDatasetConfigs {
fn ensure_in_service(
self,
datasets: &mut DatasetsEditor,
rng: &mut PlannerRng,
rng: &mut SledPlannerRng,
) {
if let Some(dataset) = self.filesystem {
datasets.ensure_in_service(dataset, rng);
Expand Down
Loading

0 comments on commit f63ed09

Please sign in to comment.