From f2e8de34d33c4f897ecfc66e325371c955bf5dbf Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 23 Jun 2025 19:11:43 +1200 Subject: [PATCH 01/42] [reconfigurator] CLI support for setting RoT versions --- Cargo.lock | 1 + dev-tools/reconfigurator-cli/Cargo.toml | 1 + dev-tools/reconfigurator-cli/src/lib.rs | 113 ++++++++++++ gateway-types/src/rot.rs | 11 ++ nexus/reconfigurator/planning/src/system.rs | 165 +++++++++++++++++- nexus/reconfigurator/simulation/src/system.rs | 8 + 6 files changed, 291 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fce0308c10..05067935a31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10338,6 +10338,7 @@ dependencies = [ "datatest-stable", "dropshot", "expectorate", + "gateway-types", "humantime", "indent_write", "internal-dns-types", diff --git a/dev-tools/reconfigurator-cli/Cargo.toml b/dev-tools/reconfigurator-cli/Cargo.toml index 668a4c9df8c..112ea1fd39d 100644 --- a/dev-tools/reconfigurator-cli/Cargo.toml +++ b/dev-tools/reconfigurator-cli/Cargo.toml @@ -21,6 +21,7 @@ humantime.workspace = true indent_write.workspace = true internal-dns-types.workspace = true itertools.workspace = true +gateway-types.workspace = true newtype-uuid.workspace = true nexus-inventory.workspace = true nexus-reconfigurator-blippy.workspace = true diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index 9f36886fd7e..ee105921ffc 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -8,6 +8,7 @@ use anyhow::{Context, anyhow, bail}; use camino::Utf8PathBuf; use clap::ValueEnum; use clap::{Args, Parser, Subcommand}; +use gateway_types::rot::RotSlot; use indent_write::fmt::IndentWriter; use internal_dns_types::diff::DnsDiff; use itertools::Itertools; @@ -209,6 +210,7 @@ fn process_command( Commands::SledRemove(args) => cmd_sled_remove(sim, args), Commands::SledShow(args) => cmd_sled_show(sim, args), Commands::SledSetPolicy(args) => cmd_sled_set_policy(sim, args), + Commands::SledUpdateRot(args) => cmd_sled_update_rot(sim, args), Commands::SledUpdateSp(args) => cmd_sled_update_sp(sim, args), Commands::SiloList => cmd_silo_list(sim), Commands::SiloAdd(args) => cmd_silo_add(sim, args), @@ -263,6 +265,8 @@ enum Commands { SledShow(SledArgs), /// set a sled's policy SledSetPolicy(SledSetPolicyArgs), + /// simulate updating the sled's RoT versions + SledUpdateRot(SledUpdateRotArgs), /// simulate updating the sled's SP versions SledUpdateSp(SledUpdateSpArgs), @@ -390,6 +394,42 @@ struct SledUpdateSpArgs { inactive: Option, } +// TODO-K: Double check which of these need to be optional +#[derive(Debug, Args)] +struct SledUpdateRotArgs { + /// id of the sled + sled_id: SledUuid, + + /// whether we expect the "A" or "B" slot to be active + #[clap(long)] + active_slot: RotSlot, + + /// sets the version reported for the RoT slot a + #[clap(long, required_unless_present_any = &["slot_b"])] + slot_a: Option, + + /// sets the version reported for the RoT slot b + #[clap(long, required_unless_present_any = &["slot_a"])] + slot_b: Option, + + /// set the persistent boot preference written into the current + /// authoritative CFPA page (ping or pong). + /// Will default to the value of active_version when not set + #[clap(long)] + persistent_boot_preference: RotSlot, + + /// set the persistent boot preference written into the CFPA scratch + /// page that will become the persistent boot preference in the authoritative + /// CFPA page upon reboot, unless CFPA update of the authoritative page fails + /// for some reason + #[clap(long)] + pending_persistent_boot_preference: Option, + + /// override persistent preference selection for a single boot + #[clap(long)] + transient_boot_preference: Option, +} + #[derive(Debug, Args)] struct SledRemoveArgs { /// id of the sled @@ -914,6 +954,9 @@ fn cmd_sled_show( let sled_id = args.sled_id; let sp_active_version = description.sled_sp_active_version(sled_id)?; let sp_inactive_version = description.sled_sp_inactive_version(sled_id)?; + let rot_active_slot = description.sled_rot_active_slot(sled_id)?; + let rot_slot_a_version = description.sled_rot_slot_a_version(sled_id)?; + let rot_slot_b_version = description.sled_rot_slot_b_version(sled_id)?; let planning_input = description .to_planning_input_builder() .context("failed to generate planning_input builder")? @@ -926,6 +969,10 @@ fn cmd_sled_show( swriteln!(s, "subnet {}", sled_resources.subnet.net()); swriteln!(s, "SP active version: {:?}", sp_active_version); swriteln!(s, "SP inactive version: {:?}", sp_inactive_version); + swriteln!(s, "RoT active slot: {}", rot_active_slot); + // TODO-K: Include all other RoT settings? + swriteln!(s, "RoT slot A version: {:?}", rot_slot_a_version); + swriteln!(s, "RoT slot B version: {:?}", rot_slot_b_version); swriteln!(s, "zpools ({}):", sled_resources.zpools.len()); for (zpool, disk) in &sled_resources.zpools { swriteln!(s, " {:?}", zpool); @@ -993,6 +1040,72 @@ fn cmd_sled_update_sp( ))) } +fn cmd_sled_update_rot( + sim: &mut ReconfiguratorSim, + args: SledUpdateRotArgs, +) -> anyhow::Result> { + let mut labels = Vec::new(); + + labels.push(format!("active slot -> {}", &args.active_slot)); + if let Some(slot_a) = &args.slot_a { + labels.push(format!("slot a -> {}", slot_a)); + } + if let Some(slot_b) = &args.slot_b { + labels.push(format!("slot b -> {}", slot_b)); + } + // TODO-K: Do I need these settings as well? + labels.push(format!( + "persistent boot preference -> {}", + &args.persistent_boot_preference + )); + if let Some(pending_persistent_boot_preference) = + &args.pending_persistent_boot_preference + { + labels.push(format!( + "pending persistent boot preference -> {}", + pending_persistent_boot_preference + )); + } + labels.push(format!( + "pending persistent boot preference -> {}", + &args.persistent_boot_preference + )); + if let Some(transient_boot_preference) = &args.transient_boot_preference { + labels.push(format!( + "transient boot preference -> {}", + transient_boot_preference + )); + } + + assert!( + !labels.is_empty(), + "clap configuration requires that at least one argument is specified" + ); + + let mut state = sim.current_state().to_mut(); + state.system_mut().description_mut().sled_update_rot_versions( + args.sled_id, + args.slot_a, + args.slot_b, + )?; + + sim.commit_and_bump( + format!( + "reconfigurator-cli sled-update-rot: {}: {}", + args.sled_id, + labels.join(", "), + ), + state, + ); + + Ok(Some(format!( + // TODO-K: Is "RoT settings" what I want here? + "set sled {} RoT settings: {}", + args.sled_id, + labels.join(", ") + ))) +} + fn cmd_inventory_list( sim: &mut ReconfiguratorSim, ) -> anyhow::Result> { diff --git a/gateway-types/src/rot.rs b/gateway-types/src/rot.rs index 44dd8b8d8d6..77b200b29ca 100644 --- a/gateway-types/src/rot.rs +++ b/gateway-types/src/rot.rs @@ -5,6 +5,7 @@ use daft::Diffable; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::fmt::Display; use std::str::FromStr; #[derive( @@ -200,6 +201,16 @@ impl RotSlot { } } +impl Display for RotSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + RotSlot::A => "A", + RotSlot::B => "B", + }; + write!(f, "{s}") + } +} + impl FromStr for RotSlot { type Err = String; diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index abd9e647e49..eb413ddd7e1 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -467,6 +467,77 @@ impl SystemDescription { Ok(sled.sp_inactive_caboose().map(|c| c.version.as_ref())) } + pub fn sled_sp_state( + &self, + sled_id: SledUuid, + ) -> anyhow::Result> { + let sled = self.sleds.get(&sled_id).with_context(|| { + format!("attempted to access sled {} not found in system", sled_id) + })?; + Ok(sled.sp_state()) + } + + /// Update the RoT versions reported for a sled. + /// + /// Where `None` is provided, no changes are made. + pub fn sled_update_rot_versions( + &mut self, + sled_id: SledUuid, + slot_a_version: Option, + slot_b_version: Option, + ) -> anyhow::Result<&mut Self> { + let sled = self.sleds.get_mut(&sled_id).with_context(|| { + format!("attempted to access sled {} not found in system", sled_id) + })?; + let sled = Arc::make_mut(sled); + sled.set_rot_versions(slot_a_version, slot_b_version); + // TODO-K: Should I include all the other settings as well? + // Is there a point to that? + Ok(self) + } + + pub fn sled_rot_active_slot( + &self, + sled_id: SledUuid, + ) -> anyhow::Result<&RotSlot> { + let sp_state = self.sled_sp_state(sled_id)?; + if let Some((_hw_slot, sp_state)) = sp_state { + let slot = match &sp_state.rot { + RotState::V2 { active, .. } | RotState::V3 { active, .. } => { + active + } + RotState::CommunicationFailed { message } => { + return Err(anyhow!( + "failed to retrieve active RoT slot due to communication failure: {message}" + )); + } + }; + Ok(slot) + } else { + Err(anyhow!("failed to retrieve SP state from sled id: {sled_id}")) + } + } + + pub fn sled_rot_slot_a_version( + &self, + sled_id: SledUuid, + ) -> anyhow::Result> { + let sled = self.sleds.get(&sled_id).with_context(|| { + format!("attempted to access sled {} not found in system", sled_id) + })?; + Ok(sled.rot_slot_a_caboose().map(|c| c.version.as_ref())) + } + + pub fn sled_rot_slot_b_version( + &self, + sled_id: SledUuid, + ) -> anyhow::Result> { + let sled = self.sleds.get(&sled_id).with_context(|| { + format!("attempted to access sled {} not found in system", sled_id) + })?; + Ok(sled.rot_slot_b_caboose().map(|c| c.version.as_ref())) + } + pub fn set_target_release( &mut self, tuf_repo: Option, @@ -703,6 +774,8 @@ pub struct SledHwInventory<'a> { pub rot: &'a nexus_types::inventory::RotState, pub sp_active: Option>, pub sp_inactive: Option>, + pub rot_slot_a: Option>, + pub rot_slot_b: Option>, } /// Our abstract description of a `Sled` @@ -719,6 +792,8 @@ pub struct Sled { resources: SledResources, sp_active_caboose: Option>, sp_inactive_caboose: Option>, + rot_slot_a_caboose: Option>, + rot_slot_b_caboose: Option>, } impl Sled { @@ -865,10 +940,14 @@ impl Sled { }, state: SledState::Active, resources: SledResources { subnet: sled_subnet, zpools }, - sp_active_caboose: Some(Arc::new(Self::default_sp_caboose( - String::from("0.0.1"), - ))), + sp_active_caboose: Some(Arc::new( + Self::default_sp_component_caboose(String::from("0.0.1")), + )), sp_inactive_caboose: None, + rot_slot_a_caboose: Some(Arc::new( + Self::default_sp_component_caboose(String::from("0.0.2")), + )), + rot_slot_b_caboose: None, } } @@ -903,6 +982,10 @@ impl Sled { inventory_sp.as_ref().and_then(|hw| hw.sp_active.clone()); let sp_inactive_caboose = inventory_sp.as_ref().and_then(|hw| hw.sp_inactive.clone()); + let rot_slot_a_caboose = + inventory_sp.as_ref().and_then(|hw| hw.rot_slot_a.clone()); + let rot_slot_b_caboose = + inventory_sp.as_ref().and_then(|hw| hw.rot_slot_b.clone()); let inventory_sp = inventory_sp.map(|sledhw| { // RotStateV3 unconditionally sets all of these let sp_state = if sledhw.rot.slot_a_sha3_256_digest.is_some() @@ -1012,6 +1095,8 @@ impl Sled { resources: sled_resources, sp_active_caboose, sp_inactive_caboose, + rot_slot_a_caboose, + rot_slot_b_caboose, } } @@ -1042,6 +1127,14 @@ impl Sled { &self.inventory_sled_agent } + fn rot_slot_a_caboose(&self) -> Option<&Caboose> { + self.rot_slot_a_caboose.as_deref() + } + + fn rot_slot_b_caboose(&self) -> Option<&Caboose> { + self.rot_slot_b_caboose.as_deref() + } + fn sp_active_caboose(&self) -> Option<&Caboose> { self.sp_active_caboose.as_deref() } @@ -1065,7 +1158,7 @@ impl Sled { Arc::make_mut(caboose).version = active_version.to_string() } new @ None => { - *new = Some(Arc::new(Self::default_sp_caboose( + *new = Some(Arc::new(Self::default_sp_component_caboose( active_version.to_string(), ))); } @@ -1083,9 +1176,65 @@ impl Sled { Arc::make_mut(caboose).version = v.to_string() } new @ None => { - *new = Some(Arc::new(Self::default_sp_caboose( - v.to_string(), - ))); + *new = Some(Arc::new( + Self::default_sp_component_caboose( + v.to_string(), + ), + )); + } + } + } + } + } + } + + /// Update the reported RoT versions + /// + /// If either field is `None`, that field is _unchanged_. + // Note that this means there's no way to _unset_ the version. + fn set_rot_versions( + &mut self, + slot_a_version: Option, + slot_b_version: Option, + ) { + if let Some(slot_a_version) = slot_a_version { + match slot_a_version { + ExpectedVersion::NoValidVersion => { + self.rot_slot_a_caboose = None; + } + ExpectedVersion::Version(v) => { + match &mut self.rot_slot_a_caboose { + Some(caboose) => { + Arc::make_mut(caboose).version = v.to_string() + } + new @ None => { + *new = Some(Arc::new( + Self::default_sp_component_caboose( + v.to_string(), + ), + )); + } + } + } + } + } + + if let Some(slot_b_version) = slot_b_version { + match slot_b_version { + ExpectedVersion::NoValidVersion => { + self.rot_slot_b_caboose = None; + } + ExpectedVersion::Version(v) => { + match &mut self.rot_slot_b_caboose { + Some(caboose) => { + Arc::make_mut(caboose).version = v.to_string() + } + new @ None => { + *new = Some(Arc::new( + Self::default_sp_component_caboose( + v.to_string(), + ), + )); } } } @@ -1093,7 +1242,7 @@ impl Sled { } } - fn default_sp_caboose(version: String) -> Caboose { + fn default_sp_component_caboose(version: String) -> Caboose { let board = sp_sim::SIM_GIMLET_BOARD.to_string(); Caboose { board: board.clone(), diff --git a/nexus/reconfigurator/simulation/src/system.rs b/nexus/reconfigurator/simulation/src/system.rs index 828d46dbe7a..d660df666b6 100644 --- a/nexus/reconfigurator/simulation/src/system.rs +++ b/nexus/reconfigurator/simulation/src/system.rs @@ -705,6 +705,12 @@ impl SimSystemBuilderInner { let sp_inactive = primary_collection .caboose_for(CabooseWhich::SpSlot1, baseboard_id) .map(|c| c.caboose.clone()); + let rot_slot_a = primary_collection + .caboose_for(CabooseWhich::RotSlotA, baseboard_id) + .map(|c| c.caboose.clone()); + let rot_slot_b = primary_collection + .caboose_for(CabooseWhich::RotSlotB, baseboard_id) + .map(|c| c.caboose.clone()); if let (Some(inv_sp), Some(inv_rot)) = (inv_sp, inv_rot) { Some(SledHwInventory { baseboard_id: &baseboard_id, @@ -712,6 +718,8 @@ impl SimSystemBuilderInner { rot: inv_rot, sp_active, sp_inactive, + rot_slot_a, + rot_slot_b, }) } else { None From fd098df301d5dbc100f0ecfc1749da4d907a95d5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 24 Jun 2025 13:48:40 +1200 Subject: [PATCH 02/42] expectorate --- .../tests/output/cmds-example-stdout | 6 ++++++ .../tests/output/cmds-stdout | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index ddaef941497..353fa170e84 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -42,6 +42,9 @@ serial serial1 subnet fd00:1122:3344:102::/64 SP active version: Some("0.0.1") SP inactive version: None +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 055c4910-b641-46d9-b52d-313aae9d9cbf (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-055c4910-b641-46d9-b52d-313aae9d9cbf" }, disk_id: 6a0cb52f-5cc2-48a5-9f44-ac8dea3ac45b (physical_disk), policy: InService, state: Active } @@ -403,6 +406,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("0.0.1") SP inactive version: None +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (4): 0477165a-a72e-4814-b8d6-74aa02cb2040 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-0477165a-a72e-4814-b8d6-74aa02cb2040" }, disk_id: 6a5a31ab-4edc-44e0-a7a1-4190bfe582f7 (physical_disk), policy: InService, state: Active } diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout index a9b9d834056..7cb56d8b8be 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout @@ -30,6 +30,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("0.0.1") SP inactive version: None +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -77,6 +80,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("1.0.0") SP inactive version: None +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -109,6 +115,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("1.0.0") SP inactive version: Some("2.0.0") +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -141,6 +150,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("3.0.0") SP inactive version: Some("2.0.0") +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -173,6 +185,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: None +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -205,6 +220,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: Some("0.0.2") +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -269,6 +287,9 @@ serial serial0 subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: None +RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } From 799b1691b2e181169b27a2d90ae5d4e3974e798b Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 25 Jun 2025 19:26:55 +1200 Subject: [PATCH 03/42] planner support for configuring RoT update --- Cargo.lock | 1 + dev-tools/reconfigurator-cli/src/lib.rs | 4 - .../tests/input/target-release.txt | 22 +- .../tests/output/cmds-stdout | 2 +- .../tests/output/target-release-stdout | 106 ++++---- .../src/test_util/test_artifacts.rs | 1 + nexus/reconfigurator/planning/Cargo.toml | 1 + .../planning/src/mgs_updates/mod.rs | 242 +++++++++++++++++- nexus/reconfigurator/planning/src/planner.rs | 25 +- nexus/reconfigurator/planning/src/system.rs | 53 +++- nexus/types/src/deployment/planning_input.rs | 8 + nexus/types/src/inventory.rs | 13 + update-common/manifests/fake.toml | 2 +- 13 files changed, 406 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05067935a31..f974941e047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6506,6 +6506,7 @@ dependencies = [ "dropshot", "expectorate", "gateway-client", + "gateway-types", "id-map", "illumos-utils", "indexmap 2.9.0", diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index ee105921ffc..b978e30427e 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -1066,10 +1066,6 @@ fn cmd_sled_update_rot( pending_persistent_boot_preference )); } - labels.push(format!( - "pending persistent boot preference -> {}", - &args.persistent_boot_preference - )); if let Some(transient_boot_preference) = &args.transient_boot_preference { labels.push(format!( "transient boot preference -> {}", diff --git a/dev-tools/reconfigurator-cli/tests/input/target-release.txt b/dev-tools/reconfigurator-cli/tests/input/target-release.txt index 1e224e8f892..e9080d4f38e 100644 --- a/dev-tools/reconfigurator-cli/tests/input/target-release.txt +++ b/dev-tools/reconfigurator-cli/tests/input/target-release.txt @@ -28,7 +28,7 @@ sled-list blueprint-list inventory-list -# First step: upgrade one SP. +# First step: upgrade one RoT. blueprint-plan dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 f45ba181-4b56-42cc-a762-874d90184a43 blueprint-diff dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -36,33 +36,39 @@ blueprint-diff dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f64 blueprint-plan 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 f45ba181-4b56-42cc-a762-874d90184a43 blueprint-diff 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 58d5e830-0884-47d8-a7cd-b2b3751adeb4 -# Now, update the simulated SP to reflect that the update completed. +# Now, update the simulated RoT to reflect that the update completed. # Collect inventory from it and use that collection for another planning step. # This should report that the update completed, remove that update, and add one # for another sled. -sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +# sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a inventory-generate blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 # This time, make it more interesting. Change the inactive slot contents of -# the simulated SP. This should make the configured update impossible and cause +# the simulated RoT. This should make the configured update impossible and cause # the planner to fix it. -sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 +# sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 +sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 --active-slot a --persistent-boot-preference a inventory-generate blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 # Now simulate the update completing successfully. # Another planning step should try to update the last sled. -sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +# sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 --active-slot a --persistent-boot-preference a inventory-generate blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba # Finish updating the last sled and do one more planning run. -# There should be nothing left to do. -sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 +# Now we should see a pending SP update. (TODO: Instead we see there's nothing left to do :( ) +# sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 +sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a inventory-generate blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac + +# TODO-K: Do an SP update as well \ No newline at end of file diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout index 7cb56d8b8be..a6fb66d987b 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout @@ -288,7 +288,7 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") RoT active slot: A -RoT slot A version: None +RoT slot A version: Some("0.0.2") RoT slot B version: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) diff --git a/dev-tools/reconfigurator-cli/tests/output/target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/target-release-stdout index 89c2c495bb0..adf561a5163 100644 --- a/dev-tools/reconfigurator-cli/tests/output/target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/target-release-stdout @@ -24,8 +24,8 @@ target release: unset INFO extracting uploaded archive to INFO created directory to store extracted artifacts, path: INFO added artifact, name: SimGimletSp, kind: gimlet_sp, version: 1.0.0, hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, length: 747 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 INFO added artifact, name: fake-gimlet-rot-bootloader, kind: gimlet_rot_bootloader, version: 1.0.0, hash: 005ea358f1cd316df42465b1e3a0334ea22cc0c0442cf9ddf9b42fbf49780236, length: 750 INFO added artifact, name: fake-host, kind: host_phase_1, version: 1.0.0, hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, length: 524288 INFO added artifact, name: fake-host, kind: host_phase_2, version: 1.0.0, hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, length: 1048576 @@ -53,8 +53,8 @@ external DNS generations: 1 target number of Nexus instances: default target release: 1.0.0 (system-update-v1.0.0.zip) artifact: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 gimlet_sp (SimGimletSp version 1.0.0) - artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_a (fake-gimlet-rot version 1.0.0) - artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_b (fake-gimlet-rot version 1.0.0) + artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_a (SimRot version 1.0.0) + artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_b (SimRot version 1.0.0) artifact: 005ea358f1cd316df42465b1e3a0334ea22cc0c0442cf9ddf9b42fbf49780236 gimlet_rot_bootloader (fake-gimlet-rot-bootloader version 1.0.0) artifact: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89 host_phase_1 (fake-host version 1.0.0) artifact: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008 host_phase_2 (fake-host version 1.0.0) @@ -117,8 +117,8 @@ external DNS generations: 1 target number of Nexus instances: default target release: 1.0.0 (system-update-v1.0.0.zip) artifact: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 gimlet_sp (SimGimletSp version 1.0.0) - artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_a (fake-gimlet-rot version 1.0.0) - artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_b (fake-gimlet-rot version 1.0.0) + artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_a (SimRot version 1.0.0) + artifact: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a gimlet_rot_image_b (SimRot version 1.0.0) artifact: 005ea358f1cd316df42465b1e3a0334ea22cc0c0442cf9ddf9b42fbf49780236 gimlet_rot_bootloader (fake-gimlet-rot-bootloader version 1.0.0) artifact: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89 host_phase_1 (fake-host version 1.0.0) artifact: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008 host_phase_2 (fake-host version 1.0.0) @@ -156,7 +156,7 @@ ID NERRORS TIME_DONE f45ba181-4b56-42cc-a762-874d90184a43 0 -> # First step: upgrade one SP. +> # First step: upgrade one RoT. > blueprint-plan dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 f45ba181-4b56-42cc-a762-874d90184a43 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 @@ -168,7 +168,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 @@ -193,10 +193,10 @@ to: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -+ sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ++ sled 0 model0 serial0 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -350,7 +350,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO SP update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -512,12 +512,13 @@ external DNS: -> # Now, update the simulated SP to reflect that the update completed. +> # Now, update the simulated RoT to reflect that the update completed. > # Collect inventory from it and use that collection for another planning step. > # This should report that the update completed, remove that update, and add one > # for another sled. -> sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 +> # sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +> sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A > inventory-generate generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configured sleds @@ -533,9 +534,9 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO skipping board for SP update, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 @@ -560,11 +561,11 @@ to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } -+ sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 0 model0 serial0 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } ++ sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -707,10 +708,11 @@ external DNS: > # This time, make it more interesting. Change the inactive slot contents of -> # the simulated SP. This should make the configured update impossible and cause +> # the simulated RoT. This should make the configured update impossible and cause > # the planner to fix it. -> sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: inactive -> 0.5.0 +> # sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 +> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 --active-slot a --persistent-boot-preference a +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: active slot -> A, slot b -> 0.5.0, persistent boot preference -> A > inventory-generate generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds @@ -726,8 +728,8 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO SP update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 @@ -752,11 +754,11 @@ to: blueprint df06bb57-ad42-4431-9206-abff322896c7 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -* sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 - Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } - └─ + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) } + -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +* sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + └─ + Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -900,8 +902,9 @@ external DNS: > # Now simulate the update completing successfully. > # Another planning step should try to update the last sled. -> sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: active -> 1.0.0 +> # sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A > inventory-generate generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds @@ -917,10 +920,10 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO skipping board for SP update, serial_number: serial1, part_number: model1 INFO skipping board for SP update, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO ran out of boards for SP update INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 @@ -945,11 +948,11 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) } -+ sled 2 model2 serial2 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + sp_type slot part_number serial_number artifact_hash artifact_version details + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +- sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } ++ sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -1092,9 +1095,10 @@ external DNS: > # Finish updating the last sled and do one more planning run. -> # There should be nothing left to do. -> sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 SP versions: active -> 1.0.0 +> # Now we should see a pending SP update. (TODO: Instead we see there's nothing left to do :( ) +> # sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 +> sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A > inventory-generate generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds @@ -1110,7 +1114,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO skipping board for SP update, serial_number: serial2, part_number: model2 INFO skipping board for SP update, serial_number: serial0, part_number: model0 INFO skipping board for SP update, serial_number: serial1, part_number: model1 @@ -1139,10 +1143,10 @@ to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 2 model2 serial2 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -1283,3 +1287,5 @@ external DNS: + +> # TODO: Do an SP update as well diff --git a/nexus/mgs-updates/src/test_util/test_artifacts.rs b/nexus/mgs-updates/src/test_util/test_artifacts.rs index 33cb4853009..8d9ecfbdd02 100644 --- a/nexus/mgs-updates/src/test_util/test_artifacts.rs +++ b/nexus/mgs-updates/src/test_util/test_artifacts.rs @@ -79,6 +79,7 @@ impl TestArtifacts { // Make an RoT update artifact for SimGimlet. let rot_gimlet_artifact_caboose = CabooseBuilder::default() .git_commit("fake-git-commit") + // TODO-K: change to SIM_ROT .board(SIM_GIMLET_BOARD) .version("0.0.0") .name("fake-name") diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml index 756d7dd604c..9852af62fa5 100644 --- a/nexus/reconfigurator/planning/Cargo.toml +++ b/nexus/reconfigurator/planning/Cargo.toml @@ -13,6 +13,7 @@ chrono.workspace = true debug-ignore.workspace = true daft.workspace = true gateway-client.workspace = true +gateway-types.workspace = true id-map.workspace = true illumos-utils.workspace = true indexmap.workspace = true diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index ee3dc21f24f..0a0b70cc047 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -4,6 +4,8 @@ //! Facilities for making choices about MGS-managed updates +use gateway_types::rot::RotSlot; +use nexus_types::deployment::ExpectedActiveRotSlot; use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; @@ -17,6 +19,7 @@ use slog_error_chain::InlineErrorChain; use std::collections::BTreeSet; use std::sync::Arc; use thiserror::Error; +use tufaceous_artifact::ArtifactKind; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::KnownArtifactKind; @@ -69,6 +72,7 @@ pub fn plan_mgs_updates( boards_preferred.insert(update.baseboard_id.clone()); } Ok(MgsUpdateStatus::Impossible) => { + // TODO-K: Would be nice if it said why it was impossible info!( log, "SP update impossible \ @@ -124,8 +128,10 @@ pub fn plan_mgs_updates( return rv; } + // TODO-K: Add RoT update here match try_make_update(log, board, inventory, current_artifacts) { Some(update) => { + // TODO-K: should this somehow change to say RoT or bootloader or whatever? info!(log, "configuring SP update"; &update); rv.insert(update); } @@ -201,6 +207,7 @@ fn mgs_update_status( .map(|c| c.caboose.version.as_ref()); Ok(mgs_update_status_sp( + log, desired_version, expected_active_version, expected_inactive_version, @@ -208,8 +215,42 @@ fn mgs_update_status( found_inactive_version, )) } - PendingMgsUpdateDetails::Rot { .. } - | PendingMgsUpdateDetails::RotBootloader { .. } => { + PendingMgsUpdateDetails::Rot { + expected_active_slot, + expected_inactive_version, + .. + // TODO-K: incorporate these + // expected_persistent_boot_preference, + // expected_pending_persistent_boot_preference, + // expected_transient_boot_preference, + } => { + + let active_caboose_which = match &expected_active_slot.slot { + RotSlot::A => CabooseWhich::RotSlotA, + RotSlot::B => CabooseWhich::RotSlotB + }; + + let Some(active_caboose) = + inventory.caboose_for(active_caboose_which, baseboard_id) + else { + return Err(MgsUpdateStatusError::MissingActiveCaboose); + }; + + let found_inactive_version = inventory + .caboose_for(active_caboose_which.toggled_slot(), baseboard_id) + .map(|c| c.caboose.version.as_ref()); + + // TODO-K: Make an mgs_update_status_rot? + Ok(mgs_update_status_sp( + log, + desired_version, + &expected_active_slot.version, + expected_inactive_version, + &active_caboose.caboose.version, + found_inactive_version, + )) + } + PendingMgsUpdateDetails::RotBootloader { .. } => { return Err(MgsUpdateStatusError::NotYetImplemented); } }; @@ -257,6 +298,8 @@ fn mgs_update_status( /// Compares a configured SP update with information from inventory and /// determines the current status of the update. See `MgsUpdateStatus`. fn mgs_update_status_sp( + // TODO-K: remove logger + _log: &slog::Logger, desired_version: &ArtifactVersion, expected_active_version: &ArtifactVersion, expected_inactive_version: &ExpectedVersion, @@ -288,6 +331,12 @@ fn mgs_update_status_sp( match (found_inactive_version, expected_inactive_version) { (Some(_), ExpectedVersion::NoValidVersion) => { // We expected nothing in the inactive slot, but found something. + // info!( + // &log, + // "DEBUG: found {:?} != expected {}", + // found_inactive_version, + // expected_inactive_version + // ); MgsUpdateStatus::Impossible } (Some(found), ExpectedVersion::Version(expected)) => { @@ -335,11 +384,16 @@ fn try_make_update( inventory: &Collection, current_artifacts: &TufRepoDescription, ) -> Option { - // TODO When we add support for planning RoT, RoT bootloader, and host OS + // TODO When we add support for planning RoT bootloader, and host OS // updates, we'll try these in a hardcoded priority order until any of them // returns `Some`. The order is described in RFD 565 section "Update - // Sequence". For now, we only plan SP updates. - try_make_update_sp(log, baseboard_id, inventory, current_artifacts) + // Sequence". For now, we only plan SP and RoT updates. + + // let cabooses = &inventory.cabooses_found; + // info!(&log, "DEBUG: Components"; "baseboard-id" => ?baseboard_id, "cabooses" => ?cabooses, "artifacts" => ?current_artifacts); + try_make_update_sp(log, baseboard_id, inventory, current_artifacts); + // TODO-K: RoT must be before SP + try_make_update_rot(log, baseboard_id, inventory, current_artifacts) } /// Determine if the given baseboard needs an SP update and, if so, returns it. @@ -440,6 +494,7 @@ fn try_make_update_sp( // If the artifact's version matches what's deployed, then no update is // needed. if artifact.id.version == expected_active_version { + // TODO-K: bails out here when trying to do an RoT update debug!(log, "no SP update needed for board"; baseboard_id); return None; } @@ -476,6 +531,183 @@ fn try_make_update_sp( }) } +/// Determine if the given baseboard needs an SP update and, if so, returns it. +fn try_make_update_rot( + log: &slog::Logger, + baseboard_id: &Arc, + inventory: &Collection, + current_artifacts: &TufRepoDescription, +) -> Option { + let Some(sp_info) = inventory.sps.get(baseboard_id) else { + warn!( + log, + "cannot configure RoT update for board \ + (missing SP info from inventory)"; + baseboard_id + ); + return None; + }; + + let Some(rot_state) = inventory.rots.get(baseboard_id) else { + warn!( + log, + "cannot configure RoT update for board \ + (missing RoT state from inventory)"; + baseboard_id + ); + return None; + }; + + let active_slot = rot_state.active_slot; + + let active_caboose = match active_slot { + RotSlot::A => CabooseWhich::RotSlotA, + RotSlot::B => CabooseWhich::RotSlotB, + }; + + let Some(active_caboose) = + inventory.caboose_for(active_caboose, baseboard_id) + else { + warn!( + log, + "cannot configure RoT update for board \ + (missing active slot {active_slot} caboose from inventory)"; + baseboard_id, + ); + return None; + }; + + let Ok(expected_active_version) = active_caboose.caboose.version.parse() + else { + warn!( + log, + "cannot configure RoT update for board \ + (cannot parse current active version as an ArtifactVersion)"; + baseboard_id, + "found_version" => &active_caboose.caboose.version, + ); + return None; + }; + + let board = &active_caboose.caboose.board; + let matching_artifacts: Vec<_> = current_artifacts + .artifacts + .iter() + .filter(|a| { + // A matching RoT artifact will have: + // + // - "name" matching the board name (found above from caboose) + // - "kind" matching one of the known RoT kinds + + if a.id.name != *board { + return false; + } + + // info!(&log, "DEBUG: active slot {}", active_slot); + match active_slot { + RotSlot::A => { + let slot_a_artifacts = [ + ArtifactKind::GIMLET_ROT_IMAGE_A, + ArtifactKind::PSC_ROT_IMAGE_A, + ArtifactKind::SWITCH_ROT_IMAGE_A, + ]; + // info!(&log, "DEBUG: match slot a artifact kind {}", a.id.kind); + + if slot_a_artifacts.contains(&a.id.kind) { + return true; + } + } + RotSlot::B => { + let slot_b_artifacts = [ + ArtifactKind::GIMLET_ROT_IMAGE_B, + ArtifactKind::PSC_ROT_IMAGE_B, + ArtifactKind::SWITCH_ROT_IMAGE_B, + ]; + + // info!(&log, "DEBUG: match slot b artifact kind {}", a.id.kind); + if slot_b_artifacts.contains(&a.id.kind) { + return true; + } + } + } + + // info!(&log, "DEBUG: no match artifact kind {}", a.id.kind); + false + }) + .collect(); + if matching_artifacts.is_empty() { + warn!( + log, + "cannot configure RoT update for board (no matching artifact)"; + baseboard_id, + ); + return None; + } + + if matching_artifacts.len() > 1 { + // This should be impossible unless we shipped a TUF repo with more + // than 1 artifact for the same board and slot. But it doesn't prevent + // us from picking one and proceeding. Make a note and proceed. + warn!(log, "found more than one matching artifact for RoT update"); + } + + let artifact = matching_artifacts[0]; + + // If the artifact's version matches what's deployed, then no update is + // needed. + if artifact.id.version == expected_active_version { + debug!(log, "no RoT update needed for board"; baseboard_id); + return None; + } + + let expected_active_slot = ExpectedActiveRotSlot { + slot: active_slot, + version: expected_active_version, + }; + + // Begin configuring an update. + let inactive_caboose = match active_slot.toggled() { + RotSlot::A => CabooseWhich::RotSlotA, + RotSlot::B => CabooseWhich::RotSlotB, + }; + + let expected_inactive_version = match inventory + .caboose_for(inactive_caboose, baseboard_id) + .map(|c| c.caboose.version.parse::()) + .transpose() + { + Ok(None) => ExpectedVersion::NoValidVersion, + Ok(Some(v)) => ExpectedVersion::Version(v), + Err(_) => { + warn!( + log, + "cannot configure RoT update for board \ + (found inactive slot contents but version was not valid)"; + baseboard_id + ); + return None; + } + }; + + Some(PendingMgsUpdate { + baseboard_id: baseboard_id.clone(), + sp_type: sp_info.sp_type, + slot_id: u32::from(sp_info.sp_slot), + details: PendingMgsUpdateDetails::Rot { + expected_active_slot, + expected_inactive_version, + expected_persistent_boot_preference: rot_state + .persistent_boot_preference, + expected_pending_persistent_boot_preference: rot_state + .pending_persistent_boot_preference, + expected_transient_boot_preference: rot_state + .transient_boot_preference, + }, + artifact_hash: artifact.hash, + artifact_version: artifact.id.version.clone(), + }) +} + #[cfg(test)] mod test { use crate::mgs_updates::plan_mgs_updates; diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index dcf100db8ff..4054e69bc5f 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -33,6 +33,7 @@ use nexus_types::deployment::ZpoolFilter; use nexus_types::external_api::views::PhysicalDiskPolicy; use nexus_types::external_api::views::SledPolicy; use nexus_types::external_api::views::SledState; +use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_uuid_kinds::PhysicalDiskUuid; @@ -951,12 +952,22 @@ impl<'a> Planner<'a> { // For better or worse, switches and PSCs do not have the same idea of // being adopted into the control plane. If they're present, they're // part of the system, and we will update them. - let included_sled_baseboards: BTreeSet<_> = self + let mut included_sled_baseboards: BTreeSet<_> = self .input .all_sleds(SledFilter::SpsUpdatedByReconfigurator) .map(|(_sled_id, details)| &details.baseboard_id) .collect(); - let included_baseboards = + + // TODO-K: I don't think I need this + let mut included_rot_sled_baseboards: BTreeSet<_> = self + .input + .all_sleds(SledFilter::RotsUpdatedByReconfigurator) + .map(|(_sled_id, details)| &details.baseboard_id) + .collect(); + + included_sled_baseboards.append(&mut included_rot_sled_baseboards); + + let mut included_baseboards: BTreeSet> = self.inventory .sps .iter() @@ -971,6 +982,16 @@ impl<'a> Planner<'a> { }) .collect(); + // TODO-K: I don't think I need this + let mut included_rot_baseboards = self + .inventory + .rots + .keys() + .map(|baseboard_id| baseboard_id.clone()) + .collect(); + + included_baseboards.append(&mut included_rot_baseboards); + // Compute the new set of PendingMgsUpdates. let current_updates = &self.blueprint.parent_blueprint().pending_mgs_updates; diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index eb413ddd7e1..6afd5728db5 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -608,6 +608,42 @@ impl SystemDescription { ) .context("recording SP inactive caboose")?; } + + if let Some(slot_a) = &s.rot_slot_a_caboose() { + builder + .found_caboose( + &baseboard_id, + CabooseWhich::RotSlotA, + "fake MGS 1", + SpComponentCaboose { + board: slot_a.board.clone(), + epoch: None, + git_commit: slot_a.git_commit.clone(), + name: slot_a.name.clone(), + sign: slot_a.sign.clone(), + version: slot_a.version.clone(), + }, + ) + .context("recording RoT slot a caboose")?; + } + + if let Some(slot_b) = &s.rot_slot_b_caboose() { + builder + .found_caboose( + &baseboard_id, + CabooseWhich::RotSlotB, + "fake MGS 1", + SpComponentCaboose { + board: slot_b.board.clone(), + epoch: None, + git_commit: slot_b.git_commit.clone(), + name: slot_b.name.clone(), + sign: slot_b.sign.clone(), + version: slot_b.version.clone(), + }, + ) + .context("recording RoT slot b caboose")?; + } } builder @@ -945,7 +981,7 @@ impl Sled { )), sp_inactive_caboose: None, rot_slot_a_caboose: Some(Arc::new( - Self::default_sp_component_caboose(String::from("0.0.2")), + Self::default_rot_component_caboose(String::from("0.0.2")), )), rot_slot_b_caboose: None, } @@ -1209,7 +1245,7 @@ impl Sled { } new @ None => { *new = Some(Arc::new( - Self::default_sp_component_caboose( + Self::default_rot_component_caboose( v.to_string(), ), )); @@ -1231,7 +1267,7 @@ impl Sled { } new @ None => { *new = Some(Arc::new( - Self::default_sp_component_caboose( + Self::default_rot_component_caboose( v.to_string(), ), )); @@ -1252,6 +1288,17 @@ impl Sled { sign: None, } } + + fn default_rot_component_caboose(version: String) -> Caboose { + let board = sp_sim::SIM_ROT_BOARD.to_string(); + Caboose { + board: board.clone(), + git_commit: String::from("unknown"), + name: board, + version: version.to_string(), + sign: None, + } + } } #[derive(Clone, Copy, Debug)] diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index 51ff668b0d2..681587c59cd 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -728,6 +728,9 @@ pub enum SledFilter { /// Sleds whose SPs should be updated by Reconfigurator SpsUpdatedByReconfigurator, + + /// Sleds whose RoTs should be updated by Reconfigurator + RotsUpdatedByReconfigurator, } impl SledFilter { @@ -785,6 +788,7 @@ impl SledPolicy { SledFilter::VpcFirewall => true, SledFilter::TufArtifactReplication => true, SledFilter::SpsUpdatedByReconfigurator => true, + SledFilter::RotsUpdatedByReconfigurator => true, }, SledPolicy::InService { provision_policy: SledProvisionPolicy::NonProvisionable, @@ -800,6 +804,7 @@ impl SledPolicy { SledFilter::VpcFirewall => true, SledFilter::TufArtifactReplication => true, SledFilter::SpsUpdatedByReconfigurator => true, + SledFilter::RotsUpdatedByReconfigurator => true, }, SledPolicy::Expunged => match filter { SledFilter::All => true, @@ -813,6 +818,7 @@ impl SledPolicy { SledFilter::VpcFirewall => false, SledFilter::TufArtifactReplication => false, SledFilter::SpsUpdatedByReconfigurator => false, + SledFilter::RotsUpdatedByReconfigurator => false, }, } } @@ -848,6 +854,7 @@ impl SledState { SledFilter::VpcFirewall => true, SledFilter::TufArtifactReplication => true, SledFilter::SpsUpdatedByReconfigurator => true, + SledFilter::RotsUpdatedByReconfigurator => true, }, SledState::Decommissioned => match filter { SledFilter::All => true, @@ -861,6 +868,7 @@ impl SledState { SledFilter::VpcFirewall => false, SledFilter::TufArtifactReplication => false, SledFilter::SpsUpdatedByReconfigurator => false, + SledFilter::RotsUpdatedByReconfigurator => false, }, } } diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index f8699b216cb..eaf7a00548d 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -366,6 +366,19 @@ pub enum CabooseWhich { Stage0Next, } +impl CabooseWhich { + pub fn toggled_slot(&self) -> Self { + match self { + CabooseWhich::RotSlotA => CabooseWhich::RotSlotB, + CabooseWhich::RotSlotB => CabooseWhich::RotSlotA, + CabooseWhich::SpSlot0 => CabooseWhich::SpSlot1, + CabooseWhich::SpSlot1 => CabooseWhich::SpSlot0, + CabooseWhich::Stage0 => CabooseWhich::Stage0Next, + CabooseWhich::Stage0Next => CabooseWhich::Stage0, + } + } +} + /// Root of trust page contents found during a collection /// /// These are normalized in the database. Each distinct `RotPage` is assigned a diff --git a/update-common/manifests/fake.toml b/update-common/manifests/fake.toml index 705abf226ee..b0027227fda 100644 --- a/update-common/manifests/fake.toml +++ b/update-common/manifests/fake.toml @@ -10,7 +10,7 @@ version = "1.0.0" source = { kind = "fake", size = "1MiB" } [[artifact.gimlet_rot]] -name = "fake-gimlet-rot" +name = "SimRot" version = "1.0.0" [artifact.gimlet_rot.source] kind = "composite-rot" From 914d4d5f79b0203de782c100e82e1e25b3276b28 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 25 Jun 2025 21:06:55 +1200 Subject: [PATCH 04/42] set update hierarchy and clean up --- dev-tools/reconfigurator-cli/src/lib.rs | 5 +- .../tests/input/target-release.txt | 47 +- .../tests/output/target-release-stdout | 661 ++++++++++++++++-- .../planning/src/mgs_updates/mod.rs | 5 +- nexus/reconfigurator/planning/src/planner.rs | 23 +- nexus/types/src/deployment/planning_input.rs | 8 - 6 files changed, 657 insertions(+), 92 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index b978e30427e..f3687ad353b 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -394,7 +394,7 @@ struct SledUpdateSpArgs { inactive: Option, } -// TODO-K: Double check which of these need to be optional +// TODO-K: Double check which of these need to exist #[derive(Debug, Args)] struct SledUpdateRotArgs { /// id of the sled @@ -1095,7 +1095,8 @@ fn cmd_sled_update_rot( ); Ok(Some(format!( - // TODO-K: Is "RoT settings" what I want here? + // TODO-K: Is "RoT settings" what I want here? or "versions" + // if that's the only thing we need "set sled {} RoT settings: {}", args.sled_id, labels.join(", ") diff --git a/dev-tools/reconfigurator-cli/tests/input/target-release.txt b/dev-tools/reconfigurator-cli/tests/input/target-release.txt index e9080d4f38e..d1d8352a91b 100644 --- a/dev-tools/reconfigurator-cli/tests/input/target-release.txt +++ b/dev-tools/reconfigurator-cli/tests/input/target-release.txt @@ -39,36 +39,55 @@ blueprint-diff 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 58d5e830-0884-47d8-a7cd-b2b3 # Now, update the simulated RoT to reflect that the update completed. # Collect inventory from it and use that collection for another planning step. # This should report that the update completed, remove that update, and add one -# for another sled. -# sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +# for an SP on the same sled. sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a inventory-generate blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 +# After the RoT update has completed, we update the simulated SP to reflect that +# update has completed as well. +# Like before, collect inventory from it and use that collection for the next step. +# This should report that the update completed, remove that update, and add one +# for another sled. +sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +inventory-generate +blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 +blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 + # This time, make it more interesting. Change the inactive slot contents of # the simulated RoT. This should make the configured update impossible and cause # the planner to fix it. # sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 --active-slot a --persistent-boot-preference a inventory-generate -blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 -blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 +blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 +blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba # Now simulate the update completing successfully. -# Another planning step should try to update the last sled. -# sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +# Like before, we should see a pending SP update for this sled. sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 --active-slot a --persistent-boot-preference a inventory-generate -blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 -blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 +blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac -# Finish updating the last sled and do one more planning run. -# Now we should see a pending SP update. (TODO: Instead we see there's nothing left to do :( ) -# sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 +# Let's simulate the successful SP update as well. +# Another couple of planning steps should try to update the last sled. +sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +inventory-generate +blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 +blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 + +# Update the RoT on the last sled. +# There should be one last pending SP update. sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a inventory-generate -blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 -blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac +blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 +blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 -# TODO-K: Do an SP update as well \ No newline at end of file +# Finish updating the last sled and do one more planning run. +# Now we should see there's nothing left to do! +sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 +inventory-generate +blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 +blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b \ No newline at end of file diff --git a/dev-tools/reconfigurator-cli/tests/output/target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/target-release-stdout index adf561a5163..9d6da47263a 100644 --- a/dev-tools/reconfigurator-cli/tests/output/target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/target-release-stdout @@ -515,8 +515,7 @@ external DNS: > # Now, update the simulated RoT to reflect that the update completed. > # Collect inventory from it and use that collection for another planning step. > # This should report that the update completed, remove that update, and add one -> # for another sled. -> # sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +> # for an SP on the same sled. > sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A @@ -535,8 +534,7 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO skipping board for SP update, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 @@ -545,6 +543,201 @@ generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent bluepri from: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + PENDING MGS UPDATES: + + Pending MGS-managed updates (all baseboards): + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +* sled 0 model0 serial0 - 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + └─ + 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) + AAAA fd00:1122:3344:101::27 + name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) + AAAA fd00:1122:3344:101::22 + name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) + AAAA fd00:1122:3344:102::1 + name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) + AAAA fd00:1122:3344:102::23 + name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) + AAAA fd00:1122:3344:103::22 + name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) + AAAA fd00:1122:3344:2::1 + name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) + AAAA fd00:1122:3344:102::22 + name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) + AAAA fd00:1122:3344:101::25 + name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) + AAAA fd00:1122:3344:102::21 + name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) + AAAA fd00:1122:3344:101::21 + name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) + AAAA fd00:1122:3344:103::26 + name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) + AAAA fd00:1122:3344:102::24 + name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) + AAAA fd00:1122:3344:103::24 + name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) + AAAA fd00:1122:3344:103::27 + name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) + AAAA fd00:1122:3344:101::23 + name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) + AAAA fd00:1122:3344:102::28 + name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) + AAAA fd00:1122:3344:101::1 + name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) + AAAA fd00:1122:3344:1::1 + name: @ (records: 3) + NS ns1.control-plane.oxide.internal + NS ns2.control-plane.oxide.internal + NS ns3.control-plane.oxide.internal + name: _clickhouse-admin-single-server._tcp (records: 1) + SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse-native._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse._tcp (records: 1) + SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _crucible-pantry._tcp (records: 3) + SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal + SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal + SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal + name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) + SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal + name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) + SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal + name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) + SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal + name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) + SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal + name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) + SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal + name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) + SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal + name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) + SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal + name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) + SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal + name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) + SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal + name: _external-dns._tcp (records: 3) + SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal + SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal + SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal + name: _internal-ntp._tcp (records: 3) + SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal + SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal + SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal + name: _nameservice._tcp (records: 3) + SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal + SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal + SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal + name: _nexus._tcp (records: 3) + SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal + SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal + SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal + name: _oximeter-reader._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _repo-depot._tcp (records: 3) + SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal + SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal + SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal + name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) + AAAA fd00:1122:3344:102::25 + name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) + AAAA fd00:1122:3344:101::24 + name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) + AAAA fd00:1122:3344:102::26 + name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) + AAAA fd00:1122:3344:103::1 + name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) + AAAA fd00:1122:3344:101::26 + name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) + AAAA fd00:1122:3344:102::27 + name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) + AAAA fd00:1122:3344:3::1 + name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) + AAAA fd00:1122:3344:103::21 + name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) + AAAA fd00:1122:3344:103::25 + name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) + AAAA fd00:1122:3344:103::23 + name: ns1 (records: 1) + AAAA fd00:1122:3344:1::1 + name: ns2 (records: 1) + AAAA fd00:1122:3344:2::1 + name: ns3 (records: 1) + AAAA fd00:1122:3344:3::1 + +external DNS: + DNS zone: "oxide.example" (unchanged) + name: @ (records: 3) + NS ns1.oxide.example + NS ns2.oxide.example + NS ns3.oxide.example + name: example-silo.sys (records: 3) + A 192.0.2.2 + A 192.0.2.3 + A 192.0.2.4 + name: ns1 (records: 1) + A 198.51.100.1 + name: ns2 (records: 1) + A 198.51.100.2 + name: ns3 (records: 1) + A 198.51.100.3 + + + + +> # After the RoT update has completed, we update the simulated SP to reflect that +> # update has completed as well. +> # Like before, collect inventory from it and use that collection for the next step. +> # This should report that the update completed, remove that update, and add one +> # for another sled. +> sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 + +> inventory-generate +generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds + +> blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 +INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 +INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 +INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO skipping board for SP update, serial_number: serial0, part_number: model0 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO reached maximum number of pending SP updates, max: 1 +INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify +generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 + +> blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 +from: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 +to: blueprint df06bb57-ad42-4431-9206-abff322896c7 + COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) cluster.preserve_downgrade_option: (do not modify) (unchanged) @@ -564,7 +757,7 @@ to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- sp_type slot part_number serial_number artifact_hash artifact_version details ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 0 model0 serial0 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } +- sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } @@ -715,9 +908,9 @@ external DNS: set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: active slot -> A, slot b -> 0.5.0, persistent boot preference -> A > inventory-generate -generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds +generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds -> blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 +> blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -732,11 +925,11 @@ INFO SP update impossible (will remove it and re-evaluate board), artifact_versi INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 +generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 -> blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 -from: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 -to: blueprint df06bb57-ad42-4431-9206-abff322896c7 +> blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +from: blueprint df06bb57-ad42-4431-9206-abff322896c7 +to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -901,15 +1094,14 @@ external DNS: > # Now simulate the update completing successfully. -> # Another planning step should try to update the last sled. -> # sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +> # Like before, we should see a pending SP update for this sled. > sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 --active-slot a --persistent-boot-preference a set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A > inventory-generate -generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds +generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds -> blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 +> blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -921,16 +1113,14 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO skipping board for SP update, serial_number: serial1, part_number: model1 -INFO skipping board for SP update, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO ran out of boards for SP update +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 +generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -> blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -from: blueprint df06bb57-ad42-4431-9206-abff322896c7 -to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +> blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac +from: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -948,11 +1138,11 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - sp_type slot part_number serial_number artifact_hash artifact_version details - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ -- sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } -+ sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +* sled 1 model1 serial1 - 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + └─ + 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } internal DNS: @@ -1094,16 +1284,15 @@ external DNS: -> # Finish updating the last sled and do one more planning run. -> # Now we should see a pending SP update. (TODO: Instead we see there's nothing left to do :( ) -> # sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 -> sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A +> # Let's simulate the successful SP update as well. +> # Another couple of planning steps should try to update the last sled. +> sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: active -> 1.0.0 > inventory-generate -generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds +generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds -> blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 +> blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1114,18 +1303,17 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO skipping board for SP update, serial_number: serial2, part_number: model2 -INFO skipping board for SP update, serial_number: serial0, part_number: model0 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO skipping board for SP update, serial_number: serial1, part_number: model1 +INFO skipping board for SP update, serial_number: serial0, part_number: model0 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO ran out of boards for SP update -INFO all zones up-to-date INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +generated blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 based on parent blueprint 9034c710-3e57-45f3-99e5-4316145e87ac -> blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac -from: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac +> blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 +from: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac +to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -1146,7 +1334,8 @@ to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- sp_type slot part_number serial_number artifact_hash artifact_version details ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } +- sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } ++ sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -1288,4 +1477,386 @@ external DNS: -> # TODO: Do an SP update as well +> # Update the RoT on the last sled. +> # There should be one last pending SP update. +> sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A + +> inventory-generate +generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds + +> blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 +INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 +INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 +INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO reached maximum number of pending SP updates, max: 1 +INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify +generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 + +> blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +from: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 +to: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + PENDING MGS UPDATES: + + Pending MGS-managed updates (all baseboards): + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +* sled 2 model2 serial2 - 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + └─ + 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) + AAAA fd00:1122:3344:101::27 + name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) + AAAA fd00:1122:3344:101::22 + name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) + AAAA fd00:1122:3344:102::1 + name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) + AAAA fd00:1122:3344:102::23 + name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) + AAAA fd00:1122:3344:103::22 + name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) + AAAA fd00:1122:3344:2::1 + name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) + AAAA fd00:1122:3344:102::22 + name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) + AAAA fd00:1122:3344:101::25 + name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) + AAAA fd00:1122:3344:102::21 + name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) + AAAA fd00:1122:3344:101::21 + name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) + AAAA fd00:1122:3344:103::26 + name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) + AAAA fd00:1122:3344:102::24 + name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) + AAAA fd00:1122:3344:103::24 + name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) + AAAA fd00:1122:3344:103::27 + name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) + AAAA fd00:1122:3344:101::23 + name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) + AAAA fd00:1122:3344:102::28 + name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) + AAAA fd00:1122:3344:101::1 + name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) + AAAA fd00:1122:3344:1::1 + name: @ (records: 3) + NS ns1.control-plane.oxide.internal + NS ns2.control-plane.oxide.internal + NS ns3.control-plane.oxide.internal + name: _clickhouse-admin-single-server._tcp (records: 1) + SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse-native._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse._tcp (records: 1) + SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _crucible-pantry._tcp (records: 3) + SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal + SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal + SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal + name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) + SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal + name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) + SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal + name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) + SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal + name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) + SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal + name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) + SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal + name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) + SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal + name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) + SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal + name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) + SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal + name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) + SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal + name: _external-dns._tcp (records: 3) + SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal + SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal + SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal + name: _internal-ntp._tcp (records: 3) + SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal + SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal + SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal + name: _nameservice._tcp (records: 3) + SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal + SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal + SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal + name: _nexus._tcp (records: 3) + SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal + SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal + SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal + name: _oximeter-reader._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _repo-depot._tcp (records: 3) + SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal + SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal + SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal + name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) + AAAA fd00:1122:3344:102::25 + name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) + AAAA fd00:1122:3344:101::24 + name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) + AAAA fd00:1122:3344:102::26 + name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) + AAAA fd00:1122:3344:103::1 + name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) + AAAA fd00:1122:3344:101::26 + name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) + AAAA fd00:1122:3344:102::27 + name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) + AAAA fd00:1122:3344:3::1 + name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) + AAAA fd00:1122:3344:103::21 + name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) + AAAA fd00:1122:3344:103::25 + name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) + AAAA fd00:1122:3344:103::23 + name: ns1 (records: 1) + AAAA fd00:1122:3344:1::1 + name: ns2 (records: 1) + AAAA fd00:1122:3344:2::1 + name: ns3 (records: 1) + AAAA fd00:1122:3344:3::1 + +external DNS: + DNS zone: "oxide.example" (unchanged) + name: @ (records: 3) + NS ns1.oxide.example + NS ns2.oxide.example + NS ns3.oxide.example + name: example-silo.sys (records: 3) + A 192.0.2.2 + A 192.0.2.3 + A 192.0.2.4 + name: ns1 (records: 1) + A 198.51.100.1 + name: ns2 (records: 1) + A 198.51.100.2 + name: ns3 (records: 1) + A 198.51.100.3 + + + + +> # Finish updating the last sled and do one more planning run. +> # Now we should see there's nothing left to do! +> sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 +set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 SP versions: active -> 1.0.0 + +> inventory-generate +generated inventory collection 39363465-89ae-4ac2-9be1-099068da9d45 from configured sleds + +> blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 +INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 +INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 +INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO skipping board for SP update, serial_number: serial2, part_number: model2 +INFO skipping board for SP update, serial_number: serial0, part_number: model0 +INFO skipping board for SP update, serial_number: serial1, part_number: model1 +INFO ran out of boards for SP update +INFO all zones up-to-date +INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify +generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 + +> blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b +from: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +to: blueprint 626487fa-7139-45ec-8416-902271fc730b + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + PENDING MGS UPDATES: + + Pending MGS-managed updates (all baseboards): + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 2 model2 serial2 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) + AAAA fd00:1122:3344:101::27 + name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) + AAAA fd00:1122:3344:101::22 + name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) + AAAA fd00:1122:3344:102::1 + name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) + AAAA fd00:1122:3344:102::23 + name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) + AAAA fd00:1122:3344:103::22 + name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) + AAAA fd00:1122:3344:2::1 + name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) + AAAA fd00:1122:3344:102::22 + name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) + AAAA fd00:1122:3344:101::25 + name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) + AAAA fd00:1122:3344:102::21 + name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) + AAAA fd00:1122:3344:101::21 + name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) + AAAA fd00:1122:3344:103::26 + name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) + AAAA fd00:1122:3344:102::24 + name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) + AAAA fd00:1122:3344:103::24 + name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) + AAAA fd00:1122:3344:103::27 + name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) + AAAA fd00:1122:3344:101::23 + name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) + AAAA fd00:1122:3344:102::28 + name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) + AAAA fd00:1122:3344:101::1 + name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) + AAAA fd00:1122:3344:1::1 + name: @ (records: 3) + NS ns1.control-plane.oxide.internal + NS ns2.control-plane.oxide.internal + NS ns3.control-plane.oxide.internal + name: _clickhouse-admin-single-server._tcp (records: 1) + SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse-native._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse._tcp (records: 1) + SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _crucible-pantry._tcp (records: 3) + SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal + SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal + SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal + name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) + SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal + name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) + SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal + name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) + SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal + name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) + SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal + name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) + SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal + name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) + SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal + name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) + SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal + name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) + SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal + name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) + SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal + name: _external-dns._tcp (records: 3) + SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal + SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal + SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal + name: _internal-ntp._tcp (records: 3) + SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal + SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal + SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal + name: _nameservice._tcp (records: 3) + SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal + SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal + SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal + name: _nexus._tcp (records: 3) + SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal + SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal + SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal + name: _oximeter-reader._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _repo-depot._tcp (records: 3) + SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal + SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal + SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal + name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) + AAAA fd00:1122:3344:102::25 + name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) + AAAA fd00:1122:3344:101::24 + name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) + AAAA fd00:1122:3344:102::26 + name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) + AAAA fd00:1122:3344:103::1 + name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) + AAAA fd00:1122:3344:101::26 + name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) + AAAA fd00:1122:3344:102::27 + name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) + AAAA fd00:1122:3344:3::1 + name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) + AAAA fd00:1122:3344:103::21 + name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) + AAAA fd00:1122:3344:103::25 + name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) + AAAA fd00:1122:3344:103::23 + name: ns1 (records: 1) + AAAA fd00:1122:3344:1::1 + name: ns2 (records: 1) + AAAA fd00:1122:3344:2::1 + name: ns3 (records: 1) + AAAA fd00:1122:3344:3::1 + +external DNS: + DNS zone: "oxide.example" (unchanged) + name: @ (records: 3) + NS ns1.oxide.example + NS ns2.oxide.example + NS ns3.oxide.example + name: example-silo.sys (records: 3) + A 192.0.2.2 + A 192.0.2.3 + A 192.0.2.4 + name: ns1 (records: 1) + A 198.51.100.1 + name: ns2 (records: 1) + A 198.51.100.2 + name: ns3 (records: 1) + A 198.51.100.3 + + + diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 0a0b70cc047..31a841bccfa 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -391,9 +391,10 @@ fn try_make_update( // let cabooses = &inventory.cabooses_found; // info!(&log, "DEBUG: Components"; "baseboard-id" => ?baseboard_id, "cabooses" => ?cabooses, "artifacts" => ?current_artifacts); - try_make_update_sp(log, baseboard_id, inventory, current_artifacts); - // TODO-K: RoT must be before SP try_make_update_rot(log, baseboard_id, inventory, current_artifacts) + .or_else(|| { + try_make_update_sp(log, baseboard_id, inventory, current_artifacts) + }) } /// Determine if the given baseboard needs an SP update and, if so, returns it. diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 4054e69bc5f..aa150aef69e 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -952,22 +952,13 @@ impl<'a> Planner<'a> { // For better or worse, switches and PSCs do not have the same idea of // being adopted into the control plane. If they're present, they're // part of the system, and we will update them. - let mut included_sled_baseboards: BTreeSet<_> = self + let included_sled_baseboards: BTreeSet<_> = self .input .all_sleds(SledFilter::SpsUpdatedByReconfigurator) .map(|(_sled_id, details)| &details.baseboard_id) .collect(); - // TODO-K: I don't think I need this - let mut included_rot_sled_baseboards: BTreeSet<_> = self - .input - .all_sleds(SledFilter::RotsUpdatedByReconfigurator) - .map(|(_sled_id, details)| &details.baseboard_id) - .collect(); - - included_sled_baseboards.append(&mut included_rot_sled_baseboards); - - let mut included_baseboards: BTreeSet> = + let included_baseboards: BTreeSet> = self.inventory .sps .iter() @@ -982,16 +973,6 @@ impl<'a> Planner<'a> { }) .collect(); - // TODO-K: I don't think I need this - let mut included_rot_baseboards = self - .inventory - .rots - .keys() - .map(|baseboard_id| baseboard_id.clone()) - .collect(); - - included_baseboards.append(&mut included_rot_baseboards); - // Compute the new set of PendingMgsUpdates. let current_updates = &self.blueprint.parent_blueprint().pending_mgs_updates; diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index 681587c59cd..51ff668b0d2 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -728,9 +728,6 @@ pub enum SledFilter { /// Sleds whose SPs should be updated by Reconfigurator SpsUpdatedByReconfigurator, - - /// Sleds whose RoTs should be updated by Reconfigurator - RotsUpdatedByReconfigurator, } impl SledFilter { @@ -788,7 +785,6 @@ impl SledPolicy { SledFilter::VpcFirewall => true, SledFilter::TufArtifactReplication => true, SledFilter::SpsUpdatedByReconfigurator => true, - SledFilter::RotsUpdatedByReconfigurator => true, }, SledPolicy::InService { provision_policy: SledProvisionPolicy::NonProvisionable, @@ -804,7 +800,6 @@ impl SledPolicy { SledFilter::VpcFirewall => true, SledFilter::TufArtifactReplication => true, SledFilter::SpsUpdatedByReconfigurator => true, - SledFilter::RotsUpdatedByReconfigurator => true, }, SledPolicy::Expunged => match filter { SledFilter::All => true, @@ -818,7 +813,6 @@ impl SledPolicy { SledFilter::VpcFirewall => false, SledFilter::TufArtifactReplication => false, SledFilter::SpsUpdatedByReconfigurator => false, - SledFilter::RotsUpdatedByReconfigurator => false, }, } } @@ -854,7 +848,6 @@ impl SledState { SledFilter::VpcFirewall => true, SledFilter::TufArtifactReplication => true, SledFilter::SpsUpdatedByReconfigurator => true, - SledFilter::RotsUpdatedByReconfigurator => true, }, SledState::Decommissioned => match filter { SledFilter::All => true, @@ -868,7 +861,6 @@ impl SledState { SledFilter::VpcFirewall => false, SledFilter::TufArtifactReplication => false, SledFilter::SpsUpdatedByReconfigurator => false, - SledFilter::RotsUpdatedByReconfigurator => false, }, } } From 3c7cc8b1d4a7e7ba7e9c7c62030eb2431f90f8d5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 25 Jun 2025 21:25:02 +1200 Subject: [PATCH 05/42] Remove unnecessary todo --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 31a841bccfa..bbcddc47595 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -128,7 +128,6 @@ pub fn plan_mgs_updates( return rv; } - // TODO-K: Add RoT update here match try_make_update(log, board, inventory, current_artifacts) { Some(update) => { // TODO-K: should this somehow change to say RoT or bootloader or whatever? From d9ca092f5f1477a03e7a9d03b682adcf403102e8 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 26 Jun 2025 19:11:33 +1200 Subject: [PATCH 06/42] Add additional checks --- dev-tools/reconfigurator-cli/src/lib.rs | 1 - .../planning/src/mgs_updates/mod.rs | 177 ++++++++++++++++-- 2 files changed, 157 insertions(+), 21 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index b1959d3d7da..e67d724a3b5 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -394,7 +394,6 @@ struct SledUpdateSpArgs { inactive: Option, } -// TODO-K: Double check which of these need to exist #[derive(Debug, Args)] struct SledUpdateRotArgs { /// id of the sled diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index bbcddc47595..11cad241293 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use thiserror::Error; use tufaceous_artifact::ArtifactKind; use tufaceous_artifact::ArtifactVersion; +use tufaceous_artifact::ArtifactVersionError; use tufaceous_artifact::KnownArtifactKind; /// Generates a new set of `PendingMgsUpdates` based on: @@ -171,8 +172,12 @@ enum MgsUpdateStatusError { MissingSpInfo, #[error("no caboose found for active slot in inventory")] MissingActiveCaboose, + #[error("no RoT state found in inventory")] + MissingRotState, #[error("not yet implemented")] NotYetImplemented, + #[error("unable to parse input into ArtifactVersion: {0:?}")] + FailedArtifactVersionParse(ArtifactVersionError), } /// Determine the status of a single MGS update based on what's in inventory for @@ -206,7 +211,6 @@ fn mgs_update_status( .map(|c| c.caboose.version.as_ref()); Ok(mgs_update_status_sp( - log, desired_version, expected_active_version, expected_inactive_version, @@ -217,16 +221,13 @@ fn mgs_update_status( PendingMgsUpdateDetails::Rot { expected_active_slot, expected_inactive_version, - .. - // TODO-K: incorporate these - // expected_persistent_boot_preference, - // expected_pending_persistent_boot_preference, - // expected_transient_boot_preference, + expected_persistent_boot_preference, + expected_pending_persistent_boot_preference, + expected_transient_boot_preference, } => { - let active_caboose_which = match &expected_active_slot.slot { RotSlot::A => CabooseWhich::RotSlotA, - RotSlot::B => CabooseWhich::RotSlotB + RotSlot::B => CabooseWhich::RotSlotB, }; let Some(active_caboose) = @@ -239,14 +240,44 @@ fn mgs_update_status( .caboose_for(active_caboose_which.toggled_slot(), baseboard_id) .map(|c| c.caboose.version.as_ref()); + let rot_state = inventory + .rots + .get(baseboard_id) + .ok_or(MgsUpdateStatusError::MissingRotState)?; + + let found_active_version = + ArtifactVersion::new(active_caboose.caboose.version.clone()) + .map_err(|e| { + MgsUpdateStatusError::FailedArtifactVersionParse(e) + })?; + + let found_active_slot = ExpectedActiveRotSlot { + slot: rot_state.active_slot, + version: found_active_version, + }; + // TODO-K: Make an mgs_update_status_rot? - Ok(mgs_update_status_sp( - log, + //Ok(mgs_update_status_sp( + // log, + // desired_version, + // &expected_active_slot.version, + // expected_inactive_version, + // &active_caboose.caboose.version, + // found_inactive_version, + //)) + + Ok(mgs_update_status_rot( desired_version, - &expected_active_slot.version, + &expected_active_slot, expected_inactive_version, - &active_caboose.caboose.version, + expected_persistent_boot_preference, + expected_pending_persistent_boot_preference, + expected_transient_boot_preference, + &found_active_slot, found_inactive_version, + &rot_state.persistent_boot_preference, + &rot_state.pending_persistent_boot_preference, + &rot_state.transient_boot_preference, )) } PendingMgsUpdateDetails::RotBootloader { .. } => { @@ -297,8 +328,6 @@ fn mgs_update_status( /// Compares a configured SP update with information from inventory and /// determines the current status of the update. See `MgsUpdateStatus`. fn mgs_update_status_sp( - // TODO-K: remove logger - _log: &slog::Logger, desired_version: &ArtifactVersion, expected_active_version: &ArtifactVersion, expected_inactive_version: &ExpectedVersion, @@ -330,12 +359,120 @@ fn mgs_update_status_sp( match (found_inactive_version, expected_inactive_version) { (Some(_), ExpectedVersion::NoValidVersion) => { // We expected nothing in the inactive slot, but found something. - // info!( - // &log, - // "DEBUG: found {:?} != expected {}", - // found_inactive_version, - // expected_inactive_version - // ); + MgsUpdateStatus::Impossible + } + (Some(found), ExpectedVersion::Version(expected)) => { + if found == expected.as_str() { + // We found something in the inactive slot that matches what we + // expected. + MgsUpdateStatus::NotDone + } else { + // We found something in the inactive slot that differs from + // what we expected. + MgsUpdateStatus::Impossible + } + } + (None, ExpectedVersion::Version(_)) => { + // We expected something in the inactive slot, but found nothing. + // This case is tricky because we can't tell from the inventory + // whether we transiently failed to fetch the caboose for some + // reason or whether the caboose is actually garbage. We choose to + // assume that it's actually garbage, which would mean that this + // update as-configured is impossible. This will cause us to + // generate a new update that expects garbage in the inactive slot. + // If we're right, great. If we're wrong, then *that* update will + // be impossible to complete, but we should fix this again if the + // transient error goes away. + // + // If we instead assumed that this was a transient error, we'd do + // nothing here instead. But if the caboose was really missing, + // then we'd get stuck forever waiting for something that would + // never happen. + MgsUpdateStatus::Impossible + } + (None, ExpectedVersion::NoValidVersion) => { + // We expected nothing in the inactive slot and found nothing there. + // No problem! + MgsUpdateStatus::NotDone + } + } +} + +#[allow(clippy::too_many_arguments)] +fn mgs_update_status_rot( + desired_version: &ArtifactVersion, + expected_active_slot: &ExpectedActiveRotSlot, + expected_inactive_version: &ExpectedVersion, + expected_persistent_boot_preference: &RotSlot, + expected_pending_persistent_boot_preference: &Option, + expected_transient_boot_preference: &Option, + found_active_slot: &ExpectedActiveRotSlot, + found_inactive_version: Option<&str>, + found_persistent_boot_preference: &RotSlot, + found_pending_persistent_boot_preference: &Option, + found_transient_boot_preference: &Option, +) -> MgsUpdateStatus { + if &found_active_slot.version() == desired_version { + // If we find the desired version in the active slot, we're done. + return MgsUpdateStatus::Done; + } + + // The update hasn't completed. + // + // Check to make sure the contents of the active slot, persistent boot + // preference, pending persistent boot preference, and transient boot + // preference are still what they were when we configured this update. + // If not, then this update cannot proceed as currently configured. + // It will fail its precondition check. + if found_active_slot.version() != expected_active_slot.version() { + return MgsUpdateStatus::Impossible; + } + + if found_persistent_boot_preference != expected_persistent_boot_preference { + return MgsUpdateStatus::Impossible; + } + + if found_pending_persistent_boot_preference + != expected_pending_persistent_boot_preference + { + return MgsUpdateStatus::Impossible; + } + + if found_transient_boot_preference != expected_transient_boot_preference { + return MgsUpdateStatus::Impossible; + } + + // If either found pending persistent boot preference or found transient + // boot preference are not empty, then an update is not done + if found_pending_persistent_boot_preference.is_some() + || found_transient_boot_preference.is_some() + { + return MgsUpdateStatus::NotDone; + } + + // If there is a mismatch between the found persistent boot preference + // and the found active slot then the update is not done. + // + // TODO: Alternatively, this could also mean a failed update. See + // https://github.com/oxidecomputer/omicron/issues/8414 for context + // about when we'll be able to know whether an it's an ongoing update + // or an RoT in a failed state. + if found_persistent_boot_preference != &found_active_slot.slot { + // TODO-K: I am not 100% sure on this one. It may be impossible? + return MgsUpdateStatus::NotDone; + } + + // Similarly, check the contents of the inactive slot to determine if it + // still matches what we saw when we configured this update. If not, then + // this update cannot proceed as currently configured. It will fail its + // precondition check. + // + // This logic is more complex than for the active slot because unlike the + // active slot, it's possible for both the found contents and the expected + // contents to be missing and that's not necessarily an error. + match (found_inactive_version, expected_inactive_version) { + (Some(_), ExpectedVersion::NoValidVersion) => { + // We expected nothing in the inactive slot, but found something. MgsUpdateStatus::Impossible } (Some(found), ExpectedVersion::Version(expected)) => { From 7f8904e5a77b933289aad49950d0c8f197914a21 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 26 Jun 2025 19:22:17 +1200 Subject: [PATCH 07/42] clean up --- dev-tools/reconfigurator-cli/src/lib.rs | 3 +-- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 11 ----------- nexus/reconfigurator/planning/src/system.rs | 2 -- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index e67d724a3b5..0513b663071 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -1053,6 +1053,7 @@ fn cmd_sled_update_rot( labels.push(format!("slot b -> {}", slot_b)); } // TODO-K: Do I need these settings as well? + // Maybe only to test failure cases. labels.push(format!( "persistent boot preference -> {}", &args.persistent_boot_preference @@ -1094,8 +1095,6 @@ fn cmd_sled_update_rot( ); Ok(Some(format!( - // TODO-K: Is "RoT settings" what I want here? or "versions" - // if that's the only thing we need "set sled {} RoT settings: {}", args.sled_id, labels.join(", ") diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 11cad241293..dca7145ead9 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -256,16 +256,6 @@ fn mgs_update_status( version: found_active_version, }; - // TODO-K: Make an mgs_update_status_rot? - //Ok(mgs_update_status_sp( - // log, - // desired_version, - // &expected_active_slot.version, - // expected_inactive_version, - // &active_caboose.caboose.version, - // found_inactive_version, - //)) - Ok(mgs_update_status_rot( desired_version, &expected_active_slot, @@ -631,7 +621,6 @@ fn try_make_update_sp( // If the artifact's version matches what's deployed, then no update is // needed. if artifact.id.version == expected_active_version { - // TODO-K: bails out here when trying to do an RoT update debug!(log, "no SP update needed for board"; baseboard_id); return None; } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index aa00d033be2..2423b624919 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -493,8 +493,6 @@ impl SystemDescription { })?; let sled = Arc::make_mut(sled); sled.set_rot_versions(slot_a_version, slot_b_version); - // TODO-K: Should I include all the other settings as well? - // Is there a point to that? Ok(self) } From 6bee79f7e4296f69b21b4427b18a56d3c778a4f9 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 27 Jun 2025 15:13:46 +1200 Subject: [PATCH 08/42] Remove additional arguments --- dev-tools/reconfigurator-cli/src/lib.rs | 46 ++----------------- .../tests/input/target-release.txt | 9 ++-- .../tests/output/target-release-stdout | 17 ++++--- 3 files changed, 16 insertions(+), 56 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index 0513b663071..c76502fa522 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -8,7 +8,6 @@ use anyhow::{Context, anyhow, bail}; use camino::Utf8PathBuf; use clap::ValueEnum; use clap::{Args, Parser, Subcommand}; -use gateway_types::rot::RotSlot; use iddqd::IdOrdMap; use indent_write::fmt::IndentWriter; use internal_dns_types::diff::DnsDiff; @@ -399,10 +398,6 @@ struct SledUpdateRotArgs { /// id of the sled sled_id: SledUuid, - /// whether we expect the "A" or "B" slot to be active - #[clap(long)] - active_slot: RotSlot, - /// sets the version reported for the RoT slot a #[clap(long, required_unless_present_any = &["slot_b"])] slot_a: Option, @@ -411,22 +406,10 @@ struct SledUpdateRotArgs { #[clap(long, required_unless_present_any = &["slot_a"])] slot_b: Option, - /// set the persistent boot preference written into the current - /// authoritative CFPA page (ping or pong). - /// Will default to the value of active_version when not set - #[clap(long)] - persistent_boot_preference: RotSlot, - - /// set the persistent boot preference written into the CFPA scratch - /// page that will become the persistent boot preference in the authoritative - /// CFPA page upon reboot, unless CFPA update of the authoritative page fails - /// for some reason - #[clap(long)] - pending_persistent_boot_preference: Option, - - /// override persistent preference selection for a single boot - #[clap(long)] - transient_boot_preference: Option, + // TODO: In the future we could set other fields as well. + // They would be useful to simulate failures. + // These would be: active_slot, persistent_boot_preference, + // transient_boot_preference and pending_persistent_boot_preference. } #[derive(Debug, Args)] @@ -1045,33 +1028,12 @@ fn cmd_sled_update_rot( ) -> anyhow::Result> { let mut labels = Vec::new(); - labels.push(format!("active slot -> {}", &args.active_slot)); if let Some(slot_a) = &args.slot_a { labels.push(format!("slot a -> {}", slot_a)); } if let Some(slot_b) = &args.slot_b { labels.push(format!("slot b -> {}", slot_b)); } - // TODO-K: Do I need these settings as well? - // Maybe only to test failure cases. - labels.push(format!( - "persistent boot preference -> {}", - &args.persistent_boot_preference - )); - if let Some(pending_persistent_boot_preference) = - &args.pending_persistent_boot_preference - { - labels.push(format!( - "pending persistent boot preference -> {}", - pending_persistent_boot_preference - )); - } - if let Some(transient_boot_preference) = &args.transient_boot_preference { - labels.push(format!( - "transient boot preference -> {}", - transient_boot_preference - )); - } assert!( !labels.is_empty(), diff --git a/dev-tools/reconfigurator-cli/tests/input/target-release.txt b/dev-tools/reconfigurator-cli/tests/input/target-release.txt index d1d8352a91b..4984dacc555 100644 --- a/dev-tools/reconfigurator-cli/tests/input/target-release.txt +++ b/dev-tools/reconfigurator-cli/tests/input/target-release.txt @@ -40,7 +40,7 @@ blueprint-diff 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 58d5e830-0884-47d8-a7cd-b2b3 # Collect inventory from it and use that collection for another planning step. # This should report that the update completed, remove that update, and add one # for an SP on the same sled. -sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 inventory-generate blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 @@ -58,15 +58,14 @@ blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff # This time, make it more interesting. Change the inactive slot contents of # the simulated RoT. This should make the configured update impossible and cause # the planner to fix it. -# sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 -sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 --active-slot a --persistent-boot-preference a +sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 inventory-generate blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba # Now simulate the update completing successfully. # Like before, we should see a pending SP update for this sled. -sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 inventory-generate blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac @@ -80,7 +79,7 @@ blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b107 # Update the RoT on the last sled. # There should be one last pending SP update. -sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a +sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 inventory-generate blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 diff --git a/dev-tools/reconfigurator-cli/tests/output/target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/target-release-stdout index e2e41d22f4c..f40e7c218cc 100644 --- a/dev-tools/reconfigurator-cli/tests/output/target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/target-release-stdout @@ -516,8 +516,8 @@ external DNS: > # Collect inventory from it and use that collection for another planning step. > # This should report that the update completed, remove that update, and add one > # for an SP on the same sled. -> sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A +> sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT settings: slot a -> 1.0.0 > inventory-generate generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configured sleds @@ -903,9 +903,8 @@ external DNS: > # This time, make it more interesting. Change the inactive slot contents of > # the simulated RoT. This should make the configured update impossible and cause > # the planner to fix it. -> # sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 -> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 --active-slot a --persistent-boot-preference a -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: active slot -> A, slot b -> 0.5.0, persistent boot preference -> A +> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot b -> 0.5.0 > inventory-generate generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds @@ -1095,8 +1094,8 @@ external DNS: > # Now simulate the update completing successfully. > # Like before, we should see a pending SP update for this sled. -> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 --active-slot a --persistent-boot-preference a -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A +> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot a -> 1.0.0 > inventory-generate generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds @@ -1479,8 +1478,8 @@ external DNS: > # Update the RoT on the last sled. > # There should be one last pending SP update. -> sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 --active-slot a --persistent-boot-preference a -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: active slot -> A, slot a -> 1.0.0, persistent boot preference -> A +> sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 +set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot a -> 1.0.0 > inventory-generate generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds From 6e0f4445673b358bf426476f985ef0036afabaf8 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 27 Jun 2025 15:54:47 +1200 Subject: [PATCH 09/42] Add information to sled-show command --- dev-tools/reconfigurator-cli/src/lib.rs | 25 +++++- .../tests/output/cmds-example-stdout | 6 ++ .../tests/output/cmds-stdout | 21 +++++ nexus/reconfigurator/planning/src/system.rs | 85 ++++++++++++++++--- 4 files changed, 122 insertions(+), 15 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index c76502fa522..d7df9784168 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -405,8 +405,7 @@ struct SledUpdateRotArgs { /// sets the version reported for the RoT slot b #[clap(long, required_unless_present_any = &["slot_a"])] slot_b: Option, - - // TODO: In the future we could set other fields as well. + // TODO: In a follow up PR we could set other fields as well. // They would be useful to simulate failures. // These would be: active_slot, persistent_boot_preference, // transient_boot_preference and pending_persistent_boot_preference. @@ -939,6 +938,12 @@ fn cmd_sled_show( let rot_active_slot = description.sled_rot_active_slot(sled_id)?; let rot_slot_a_version = description.sled_rot_slot_a_version(sled_id)?; let rot_slot_b_version = description.sled_rot_slot_b_version(sled_id)?; + let rot_persistent_boot_preference = + description.sled_rot_persistent_boot_preference(sled_id)?; + let rot_pending_persistent_boot_preference = + description.sled_rot_pending_persistent_boot_preference(sled_id)?; + let rot_transient_boot_preference = + description.sled_rot_transient_boot_preference(sled_id)?; let planning_input = description .to_planning_input_builder() .context("failed to generate planning_input builder")? @@ -952,9 +957,23 @@ fn cmd_sled_show( swriteln!(s, "SP active version: {:?}", sp_active_version); swriteln!(s, "SP inactive version: {:?}", sp_inactive_version); swriteln!(s, "RoT active slot: {}", rot_active_slot); - // TODO-K: Include all other RoT settings? swriteln!(s, "RoT slot A version: {:?}", rot_slot_a_version); swriteln!(s, "RoT slot B version: {:?}", rot_slot_b_version); + swriteln!( + s, + "RoT persistent boot preference: {}", + rot_persistent_boot_preference + ); + swriteln!( + s, + "RoT pending persistent boot preference: {:?}", + rot_pending_persistent_boot_preference + ); + swriteln!( + s, + "RoT transient boot preference: {:?}", + rot_transient_boot_preference + ); swriteln!(s, "zpools ({}):", sled_resources.zpools.len()); for (zpool, disk) in &sled_resources.zpools { swriteln!(s, " {:?}", zpool); diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index f926c4a1451..419d05f4019 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -45,6 +45,9 @@ SP inactive version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 055c4910-b641-46d9-b52d-313aae9d9cbf (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-055c4910-b641-46d9-b52d-313aae9d9cbf" }, disk_id: 6a0cb52f-5cc2-48a5-9f44-ac8dea3ac45b (physical_disk), policy: InService, state: Active } @@ -409,6 +412,9 @@ SP inactive version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (4): 0477165a-a72e-4814-b8d6-74aa02cb2040 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-0477165a-a72e-4814-b8d6-74aa02cb2040" }, disk_id: 6a5a31ab-4edc-44e0-a7a1-4190bfe582f7 (physical_disk), policy: InService, state: Active } diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout index a6fb66d987b..f96d91ed2d9 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout @@ -33,6 +33,9 @@ SP inactive version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -83,6 +86,9 @@ SP inactive version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -118,6 +124,9 @@ SP inactive version: Some("2.0.0") RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -153,6 +162,9 @@ SP inactive version: Some("2.0.0") RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -188,6 +200,9 @@ SP inactive version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -223,6 +238,9 @@ SP inactive version: Some("5.0.0") RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -290,6 +308,9 @@ SP inactive version: Some("5.0.0") RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 2423b624919..e04fc2eebe6 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -501,21 +501,82 @@ impl SystemDescription { sled_id: SledUuid, ) -> anyhow::Result<&RotSlot> { let sp_state = self.sled_sp_state(sled_id)?; - if let Some((_hw_slot, sp_state)) = sp_state { - let slot = match &sp_state.rot { + sp_state + .ok_or_else(|| { + anyhow!("failed to retrieve SP state from sled id: {sled_id}") + }) + .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { RotState::V2 { active, .. } | RotState::V3 { active, .. } => { - active + Ok(active) } - RotState::CommunicationFailed { message } => { - return Err(anyhow!( - "failed to retrieve active RoT slot due to communication failure: {message}" - )); + RotState::CommunicationFailed { message } => Err(anyhow!( + "failed to retrieve active RoT slot due to \ + communication failure: {message}" + )), + }) + } + + pub fn sled_rot_persistent_boot_preference( + &self, + sled_id: SledUuid, + ) -> anyhow::Result<&RotSlot> { + let sp_state = self.sled_sp_state(sled_id)?; + sp_state + .ok_or_else(|| { + anyhow!("failed to retrieve SP state from sled id: {sled_id}") + }) + .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + RotState::V2 { persistent_boot_preference, .. } + | RotState::V3 { persistent_boot_preference, .. } => { + Ok(persistent_boot_preference) } - }; - Ok(slot) - } else { - Err(anyhow!("failed to retrieve SP state from sled id: {sled_id}")) - } + RotState::CommunicationFailed { message } => Err(anyhow!( + "failed to retrieve persistent boot preference slot \ + due to communication failure: {message}" + )), + }) + } + + pub fn sled_rot_pending_persistent_boot_preference( + &self, + sled_id: SledUuid, + ) -> anyhow::Result<&Option> { + let sp_state = self.sled_sp_state(sled_id)?; + sp_state + .ok_or_else(|| { + anyhow!("failed to retrieve SP state from sled id: {sled_id}") + }) + .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + RotState::V2 { pending_persistent_boot_preference, .. } + | RotState::V3 { pending_persistent_boot_preference, .. } => { + Ok(pending_persistent_boot_preference) + } + RotState::CommunicationFailed { message } => Err(anyhow!( + "failed to retrieve pending persistent boot \ + preference slot due to communication failure: {message}" + )), + }) + } + + pub fn sled_rot_transient_boot_preference( + &self, + sled_id: SledUuid, + ) -> anyhow::Result<&Option> { + let sp_state = self.sled_sp_state(sled_id)?; + sp_state + .ok_or_else(|| { + anyhow!("failed to retrieve SP state from sled id: {sled_id}") + }) + .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + RotState::V2 { transient_boot_preference, .. } + | RotState::V3 { transient_boot_preference, .. } => { + Ok(transient_boot_preference) + } + RotState::CommunicationFailed { message } => Err(anyhow!( + "failed to retrieve transient boot preference slot \ + due to communication failure: {message}" + )), + }) } pub fn sled_rot_slot_a_version( From 4827e16fdf4aa42fd0f329400640dbf5145b8ea2 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 27 Jun 2025 15:57:23 +1200 Subject: [PATCH 10/42] clean up --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index dca7145ead9..5da7b302fe5 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -73,7 +73,6 @@ pub fn plan_mgs_updates( boards_preferred.insert(update.baseboard_id.clone()); } Ok(MgsUpdateStatus::Impossible) => { - // TODO-K: Would be nice if it said why it was impossible info!( log, "SP update impossible \ @@ -131,7 +130,6 @@ pub fn plan_mgs_updates( match try_make_update(log, board, inventory, current_artifacts) { Some(update) => { - // TODO-K: should this somehow change to say RoT or bootloader or whatever? info!(log, "configuring SP update"; &update); rv.insert(update); } @@ -514,9 +512,6 @@ fn try_make_update( // updates, we'll try these in a hardcoded priority order until any of them // returns `Some`. The order is described in RFD 565 section "Update // Sequence". For now, we only plan SP and RoT updates. - - // let cabooses = &inventory.cabooses_found; - // info!(&log, "DEBUG: Components"; "baseboard-id" => ?baseboard_id, "cabooses" => ?cabooses, "artifacts" => ?current_artifacts); try_make_update_rot(log, baseboard_id, inventory, current_artifacts) .or_else(|| { try_make_update_sp(log, baseboard_id, inventory, current_artifacts) @@ -729,7 +724,6 @@ fn try_make_update_rot( return false; } - // info!(&log, "DEBUG: active slot {}", active_slot); match active_slot { RotSlot::A => { let slot_a_artifacts = [ @@ -737,7 +731,6 @@ fn try_make_update_rot( ArtifactKind::PSC_ROT_IMAGE_A, ArtifactKind::SWITCH_ROT_IMAGE_A, ]; - // info!(&log, "DEBUG: match slot a artifact kind {}", a.id.kind); if slot_a_artifacts.contains(&a.id.kind) { return true; @@ -750,14 +743,12 @@ fn try_make_update_rot( ArtifactKind::SWITCH_ROT_IMAGE_B, ]; - // info!(&log, "DEBUG: match slot b artifact kind {}", a.id.kind); if slot_b_artifacts.contains(&a.id.kind) { return true; } } } - // info!(&log, "DEBUG: no match artifact kind {}", a.id.kind); false }) .collect(); From e045766a91f3e957c0ec1566afdb86210a0d3fd1 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 27 Jun 2025 16:28:48 +1200 Subject: [PATCH 11/42] clean up --- nexus/mgs-updates/src/test_util/test_artifacts.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/mgs-updates/src/test_util/test_artifacts.rs b/nexus/mgs-updates/src/test_util/test_artifacts.rs index 8d9ecfbdd02..33cb4853009 100644 --- a/nexus/mgs-updates/src/test_util/test_artifacts.rs +++ b/nexus/mgs-updates/src/test_util/test_artifacts.rs @@ -79,7 +79,6 @@ impl TestArtifacts { // Make an RoT update artifact for SimGimlet. let rot_gimlet_artifact_caboose = CabooseBuilder::default() .git_commit("fake-git-commit") - // TODO-K: change to SIM_ROT .board(SIM_GIMLET_BOARD) .version("0.0.0") .name("fake-name") From f515fb77e036715af15a41dceeb90a27144e3f8e Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 9 Jul 2025 14:15:32 +1200 Subject: [PATCH 12/42] fixes after merge --- .../tests/output/cmds-noop-image-source-stdout | 16 ++++++++-------- .../tests/output/cmds-target-release-stdout | 9 +++++++++ .../planning/src/mgs_updates/mod.rs | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout index 6ca20a8347c..f70cc2f849b 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-noop-image-source-stdout @@ -40,8 +40,8 @@ created repo-2.0.0.zip for system version 2.0.0 INFO extracting uploaded archive to INFO created directory to store extracted artifacts, path: INFO added artifact, name: SimGimletSp, kind: gimlet_sp, version: 1.0.0, hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, length: 747 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 INFO added artifact, name: fake-gimlet-rot-bootloader, kind: gimlet_rot_bootloader, version: 1.0.0, hash: 005ea358f1cd316df42465b1e3a0334ea22cc0c0442cf9ddf9b42fbf49780236, length: 750 INFO added artifact, name: fake-host, kind: host_phase_1, version: 1.0.0, hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, length: 524288 INFO added artifact, name: fake-host, kind: host_phase_2, version: 1.0.0, hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, length: 1048576 @@ -185,7 +185,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -337,10 +337,10 @@ to: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -+ sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ++ sled 0 model0 serial0 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } internal DNS: @@ -530,7 +530,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO SP update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 78634a5374a..80822bb5ae4 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -760,6 +760,9 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds > blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 +INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1341,6 +1344,9 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: active -> 1.0.0 generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds > blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 +INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1534,6 +1540,9 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot a -> 1.0.0 generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds > blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 +INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index f1e1d5619fe..1674172807c 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -813,7 +813,7 @@ fn try_make_update_rot( Some(PendingMgsUpdate { baseboard_id: baseboard_id.clone(), sp_type: sp_info.sp_type, - slot_id: u32::from(sp_info.sp_slot), + slot_id: sp_info.sp_slot, details: PendingMgsUpdateDetails::Rot { expected_active_slot, expected_inactive_version, From c44d9bff252571414fd42c29a8e7d2aa0e399f07 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 9 Jul 2025 17:40:40 +1200 Subject: [PATCH 13/42] address style comments --- dev-tools/reconfigurator-cli/src/lib.rs | 4 - .../planning/src/mgs_updates/mod.rs | 74 ++++++++++--------- nexus/reconfigurator/planning/src/planner.rs | 3 +- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index ecfdd47d765..17cdd872343 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -509,10 +509,6 @@ struct SledUpdateRotArgs { /// sets the version reported for the RoT slot b #[clap(long, required_unless_present_any = &["slot_a"])] slot_b: Option, - // TODO: In a follow up PR we could set other fields as well. - // They would be useful to simulate failures. - // These would be: active_slot, persistent_boot_preference, - // transient_boot_preference and pending_persistent_boot_preference. } #[derive(Debug, Args)] diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 1674172807c..6ce9ff8af5e 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -258,18 +258,30 @@ fn mgs_update_status( version: found_active_version, }; + let expected = RotUpdateState { + active_slot: expected_active_slot.clone(), + persistent_boot_preference: + *expected_persistent_boot_preference, + pending_persistent_boot_preference: + *expected_pending_persistent_boot_preference, + transient_boot_preference: *expected_transient_boot_preference, + }; + + let found = RotUpdateState { + active_slot: found_active_slot, + persistent_boot_preference: rot_state + .persistent_boot_preference, + pending_persistent_boot_preference: rot_state + .pending_persistent_boot_preference, + transient_boot_preference: rot_state.transient_boot_preference, + }; + Ok(mgs_update_status_rot( desired_version, - &expected_active_slot, + expected, + found, expected_inactive_version, - expected_persistent_boot_preference, - expected_pending_persistent_boot_preference, - expected_transient_boot_preference, - &found_active_slot, found_inactive_version, - &rot_state.persistent_boot_preference, - &rot_state.pending_persistent_boot_preference, - &rot_state.transient_boot_preference, )) } PendingMgsUpdateDetails::RotBootloader { .. } => { @@ -390,21 +402,21 @@ fn mgs_update_status_sp( } } -#[allow(clippy::too_many_arguments)] +struct RotUpdateState { + active_slot: ExpectedActiveRotSlot, + persistent_boot_preference: RotSlot, + pending_persistent_boot_preference: Option, + transient_boot_preference: Option, +} + fn mgs_update_status_rot( desired_version: &ArtifactVersion, - expected_active_slot: &ExpectedActiveRotSlot, + expected: RotUpdateState, + found: RotUpdateState, expected_inactive_version: &ExpectedVersion, - expected_persistent_boot_preference: &RotSlot, - expected_pending_persistent_boot_preference: &Option, - expected_transient_boot_preference: &Option, - found_active_slot: &ExpectedActiveRotSlot, found_inactive_version: Option<&str>, - found_persistent_boot_preference: &RotSlot, - found_pending_persistent_boot_preference: &Option, - found_transient_boot_preference: &Option, ) -> MgsUpdateStatus { - if &found_active_slot.version() == desired_version { + if &found.active_slot.version() == desired_version { // If we find the desired version in the active slot, we're done. return MgsUpdateStatus::Done; } @@ -416,28 +428,20 @@ fn mgs_update_status_rot( // preference are still what they were when we configured this update. // If not, then this update cannot proceed as currently configured. // It will fail its precondition check. - if found_active_slot.version() != expected_active_slot.version() { - return MgsUpdateStatus::Impossible; - } - - if found_persistent_boot_preference != expected_persistent_boot_preference { - return MgsUpdateStatus::Impossible; - } - - if found_pending_persistent_boot_preference - != expected_pending_persistent_boot_preference + if found.active_slot.version() != expected.active_slot.version() + || found.persistent_boot_preference + != expected.persistent_boot_preference + || found.pending_persistent_boot_preference + != expected.pending_persistent_boot_preference + || found.transient_boot_preference != expected.transient_boot_preference { return MgsUpdateStatus::Impossible; } - if found_transient_boot_preference != expected_transient_boot_preference { - return MgsUpdateStatus::Impossible; - } - // If either found pending persistent boot preference or found transient // boot preference are not empty, then an update is not done - if found_pending_persistent_boot_preference.is_some() - || found_transient_boot_preference.is_some() + if found.pending_persistent_boot_preference.is_some() + || found.transient_boot_preference.is_some() { return MgsUpdateStatus::NotDone; } @@ -449,7 +453,7 @@ fn mgs_update_status_rot( // https://github.com/oxidecomputer/omicron/issues/8414 for context // about when we'll be able to know whether an it's an ongoing update // or an RoT in a failed state. - if found_persistent_boot_preference != &found_active_slot.slot { + if found.persistent_boot_preference != found.active_slot.slot { // TODO-K: I am not 100% sure on this one. It may be impossible? return MgsUpdateStatus::NotDone; } diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index b9969e8256f..e9f3670eaef 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -36,7 +36,6 @@ use nexus_types::deployment::ZpoolFilter; use nexus_types::external_api::views::PhysicalDiskPolicy; use nexus_types::external_api::views::SledPolicy; use nexus_types::external_api::views::SledState; -use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_uuid_kinds::PhysicalDiskUuid; @@ -1175,7 +1174,7 @@ impl<'a> Planner<'a> { .map(|(_sled_id, details)| &details.baseboard_id) .collect(); - let included_baseboards: BTreeSet> = + let included_baseboards = self.inventory .sps .iter() From 031a18f874c509fa3778997cd03fa7e79c3a7a29 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 9 Jul 2025 18:31:19 +1200 Subject: [PATCH 14/42] fix target-release test --- .../tests/input/cmds-target-release.txt | 19 +- .../tests/output/cmds-target-release-stdout | 234 ++++++++++++++++-- 2 files changed, 227 insertions(+), 26 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt b/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt index 41aa93435cc..0fecace4065 100644 --- a/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt +++ b/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt @@ -73,26 +73,33 @@ inventory-generate blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac +# Now we'll change the inactive slot contents of the simulated SP. Like with the +# RoT, this should make the update impossible and cause the planner to fix it. +sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 +inventory-generate +blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 +blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 + # Let's simulate the successful SP update as well. # Another couple of planning steps should try to update the last sled. sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 inventory-generate -blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 -blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 +blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 +blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 # Update the RoT on the last sled. # There should be one last pending SP update. sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 inventory-generate -blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 -blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 +blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b # Finish updating the last sled and do one more planning run. # This should update one control plane zone. sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 inventory-generate -blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 -blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b +blueprint-plan 626487fa-7139-45ec-8416-902271fc730b 04bc9001-0836-4fec-b9cb-9d4760caf8b4 +blueprint-diff 626487fa-7139-45ec-8416-902271fc730b c1a0d242-9160-40f4-96ae-61f8f40a0b1b # We should continue walking through the update. We need to build out a # reconfigurator-cli subcommand to simulate updated zone image sources (just diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 80822bb5ae4..e189c79ae06 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -1335,15 +1335,209 @@ external DNS: +> # Now we'll change the inactive slot contents of the simulated SP. Like with the +> # RoT, this should make the update impossible and cause the planner to fix it. +> sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: inactive -> 0.5.0 + +> inventory-generate +generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds + +> blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 +INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 +INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 +INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 +INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 +INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 +INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 +INFO SP update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO reached maximum number of pending SP updates, max: 1 +INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify +generated blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 based on parent blueprint 9034c710-3e57-45f3-99e5-4316145e87ac + +> blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 +from: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac +to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) (unchanged) + cluster.preserve_downgrade_option: (do not modify) (unchanged) + + METADATA: + internal DNS version::: 1 (unchanged) + external DNS version::: 1 (unchanged) + target release min gen: 1 (unchanged) + + OXIMETER SETTINGS: + generation: 1 (unchanged) + read from:: SingleNode (unchanged) + + PENDING MGS UPDATES: + + Pending MGS-managed updates (all baseboards): + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +* sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 - Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + └─ + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) } + + +internal DNS: + DNS zone: "control-plane.oxide.internal" (unchanged) + name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) + AAAA fd00:1122:3344:101::27 + name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) + AAAA fd00:1122:3344:101::22 + name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) + AAAA fd00:1122:3344:102::1 + name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) + AAAA fd00:1122:3344:102::23 + name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) + AAAA fd00:1122:3344:103::22 + name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) + AAAA fd00:1122:3344:2::1 + name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) + AAAA fd00:1122:3344:102::22 + name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) + AAAA fd00:1122:3344:101::25 + name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) + AAAA fd00:1122:3344:102::21 + name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) + AAAA fd00:1122:3344:101::21 + name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) + AAAA fd00:1122:3344:103::26 + name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) + AAAA fd00:1122:3344:102::24 + name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) + AAAA fd00:1122:3344:103::24 + name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) + AAAA fd00:1122:3344:103::27 + name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) + AAAA fd00:1122:3344:101::23 + name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) + AAAA fd00:1122:3344:102::28 + name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) + AAAA fd00:1122:3344:101::1 + name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) + AAAA fd00:1122:3344:1::1 + name: @ (records: 3) + NS ns1.control-plane.oxide.internal + NS ns2.control-plane.oxide.internal + NS ns3.control-plane.oxide.internal + name: _clickhouse-admin-single-server._tcp (records: 1) + SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse-native._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _clickhouse._tcp (records: 1) + SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _crucible-pantry._tcp (records: 3) + SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal + SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal + SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal + name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) + SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal + name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) + SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal + name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) + SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal + name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) + SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal + name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) + SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal + name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) + SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal + name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) + SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal + name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) + SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal + name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) + SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal + name: _external-dns._tcp (records: 3) + SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal + SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal + SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal + name: _internal-ntp._tcp (records: 3) + SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal + SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal + SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal + name: _nameservice._tcp (records: 3) + SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal + SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal + SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal + name: _nexus._tcp (records: 3) + SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal + SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal + SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal + name: _oximeter-reader._tcp (records: 1) + SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal + name: _repo-depot._tcp (records: 3) + SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal + SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal + SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal + name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) + AAAA fd00:1122:3344:102::25 + name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) + AAAA fd00:1122:3344:101::24 + name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) + AAAA fd00:1122:3344:102::26 + name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) + AAAA fd00:1122:3344:103::1 + name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) + AAAA fd00:1122:3344:101::26 + name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) + AAAA fd00:1122:3344:102::27 + name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) + AAAA fd00:1122:3344:3::1 + name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) + AAAA fd00:1122:3344:103::21 + name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) + AAAA fd00:1122:3344:103::25 + name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) + AAAA fd00:1122:3344:103::23 + name: ns1 (records: 1) + AAAA fd00:1122:3344:1::1 + name: ns2 (records: 1) + AAAA fd00:1122:3344:2::1 + name: ns3 (records: 1) + AAAA fd00:1122:3344:3::1 + +external DNS: + DNS zone: "oxide.example" (unchanged) + name: @ (records: 3) + NS ns1.oxide.example + NS ns2.oxide.example + NS ns3.oxide.example + name: example-silo.sys (records: 3) + A 192.0.2.2 + A 192.0.2.3 + A 192.0.2.4 + name: ns1 (records: 1) + A 198.51.100.1 + name: ns2 (records: 1) + A 198.51.100.2 + name: ns3 (records: 1) + A 198.51.100.3 + + + + > # Let's simulate the successful SP update as well. > # Another couple of planning steps should try to update the last sled. > sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: active -> 1.0.0 > inventory-generate -generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds +generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds -> blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 +> blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 @@ -1357,17 +1551,17 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO skipping board for SP update, serial_number: serial1, part_number: model1 INFO skipping board for SP update, serial_number: serial0, part_number: model0 INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO ran out of boards for SP update INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 based on parent blueprint 9034c710-3e57-45f3-99e5-4316145e87ac +generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 -> blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 -from: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac -to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 +> blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +from: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 +to: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -1388,7 +1582,7 @@ to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- sp_type slot part_number serial_number artifact_hash artifact_version details ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } +- sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) } + sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } @@ -1537,9 +1731,9 @@ external DNS: set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot a -> 1.0.0 > inventory-generate -generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds +generated inventory collection 39363465-89ae-4ac2-9be1-099068da9d45 from configured sleds -> blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 +> blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 @@ -1557,11 +1751,11 @@ INFO SP update completed (will remove it and re-evaluate board), artifact_versio INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO reached maximum number of pending SP updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 +generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 -> blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 -from: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 -to: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +> blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b +from: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +to: blueprint 626487fa-7139-45ec-8416-902271fc730b COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -1731,9 +1925,9 @@ external DNS: set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 SP versions: active -> 1.0.0 > inventory-generate -generated inventory collection 39363465-89ae-4ac2-9be1-099068da9d45 from configured sleds +generated inventory collection 04bc9001-0836-4fec-b9cb-9d4760caf8b4 from configured sleds -> blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 +> blueprint-plan 626487fa-7139-45ec-8416-902271fc730b 04bc9001-0836-4fec-b9cb-9d4760caf8b4 INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 @@ -1754,11 +1948,11 @@ INFO skipping board for SP update, serial_number: serial1, part_number: model1 INFO ran out of boards for SP update INFO updating zone image source in-place, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545, kind: Clickhouse, image_source: artifact: version 1.0.0 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +generated blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b based on parent blueprint 626487fa-7139-45ec-8416-902271fc730b -> blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b -from: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 -to: blueprint 626487fa-7139-45ec-8416-902271fc730b +> blueprint-diff 626487fa-7139-45ec-8416-902271fc730b c1a0d242-9160-40f4-96ae-61f8f40a0b1b +from: blueprint 626487fa-7139-45ec-8416-902271fc730b +to: blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b MODIFIED SLEDS: From 642fa9257e1cbd0aede50b54dd992e3798ae7db4 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 9 Jul 2025 18:38:51 +1200 Subject: [PATCH 15/42] Address comments --- .../planning/src/mgs_updates/mod.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 6ce9ff8af5e..4b3bb5d428d 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -427,6 +427,9 @@ fn mgs_update_status_rot( // preference, pending persistent boot preference, and transient boot // preference are still what they were when we configured this update. // If not, then this update cannot proceed as currently configured. + // If there is a mismatch between the found persistent boot preference + // and the found active slot then this means a failed update. We cannot + // proceed. // It will fail its precondition check. if found.active_slot.version() != expected.active_slot.version() || found.persistent_boot_preference @@ -434,27 +437,27 @@ fn mgs_update_status_rot( || found.pending_persistent_boot_preference != expected.pending_persistent_boot_preference || found.transient_boot_preference != expected.transient_boot_preference + || found.persistent_boot_preference != found.active_slot.slot { return MgsUpdateStatus::Impossible; } // If either found pending persistent boot preference or found transient // boot preference are not empty, then an update is not done + // + // TODO: Alternatively, this could also mean a failed update. See + // https://github.com/oxidecomputer/omicron/issues/8414 for context + // about when we'll be able to know whether an it's an ongoing update + // or an RoT in a failed state. if found.pending_persistent_boot_preference.is_some() || found.transient_boot_preference.is_some() { return MgsUpdateStatus::NotDone; } - // If there is a mismatch between the found persistent boot preference - // and the found active slot then the update is not done. - // - // TODO: Alternatively, this could also mean a failed update. See - // https://github.com/oxidecomputer/omicron/issues/8414 for context - // about when we'll be able to know whether an it's an ongoing update - // or an RoT in a failed state. - if found.persistent_boot_preference != found.active_slot.slot { - // TODO-K: I am not 100% sure on this one. It may be impossible? + // If there is a mismatch between the expected active slot and the found + // active slot then the update is not done. + if found.active_slot.slot != expected.active_slot.slot { return MgsUpdateStatus::NotDone; } From 04342975baf634b499b5c1576af1db61502d4b43 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 10 Jul 2025 15:42:29 +1200 Subject: [PATCH 16/42] address comments --- .../planning/src/mgs_updates/mod.rs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 4b3bb5d428d..779f75f2658 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -431,6 +431,9 @@ fn mgs_update_status_rot( // and the found active slot then this means a failed update. We cannot // proceed. // It will fail its precondition check. + // Transient boot preference is not in use yet, if we find that it is set we + // should not proceed. Once https://github.com/oxidecomputer/hubris/pull/2050 + // is implemented, we should revist this check if found.active_slot.version() != expected.active_slot.version() || found.persistent_boot_preference != expected.persistent_boot_preference @@ -438,25 +441,26 @@ fn mgs_update_status_rot( != expected.pending_persistent_boot_preference || found.transient_boot_preference != expected.transient_boot_preference || found.persistent_boot_preference != found.active_slot.slot + || found.transient_boot_preference.is_some() + || expected.transient_boot_preference.is_some() { return MgsUpdateStatus::Impossible; } - // If either found pending persistent boot preference or found transient - // boot preference are not empty, then an update is not done + // If found pending persistent boot preference is not empty, then an update + // is not done. // // TODO: Alternatively, this could also mean a failed update. See // https://github.com/oxidecomputer/omicron/issues/8414 for context // about when we'll be able to know whether an it's an ongoing update // or an RoT in a failed state. if found.pending_persistent_boot_preference.is_some() - || found.transient_boot_preference.is_some() { return MgsUpdateStatus::NotDone; } // If there is a mismatch between the expected active slot and the found - // active slot then the update is not done. + // active slot then an update is not done. if found.active_slot.slot != expected.active_slot.slot { return MgsUpdateStatus::NotDone; } @@ -735,6 +739,8 @@ fn try_make_update_rot( return false; } + // TODO-K: Do SIGN check or verify CMPA/CFPA pages + match active_slot { RotSlot::A => { let slot_a_artifacts = [ @@ -774,9 +780,10 @@ fn try_make_update_rot( if matching_artifacts.len() > 1 { // This should be impossible unless we shipped a TUF repo with more - // than 1 artifact for the same board and slot. But it doesn't prevent - // us from picking one and proceeding. Make a note and proceed. - warn!(log, "found more than one matching artifact for RoT update"); + // than 1 artifact for the same board and slot that verifies against the + // RoT's CMPA/CFPA. But it doesn't prevent us from picking one and + // proceeding. Make a note and proceed. + error!(log, "found more than one matching artifact for RoT update"); } let artifact = matching_artifacts[0]; From 599589f005e5496113dc1c4de98fe7ec3cf46449 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 10 Jul 2025 15:43:09 +1200 Subject: [PATCH 17/42] fmt --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 779f75f2658..0428169c723 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -454,8 +454,7 @@ fn mgs_update_status_rot( // https://github.com/oxidecomputer/omicron/issues/8414 for context // about when we'll be able to know whether an it's an ongoing update // or an RoT in a failed state. - if found.pending_persistent_boot_preference.is_some() - { + if found.pending_persistent_boot_preference.is_some() { return MgsUpdateStatus::NotDone; } From 1ab86229acc0031ea3fe4026043b2026811cd316 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 10:30:50 +1200 Subject: [PATCH 18/42] working test_basic --- .../planning/src/mgs_updates/mod.rs | 205 +++++++++++++----- 1 file changed, 149 insertions(+), 56 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 0428169c723..b27c6257eff 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -869,6 +869,7 @@ mod test { use std::collections::BTreeMap; use std::collections::BTreeSet; use tufaceous_artifact::ArtifactHash; + use tufaceous_artifact::ArtifactKind; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::KnownArtifactKind; @@ -894,6 +895,8 @@ mod test { const ARTIFACT_HASH_SP_PSC_B: ArtifactHash = ArtifactHash([9; 32]); /// Hash of fake artifact for fake psc-c SP const ARTIFACT_HASH_SP_PSC_C: ArtifactHash = ArtifactHash([10; 32]); + /// Hash of fake artifact for fake oxide-rot-1 RoT + const ARTIFACT_HASH_OXIDE_ROT_1: ArtifactHash = ArtifactHash([13; 32]); // unused artifact hashes @@ -901,6 +904,13 @@ mod test { const ARTIFACT_HASH_NEXUS: ArtifactHash = ArtifactHash([34; 32]); const ARTIFACT_HASH_HOST_OS: ArtifactHash = ArtifactHash([35; 32]); + /// Hash of the RoT development signing key + // TODO-K: This sign only makes sense for one kind of artifact, we want to + // have different ones for switch and power + const ROT_STAGING_DEVEL_SIGN: &str = + "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf"; + + fn test_artifact_for_board(board: &str) -> ArtifactHash { match board { "gimlet-d" => ARTIFACT_HASH_SP_GIMLET_D, @@ -909,11 +919,14 @@ mod test { "sidecar-c" => ARTIFACT_HASH_SP_SIDECAR_C, "psc-b" => ARTIFACT_HASH_SP_PSC_B, "psc-c" => ARTIFACT_HASH_SP_PSC_C, + "oxide-rot-1" => ARTIFACT_HASH_OXIDE_ROT_1, + // TODO-K: figure out how to have artifacts for different RoTs and + // bootloaders which have the same board name. _ => panic!("test bug: no artifact for board {board:?}"), } } - /// Describes the SPs in the environment used in these tests + /// Describes the SPs and RoTs in the environment used in these tests /// /// There will be: /// @@ -923,22 +936,23 @@ mod test { /// /// The specific set of hardware (boards) vary and are hardcoded: /// - /// - sled 0: gimlet-d - /// - other sleds: gimlet-e - /// - switch 0: sidecar-b - /// - switch 1: sidecar-c - /// - psc 0: psc-b - /// - psc 1: psc-c - fn test_config() -> BTreeMap<(SpType, u16), (&'static str, &'static str)> { + /// - sled 0: gimlet-d, oxide-rot-1 + /// - other sleds: gimlet-e, oxide-rot-1 + /// - switch 0: sidecar-b, oxide-rot-1 + /// - switch 1: sidecar-c, oxide-rot-1 + /// - psc 0: psc-b, oxide-rot-1 + /// - psc 1: psc-c, oxide-rot-1 + fn test_config() + -> BTreeMap<(SpType, u16, ), (&'static str, &'static str, &'static str)> { BTreeMap::from([ - ((SpType::Sled, 0), ("sled_0", "gimlet-d")), - ((SpType::Sled, 1), ("sled_1", "gimlet-e")), - ((SpType::Sled, 2), ("sled_2", "gimlet-e")), - ((SpType::Sled, 3), ("sled_3", "gimlet-e")), - ((SpType::Switch, 0), ("switch_0", "sidecar-b")), - ((SpType::Switch, 1), ("switch_1", "sidecar-c")), - ((SpType::Power, 0), ("power_0", "psc-b")), - ((SpType::Power, 1), ("power_1", "psc-c")), + ((SpType::Sled, 0), ("sled_0", "gimlet-d", "oxide-rot-1")), + ((SpType::Sled, 1), ("sled_1", "gimlet-e", "oxide-rot-1")), + ((SpType::Sled, 2), ("sled_2", "gimlet-e", "oxide-rot-1")), + ((SpType::Sled, 3), ("sled_3", "gimlet-e", "oxide-rot-1")), + ((SpType::Switch, 0), ("switch_0", "sidecar-b", "oxide-rot-1")), + ((SpType::Switch, 1), ("switch_1", "sidecar-c", "oxide-rot-1")), + ((SpType::Power, 0), ("power_0", "psc-b", "oxide-rot-1")), + ((SpType::Power, 1), ("power_1", "psc-c", "oxide-rot-1")), ]) } @@ -953,49 +967,61 @@ mod test { let artifacts = vec![ make_artifact( "control-plane", - KnownArtifactKind::ControlPlane, + KnownArtifactKind::ControlPlane.into(), ARTIFACT_HASH_CONTROL_PLANE, ), make_artifact( "nexus", - KnownArtifactKind::Zone, + KnownArtifactKind::Zone.into(), ARTIFACT_HASH_NEXUS, ), make_artifact( "host-os", - KnownArtifactKind::Host, + KnownArtifactKind::Host.into(), ARTIFACT_HASH_HOST_OS, ), make_artifact( "gimlet-d", - KnownArtifactKind::GimletSp, + KnownArtifactKind::GimletSp.into(), test_artifact_for_board("gimlet-d"), ), make_artifact( "gimlet-e", - KnownArtifactKind::GimletSp, + KnownArtifactKind::GimletSp.into(), test_artifact_for_board("gimlet-e"), ), make_artifact( "sidecar-b", - KnownArtifactKind::SwitchSp, + KnownArtifactKind::SwitchSp.into(), test_artifact_for_board("sidecar-b"), ), make_artifact( "sidecar-c", - KnownArtifactKind::SwitchSp, + KnownArtifactKind::SwitchSp.into(), test_artifact_for_board("sidecar-c"), ), make_artifact( "psc-b", - KnownArtifactKind::PscSp, + KnownArtifactKind::PscSp.into(), test_artifact_for_board("psc-b"), ), make_artifact( "psc-c", - KnownArtifactKind::PscSp, + KnownArtifactKind::PscSp.into(), test_artifact_for_board("psc-c"), ), + make_artifact( + "oxide-rot-1", + ArtifactKind::GIMLET_ROT_IMAGE_A, + test_artifact_for_board("oxide-rot-1"), + ), + // TODO-K: Does this work as expected? + make_artifact( + "oxide-rot-1", + ArtifactKind::GIMLET_ROT_IMAGE_B, + test_artifact_for_board("oxide-rot-1"), + ), + // Make more artifacts for other artifact kinds and for other slots? ]; TufRepoDescription { @@ -1012,14 +1038,14 @@ mod test { fn make_artifact( name: &str, - kind: KnownArtifactKind, + kind: ArtifactKind, hash: ArtifactHash, ) -> TufArtifactMeta { TufArtifactMeta { id: ArtifactId { name: name.to_string(), version: ARTIFACT_VERSION_2, - kind: kind.into(), + kind: kind, }, hash, size: 0, // unused here @@ -1028,12 +1054,13 @@ mod test { // Construct inventory for an environment suitable for our testing. // - // See test_config() for information about the hardware. All SPs will - // appear to be running version `active_version` except those identified in - // `active_version_exceptions`. All SPs will appear to have - // `inactive_version` in the inactive slot. + // See test_config() for information about the hardware. All SPs and RoTs + // will appear to be running version `active_version` except those + // identified in `active_version_exceptions`. All SPs and RoTs will appear + // to have `inactive_version` in the inactive slot. fn make_collection( active_version: ArtifactVersion, + // TODO-K: Use board as well to filter whether we want the SP ort RoT? active_version_exceptions: &BTreeMap<(SpType, u16), ArtifactVersion>, inactive_version: ExpectedVersion, ) -> Collection { @@ -1065,7 +1092,11 @@ mod test { }; let test_config = test_config(); - for ((sp_type, sp_slot), (serial, caboose_board)) in test_config { + for ( + (sp_type, sp_slot), + (serial, caboose_sp_board, caboose_rot_board), + ) in test_config + { let sp_state = SpState { model: format!("dummy_{}", sp_type), serial_number: serial.to_string(), @@ -1085,16 +1116,32 @@ mod test { CabooseWhich::SpSlot0, "test", SpComponentCaboose { - board: caboose_board.to_string(), + board: caboose_sp_board.to_string(), epoch: None, git_commit: String::from("unused"), - name: caboose_board.to_string(), + name: caboose_sp_board.to_string(), sign: None, version: active_version.as_str().to_string(), }, ) .unwrap(); + builder + .found_caboose( + &baseboard_id, + CabooseWhich::RotSlotA, + "test", + SpComponentCaboose { + board: caboose_rot_board.to_string(), + epoch: None, + git_commit: String::from("unused"), + name: caboose_rot_board.to_string(), + sign: Some(ROT_STAGING_DEVEL_SIGN.to_string()), + version: active_version.as_str().to_string(), + }, + ) + .unwrap(); + if let ExpectedVersion::Version(inactive_version) = &inactive_version { @@ -1104,15 +1151,31 @@ mod test { CabooseWhich::SpSlot1, "test", SpComponentCaboose { - board: caboose_board.to_string(), + board: caboose_sp_board.to_string(), epoch: None, git_commit: String::from("unused"), - name: caboose_board.to_string(), + name: caboose_sp_board.to_string(), sign: None, version: inactive_version.as_str().to_string(), }, ) .unwrap(); + + builder + .found_caboose( + &baseboard_id, + CabooseWhich::RotSlotB, + "test", + SpComponentCaboose { + board: caboose_rot_board.to_string(), + epoch: None, + git_commit: String::from("unused"), + name: caboose_rot_board.to_string(), + sign: Some(ROT_STAGING_DEVEL_SIGN.to_string()), + version: inactive_version.as_str().to_string(), + }, + ) + .unwrap(); } } @@ -1148,7 +1211,7 @@ mod test { ); assert!(updates.is_empty()); - // Test that when a TUF repo is specified and one SP is outdated, then + // Test that when a TUF repo is specified and one RoT is outdated, then // it's configured with an update (and the update looks correct). let repo = make_tuf_repo(); let updates = plan_mgs_updates( @@ -1164,7 +1227,8 @@ mod test { assert_eq!(first_update.baseboard_id.serial_number, "sled_0"); assert_eq!(first_update.sp_type, SpType::Sled); assert_eq!(first_update.slot_id, 0); - assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_SP_GIMLET_D); + // assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_SP_GIMLET_D); + assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_OXIDE_ROT_1); assert_eq!(first_update.artifact_version, ARTIFACT_VERSION_2); // Test that when an update is already pending, and nothing changes @@ -1225,11 +1289,14 @@ mod test { assert_eq!(next_update.baseboard_id.serial_number, "switch_1"); assert_eq!(next_update.sp_type, SpType::Switch); assert_eq!(next_update.slot_id, 1); - assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); + assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_OXIDE_ROT_1); + //assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); - // Finally, test that when all SPs are in spec, then no updates are - // configured. + + + // Finally, test that when all RoTs and SPs are in spec, then no updates + // are configured. let updated_collection = make_collection( ARTIFACT_VERSION_2, &BTreeMap::new(), @@ -1277,14 +1344,19 @@ mod test { // Verify the precondition details of an ordinary update. let old_update = updates.into_iter().next().expect("at least one update"); - let PendingMgsUpdateDetails::Sp { - expected_active_version: old_expected_active_version, + //let PendingMgsUpdateDetails::Sp { + // expected_active_version: old_expected_active_version, + // expected_inactive_version: old_expected_inactive_version, + //} = &old_update.details + let PendingMgsUpdateDetails::Rot { + expected_active_slot: old_expected_active_slot, expected_inactive_version: old_expected_inactive_version, + .. } = &old_update.details else { - panic!("expected SP update"); + panic!("expected RoT update"); }; - assert_eq!(ARTIFACT_VERSION_1, *old_expected_active_version); + assert_eq!(ARTIFACT_VERSION_1, old_expected_active_slot.version()); assert_eq!( ExpectedVersion::NoValidVersion, *old_expected_inactive_version @@ -1314,14 +1386,19 @@ mod test { assert_eq!(old_update.slot_id, new_update.slot_id); assert_eq!(old_update.artifact_hash, new_update.artifact_hash); assert_eq!(old_update.artifact_version, new_update.artifact_version); - let PendingMgsUpdateDetails::Sp { - expected_active_version: new_expected_active_version, + // let PendingMgsUpdateDetails::Sp { + // expected_active_version: new_expected_active_version, + // expected_inactive_version: new_expected_inactive_version, + // } = &new_update.details + let PendingMgsUpdateDetails::Rot { + expected_active_slot: new_expected_active_slot, expected_inactive_version: new_expected_inactive_version, + .. } = &new_update.details else { - panic!("expected SP update"); + panic!("expected RoT update"); }; - assert_eq!(ARTIFACT_VERSION_1, *new_expected_active_version); + assert_eq!(ARTIFACT_VERSION_1, new_expected_active_slot.version()); assert_eq!( ExpectedVersion::Version(ARTIFACT_VERSION_1), *new_expected_inactive_version @@ -1352,14 +1429,19 @@ mod test { assert_eq!(old_update.slot_id, new_update.slot_id); assert_eq!(old_update.artifact_hash, new_update.artifact_hash); assert_eq!(old_update.artifact_version, new_update.artifact_version); - let PendingMgsUpdateDetails::Sp { - expected_active_version: new_expected_active_version, + //let PendingMgsUpdateDetails::Sp { + // expected_active_version: new_expected_active_version, + // expected_inactive_version: new_expected_inactive_version, + //} = &new_update.details + let PendingMgsUpdateDetails::Rot { + expected_active_slot: new_expected_active_slot, expected_inactive_version: new_expected_inactive_version, + .. } = &new_update.details else { - panic!("expected SP update"); + panic!("expected RoT update"); }; - assert_eq!(ARTIFACT_VERSION_1_5, *new_expected_active_version); + assert_eq!(ARTIFACT_VERSION_1_5, new_expected_active_slot.version()); assert_eq!( ExpectedVersion::NoValidVersion, *new_expected_inactive_version @@ -1389,11 +1471,21 @@ mod test { // artifact. let mut expected_updates: BTreeMap<_, _> = test_config() .into_iter() - .map(|(k, (serial, board_name))| { - (k, (serial, test_artifact_for_board(board_name))) + // TODO-K: Test RoT too? + .flat_map(|(k, (serial, board_name, rot_board_name))| { + [ + (k, (serial, test_artifact_for_board(board_name))), + // TODO-K: Panics here I need to define the board name without slot I guess? + (k, (serial, test_artifact_for_board(rot_board_name))) + ] }) .collect(); + //debug + if expected_updates.len() != 10000 { + panic!("num of expected updates:{} {expected_updates:?}", expected_updates.len()); + } + for _ in 0..expected_updates.len() { // Generate an inventory collection reflecting that everything is at // version 1 except for what we've already updated. @@ -1464,7 +1556,8 @@ mod test { let mut expected_updates: BTreeMap<_, _> = test_config() .into_iter() - .map(|(k, (serial, board_name))| { + // TODO-K: Test RoT too? + .map(|(k, (serial, board_name, ..))| { (k, (serial, test_artifact_for_board(board_name))) }) .collect(); From 88702d0a5d2f76818b46f585bbbbdb5938c38d0c Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 13:14:52 +1200 Subject: [PATCH 19/42] Adapt sequential test --- .../planning/src/mgs_updates/mod.rs | 193 ++++++++++++++---- 1 file changed, 158 insertions(+), 35 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index b27c6257eff..d45635082c3 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -910,7 +910,13 @@ mod test { const ROT_STAGING_DEVEL_SIGN: &str = "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf"; - + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] + enum SpComponent { + Sp, + Rot, + RotBootloader, + } + fn test_artifact_for_board(board: &str) -> ArtifactHash { match board { "gimlet-d" => ARTIFACT_HASH_SP_GIMLET_D, @@ -942,8 +948,8 @@ mod test { /// - switch 1: sidecar-c, oxide-rot-1 /// - psc 0: psc-b, oxide-rot-1 /// - psc 1: psc-c, oxide-rot-1 - fn test_config() - -> BTreeMap<(SpType, u16, ), (&'static str, &'static str, &'static str)> { + fn test_collection_config() + -> BTreeMap<(SpType, u16), (&'static str, &'static str, &'static str)> { BTreeMap::from([ ((SpType::Sled, 0), ("sled_0", "gimlet-d", "oxide-rot-1")), ((SpType::Sled, 1), ("sled_1", "gimlet-e", "oxide-rot-1")), @@ -956,6 +962,36 @@ mod test { ]) } + /// Describes the SPs and RoTs in the environment used in these tests, but + /// spearated by component for use in sequential testing + fn test_config() + -> BTreeMap<(SpType, u16, SpComponent), (&'static str, &'static str)> { + BTreeMap::from([ + ((SpType::Sled, 0, SpComponent::Sp), ("sled_0", "gimlet-d")), + ((SpType::Sled, 1, SpComponent::Sp), ("sled_1", "gimlet-e")), + ((SpType::Sled, 2, SpComponent::Sp), ("sled_2", "gimlet-e")), + ((SpType::Sled, 3, SpComponent::Sp), ("sled_3", "gimlet-e")), + ((SpType::Switch, 0, SpComponent::Sp), ("switch_0", "sidecar-b")), + ((SpType::Switch, 1, SpComponent::Sp), ("switch_1", "sidecar-c")), + ((SpType::Power, 0, SpComponent::Sp), ("power_0", "psc-b")), + ((SpType::Power, 1, SpComponent::Sp), ("power_1", "psc-c")), + ((SpType::Sled, 0, SpComponent::Rot), ("sled_0", "oxide-rot-1")), + ((SpType::Sled, 1, SpComponent::Rot), ("sled_1", "oxide-rot-1")), + ((SpType::Sled, 2, SpComponent::Rot), ("sled_2", "oxide-rot-1")), + ((SpType::Sled, 3, SpComponent::Rot), ("sled_3", "oxide-rot-1")), + ( + (SpType::Switch, 0, SpComponent::Rot), + ("switch_0", "oxide-rot-1"), + ), + ( + (SpType::Switch, 1, SpComponent::Rot), + ("switch_1", "oxide-rot-1"), + ), + ((SpType::Power, 0, SpComponent::Rot), ("power_0", "oxide-rot-1")), + ((SpType::Power, 1, SpComponent::Rot), ("power_1", "oxide-rot-1")), + ]) + } + /// Returns a TufRepoDescription that we can use to exercise the planning /// code. fn make_tuf_repo() -> TufRepoDescription { @@ -1060,9 +1096,14 @@ mod test { // to have `inactive_version` in the inactive slot. fn make_collection( active_version: ArtifactVersion, - // TODO-K: Use board as well to filter whether we want the SP ort RoT? active_version_exceptions: &BTreeMap<(SpType, u16), ArtifactVersion>, inactive_version: ExpectedVersion, + active_rot_version: ArtifactVersion, + active_rot_version_exceptions: &BTreeMap< + (SpType, u16), + ArtifactVersion, + >, + inactive_rot_version: ExpectedVersion, ) -> Collection { let mut builder = nexus_inventory::CollectionBuilder::new( "planning_mgs_updates_basic", @@ -1091,7 +1132,7 @@ mod test { serial_number: String::from("unused"), }; - let test_config = test_config(); + let test_config = test_collection_config(); for ( (sp_type, sp_slot), (serial, caboose_sp_board, caboose_rot_board), @@ -1109,6 +1150,9 @@ mod test { let active_version = active_version_exceptions .get(&(sp_type, sp_slot)) .unwrap_or(&active_version); + let active_rot_version = active_rot_version_exceptions + .get(&(sp_type, sp_slot)) + .unwrap_or(&active_rot_version); builder .found_caboose( @@ -1137,7 +1181,7 @@ mod test { git_commit: String::from("unused"), name: caboose_rot_board.to_string(), sign: Some(ROT_STAGING_DEVEL_SIGN.to_string()), - version: active_version.as_str().to_string(), + version: active_rot_version.as_str().to_string(), }, ) .unwrap(); @@ -1160,7 +1204,11 @@ mod test { }, ) .unwrap(); + } + if let ExpectedVersion::Version(inactive_rot_version) = + &inactive_rot_version + { builder .found_caboose( &baseboard_id, @@ -1172,7 +1220,7 @@ mod test { git_commit: String::from("unused"), name: caboose_rot_board.to_string(), sign: Some(ROT_STAGING_DEVEL_SIGN.to_string()), - version: inactive_version.as_str().to_string(), + version: inactive_rot_version.as_str().to_string(), }, ) .unwrap(); @@ -1197,6 +1245,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, ); let current_boards = &collection.baseboards; let initial_updates = PendingMgsUpdates::new(); @@ -1254,6 +1305,12 @@ mod test { ((SpType::Switch, 1), ARTIFACT_VERSION_1), ]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([ + ((SpType::Sled, 0), ARTIFACT_VERSION_1), + ((SpType::Switch, 1), ARTIFACT_VERSION_1), + ]), + ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( log, @@ -1273,6 +1330,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( log, @@ -1293,14 +1353,15 @@ mod test { //assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); - - // Finally, test that when all RoTs and SPs are in spec, then no updates // are configured. let updated_collection = make_collection( ARTIFACT_VERSION_2, &BTreeMap::new(), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( log, @@ -1318,6 +1379,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, ); let updates = plan_mgs_updates( log, @@ -1368,6 +1432,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::Version(ARTIFACT_VERSION_1), + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::Version(ARTIFACT_VERSION_1), ); let new_updates = plan_mgs_updates( log, @@ -1411,6 +1478,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1_5)]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1_5)]), + ExpectedVersion::NoValidVersion, ); let new_updates = plan_mgs_updates( log, @@ -1462,29 +1532,25 @@ mod test { let repo = make_tuf_repo(); let nmax_updates = 1; - // Maintain a map of SPs that we've updated. We'll use this to + // Maintain a map of SPs and RoTs that we've updated. We'll use this to // configure the inventory collection that we create at each step. let mut exceptions = BTreeMap::new(); + let mut rot_exceptions = BTreeMap::new(); // We do not control the order of updates. But we expect to update each // of the SPs in this map. When we do, we expect to find the given // artifact. let mut expected_updates: BTreeMap<_, _> = test_config() .into_iter() - // TODO-K: Test RoT too? - .flat_map(|(k, (serial, board_name, rot_board_name))| { - [ - (k, (serial, test_artifact_for_board(board_name))), - // TODO-K: Panics here I need to define the board name without slot I guess? - (k, (serial, test_artifact_for_board(rot_board_name))) - ] + .map(|(k, (serial, board_name))| { + (k, (serial, test_artifact_for_board(board_name))) }) .collect(); - //debug - if expected_updates.len() != 10000 { - panic!("num of expected updates:{} {expected_updates:?}", expected_updates.len()); - } + // //debug + // if expected_updates.len() != 10000 { + // panic!("num of expected updates:{} {expected_updates:?}", expected_updates.len()); + // } for _ in 0..expected_updates.len() { // Generate an inventory collection reflecting that everything is at @@ -1493,6 +1559,9 @@ mod test { ARTIFACT_VERSION_1, &exceptions, ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_1, + &rot_exceptions, + ExpectedVersion::NoValidVersion, ); // For this test, all systems that are found in inventory are part @@ -1514,14 +1583,27 @@ mod test { new_updates.iter().next().expect("at least one update"); verify_one_sp_update(&mut expected_updates, update); - // Update `exceptions` for the next iteration. + // Update `exceptions` or `rot_exceptions` for the next iteration. let sp_type = update.sp_type; let sp_slot = update.slot_id; - assert!( - exceptions - .insert((sp_type, sp_slot), ARTIFACT_VERSION_2) - .is_none() - ); + match update.details { + PendingMgsUpdateDetails::Rot { .. } => { + assert!( + rot_exceptions + .insert((sp_type, sp_slot), ARTIFACT_VERSION_2) + .is_none() + ); + }, + PendingMgsUpdateDetails::Sp { .. } => { + assert!( + exceptions + .insert((sp_type, sp_slot), ARTIFACT_VERSION_2) + .is_none() + ); + }, + PendingMgsUpdateDetails::RotBootloader { .. } => unimplemented!() + } + latest_updates = new_updates; } @@ -1530,6 +1612,9 @@ mod test { ARTIFACT_VERSION_1, &exceptions, ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_1, + &rot_exceptions, + ExpectedVersion::NoValidVersion, ); let last_updates = plan_mgs_updates( log, @@ -1567,6 +1652,9 @@ mod test { ARTIFACT_VERSION_1, &BTreeMap::new(), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_1, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, ); let all_updates = plan_mgs_updates( log, @@ -1587,6 +1675,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::new(), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, ); let all_updates_done = plan_mgs_updates( log, @@ -1602,25 +1693,54 @@ mod test { } fn verify_one_sp_update( - expected_updates: &mut BTreeMap<(SpType, u16), (&str, ArtifactHash)>, + expected_updates: &mut BTreeMap< + (SpType, u16, SpComponent), + (&str, ArtifactHash), + >, update: &PendingMgsUpdate, ) { let sp_type = update.sp_type; let sp_slot = update.slot_id; + let sp_component = match &update.details { + PendingMgsUpdateDetails::Rot { .. } => SpComponent::Rot, + PendingMgsUpdateDetails::RotBootloader { .. } => { + SpComponent::RotBootloader + } + PendingMgsUpdateDetails::Sp { .. } => SpComponent::Sp, + }; println!("found update: {} slot {}", sp_type, sp_slot); + println!( + "DEBUG: REMOVE type: {} slot: {} component {:?}", + sp_type, sp_slot, sp_component + ); let (expected_serial, expected_artifact) = expected_updates - .remove(&(sp_type, sp_slot)) + .remove(&(sp_type, sp_slot, sp_component)) .expect("unexpected update"); assert_eq!(update.artifact_hash, expected_artifact); assert_eq!(update.artifact_version, ARTIFACT_VERSION_2); assert_eq!(update.baseboard_id.serial_number, *expected_serial); - let PendingMgsUpdateDetails::Sp { - expected_active_version, - expected_inactive_version, - } = &update.details - else { - panic!("expected SP update"); + let (expected_active_version, expected_inactive_version) = match &update + .details + { + PendingMgsUpdateDetails::Rot { + expected_active_slot, + expected_inactive_version, + .. + } => (&expected_active_slot.version, expected_inactive_version), + PendingMgsUpdateDetails::Sp { + expected_active_version, + expected_inactive_version, + } => (expected_active_version, expected_inactive_version), + PendingMgsUpdateDetails::RotBootloader { .. } => unimplemented!(), }; + // let PendingMgsUpdateDetails::Sp { + // expected_active_version, + // expected_inactive_version, + // } = &update.details + // else { + // panic!("expected SP update"); + // }; + // Verify update against SpComponent? assert_eq!(*expected_active_version, ARTIFACT_VERSION_1); assert_eq!(*expected_inactive_version, ExpectedVersion::NoValidVersion); } @@ -1640,6 +1760,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, ); let updates = plan_mgs_updates( log, From ce03cca7cc7fbf43f9b8abab4dac5783df7f8f90 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 16:29:17 +1200 Subject: [PATCH 20/42] get all the tests passing --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index d45635082c3..8485ad0f17e 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1641,8 +1641,7 @@ mod test { let mut expected_updates: BTreeMap<_, _> = test_config() .into_iter() - // TODO-K: Test RoT too? - .map(|(k, (serial, board_name, ..))| { + .map(|(k, (serial, board_name))| { (k, (serial, test_artifact_for_board(board_name))) }) .collect(); @@ -1664,7 +1663,11 @@ mod test { &TargetReleaseDescription::TufRepo(repo.clone()), usize::MAX, ); - assert_eq!(all_updates.len(), expected_updates.len()); + // `all_updates` counts each update per SpType. This means an update for + // SP and RoT for the same SpType count as a sinlge update. For + // `expected_updates`, each component update counts as an update, so the + // amount of `all_updates` should be half of `expected_updates`. + assert_eq!(all_updates.len(), expected_updates.len()/2); for update in &all_updates { verify_one_sp_update(&mut expected_updates, update); } From 4f3b735dcc59271fbd14b289b62de33beb5c3c88 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 16:35:11 +1200 Subject: [PATCH 21/42] get all the tests passing --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 8485ad0f17e..911f6a1bdac 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -906,7 +906,7 @@ mod test { /// Hash of the RoT development signing key // TODO-K: This sign only makes sense for one kind of artifact, we want to - // have different ones for switch and power + // have different ones for switch and power when we implement this. const ROT_STAGING_DEVEL_SIGN: &str = "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf"; @@ -1051,13 +1051,12 @@ mod test { ArtifactKind::GIMLET_ROT_IMAGE_A, test_artifact_for_board("oxide-rot-1"), ), - // TODO-K: Does this work as expected? make_artifact( "oxide-rot-1", ArtifactKind::GIMLET_ROT_IMAGE_B, test_artifact_for_board("oxide-rot-1"), ), - // Make more artifacts for other artifact kinds and for other slots? + // TODO-K: Make more artifacts for other RoT artifact kinds ]; TufRepoDescription { @@ -1547,11 +1546,6 @@ mod test { }) .collect(); - // //debug - // if expected_updates.len() != 10000 { - // panic!("num of expected updates:{} {expected_updates:?}", expected_updates.len()); - // } - for _ in 0..expected_updates.len() { // Generate an inventory collection reflecting that everything is at // version 1 except for what we've already updated. From 413ade6e8f6484caad39978798b3aa9cabf21279 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 17:09:05 +1200 Subject: [PATCH 22/42] clean up --- .../planning/src/mgs_updates/mod.rs | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 911f6a1bdac..878f7fc68de 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1261,8 +1261,9 @@ mod test { ); assert!(updates.is_empty()); - // Test that when a TUF repo is specified and one RoT is outdated, then - // it's configured with an update (and the update looks correct). + // Test that when a TUF repo is specified and one RoT and SP are + // outdated, then it's configured with an update (and the update looks + // correct). let repo = make_tuf_repo(); let updates = plan_mgs_updates( log, @@ -1277,7 +1278,6 @@ mod test { assert_eq!(first_update.baseboard_id.serial_number, "sled_0"); assert_eq!(first_update.sp_type, SpType::Sled); assert_eq!(first_update.slot_id, 0); - // assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_SP_GIMLET_D); assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_OXIDE_ROT_1); assert_eq!(first_update.artifact_version, ARTIFACT_VERSION_2); @@ -1294,9 +1294,9 @@ mod test { ); assert_eq!(updates, later_updates); - // Test that when two updates are needed, but one is already pending, - // then the other one is *not* started (because it exceeds - // nmax_updates). + // Test that when two updates for two SpTypes are needed, but one is + // already pending, then the other one is *not* started (because it + // exceeds nmax_updates). let later_collection = make_collection( ARTIFACT_VERSION_2, &BTreeMap::from([ @@ -1321,7 +1321,7 @@ mod test { ); assert_eq!(updates, later_updates); - // At this point, we're ready to test that when the first update + // At this point, we're ready to test that when the first SpType update // completes, then the second one *is* started. This tests two // different things: first that we noticed the first one completed, and // second that we noticed another thing needed an update @@ -1349,7 +1349,6 @@ mod test { assert_eq!(next_update.sp_type, SpType::Switch); assert_eq!(next_update.slot_id, 1); assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_OXIDE_ROT_1); - //assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); // Finally, test that when all RoTs and SPs are in spec, then no updates @@ -1407,10 +1406,6 @@ mod test { // Verify the precondition details of an ordinary update. let old_update = updates.into_iter().next().expect("at least one update"); - //let PendingMgsUpdateDetails::Sp { - // expected_active_version: old_expected_active_version, - // expected_inactive_version: old_expected_inactive_version, - //} = &old_update.details let PendingMgsUpdateDetails::Rot { expected_active_slot: old_expected_active_slot, expected_inactive_version: old_expected_inactive_version, @@ -1452,10 +1447,6 @@ mod test { assert_eq!(old_update.slot_id, new_update.slot_id); assert_eq!(old_update.artifact_hash, new_update.artifact_hash); assert_eq!(old_update.artifact_version, new_update.artifact_version); - // let PendingMgsUpdateDetails::Sp { - // expected_active_version: new_expected_active_version, - // expected_inactive_version: new_expected_inactive_version, - // } = &new_update.details let PendingMgsUpdateDetails::Rot { expected_active_slot: new_expected_active_slot, expected_inactive_version: new_expected_inactive_version, From 6899baad0871d21986170778a8057919846bc6d4 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 17:55:22 +1200 Subject: [PATCH 23/42] clean up --- .../planning/src/mgs_updates/mod.rs | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 878f7fc68de..7cdd6871168 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1489,10 +1489,6 @@ mod test { assert_eq!(old_update.slot_id, new_update.slot_id); assert_eq!(old_update.artifact_hash, new_update.artifact_hash); assert_eq!(old_update.artifact_version, new_update.artifact_version); - //let PendingMgsUpdateDetails::Sp { - // expected_active_version: new_expected_active_version, - // expected_inactive_version: new_expected_inactive_version, - //} = &new_update.details let PendingMgsUpdateDetails::Rot { expected_active_slot: new_expected_active_slot, expected_inactive_version: new_expected_inactive_version, @@ -1524,7 +1520,7 @@ mod test { // Maintain a map of SPs and RoTs that we've updated. We'll use this to // configure the inventory collection that we create at each step. - let mut exceptions = BTreeMap::new(); + let mut sp_exceptions = BTreeMap::new(); let mut rot_exceptions = BTreeMap::new(); // We do not control the order of updates. But we expect to update each @@ -1542,7 +1538,7 @@ mod test { // version 1 except for what we've already updated. let collection = make_collection( ARTIFACT_VERSION_1, - &exceptions, + &sp_exceptions, ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_1, &rot_exceptions, @@ -1581,7 +1577,7 @@ mod test { }, PendingMgsUpdateDetails::Sp { .. } => { assert!( - exceptions + sp_exceptions .insert((sp_type, sp_slot), ARTIFACT_VERSION_2) .is_none() ); @@ -1595,7 +1591,7 @@ mod test { // Take one more lap. It should reflect zero updates. let collection = make_collection( ARTIFACT_VERSION_1, - &exceptions, + &sp_exceptions, ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_1, &rot_exceptions, @@ -1697,10 +1693,6 @@ mod test { PendingMgsUpdateDetails::Sp { .. } => SpComponent::Sp, }; println!("found update: {} slot {}", sp_type, sp_slot); - println!( - "DEBUG: REMOVE type: {} slot: {} component {:?}", - sp_type, sp_slot, sp_component - ); let (expected_serial, expected_artifact) = expected_updates .remove(&(sp_type, sp_slot, sp_component)) .expect("unexpected update"); @@ -1721,14 +1713,6 @@ mod test { } => (expected_active_version, expected_inactive_version), PendingMgsUpdateDetails::RotBootloader { .. } => unimplemented!(), }; - // let PendingMgsUpdateDetails::Sp { - // expected_active_version, - // expected_inactive_version, - // } = &update.details - // else { - // panic!("expected SP update"); - // }; - // Verify update against SpComponent? assert_eq!(*expected_active_version, ARTIFACT_VERSION_1); assert_eq!(*expected_inactive_version, ExpectedVersion::NoValidVersion); } From 4013357b924775043eb25fa5a9c6e84f52fad118 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 11 Jul 2025 18:04:01 +1200 Subject: [PATCH 24/42] clippy and fmt --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 7cdd6871168..1fdc981fab2 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1080,7 +1080,7 @@ mod test { id: ArtifactId { name: name.to_string(), version: ARTIFACT_VERSION_2, - kind: kind, + kind, }, hash, size: 0, // unused here @@ -1574,15 +1574,17 @@ mod test { .insert((sp_type, sp_slot), ARTIFACT_VERSION_2) .is_none() ); - }, + } PendingMgsUpdateDetails::Sp { .. } => { assert!( sp_exceptions .insert((sp_type, sp_slot), ARTIFACT_VERSION_2) .is_none() ); - }, - PendingMgsUpdateDetails::RotBootloader { .. } => unimplemented!() + } + PendingMgsUpdateDetails::RotBootloader { .. } => { + unimplemented!() + } } latest_updates = new_updates; @@ -1648,7 +1650,7 @@ mod test { // SP and RoT for the same SpType count as a sinlge update. For // `expected_updates`, each component update counts as an update, so the // amount of `all_updates` should be half of `expected_updates`. - assert_eq!(all_updates.len(), expected_updates.len()/2); + assert_eq!(all_updates.len(), expected_updates.len() / 2); for update in &all_updates { verify_one_sp_update(&mut expected_updates, update); } From 4abbd8edece23062f68af20adcd385cb6a2647b5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 16 Jul 2025 17:10:08 +1200 Subject: [PATCH 25/42] expectorate --- .../tests/output/cmds-target-release-stdout | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 324a5303fd7..48fe56cd026 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -760,9 +760,9 @@ set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds > blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -956,9 +956,9 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot b -> 0.5.0 generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds > blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1150,9 +1150,9 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot a -> 1.0.0 generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds > blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1344,9 +1344,9 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: inactive -> 0.5.0 generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds > blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1538,9 +1538,9 @@ set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: active -> 1.0.0 generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds > blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1734,9 +1734,9 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot a -> 1.0.0 generated inventory collection 39363465-89ae-4ac2-9be1-099068da9d45 from configured sleds > blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 @@ -1928,9 +1928,9 @@ set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 SP versions: active -> 1.0.0 generated inventory collection 04bc9001-0836-4fec-b9cb-9d4760caf8b4 from configured sleds > blueprint-plan 626487fa-7139-45ec-8416-902271fc730b 04bc9001-0836-4fec-b9cb-9d4760caf8b4 -INFO noop converting 0/9 install-dataset zones to artifact store, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 -INFO noop converting 0/8 install-dataset zones to artifact store, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763 +INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 +INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 From 91fea61a597cd56497d02df551cd7fe49e5d223a Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 17 Jul 2025 20:56:56 +1200 Subject: [PATCH 26/42] why did I change this in the first place? --- nexus/reconfigurator/planning/src/system.rs | 42 +++++++++------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 3b794c168b3..79ae1ddd797 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -1142,13 +1142,13 @@ impl Sled { policy, state: SledState::Active, resources: SledResources { subnet: sled_subnet, zpools }, - sp_active_caboose: Some(Arc::new( - Self::default_sp_component_caboose(String::from("0.0.1")), - )), + sp_active_caboose: Some(Arc::new(Self::default_sp_caboose( + String::from("0.0.1"), + ))), sp_inactive_caboose: None, - rot_slot_a_caboose: Some(Arc::new( - Self::default_rot_component_caboose(String::from("0.0.2")), - )), + rot_slot_a_caboose: Some(Arc::new(Self::default_rot_caboose( + String::from("0.0.2"), + ))), rot_slot_b_caboose: None, } } @@ -1372,7 +1372,7 @@ impl Sled { Arc::make_mut(caboose).version = active_version.to_string() } new @ None => { - *new = Some(Arc::new(Self::default_sp_component_caboose( + *new = Some(Arc::new(Self::default_sp_caboose( active_version.to_string(), ))); } @@ -1390,11 +1390,9 @@ impl Sled { Arc::make_mut(caboose).version = v.to_string() } new @ None => { - *new = Some(Arc::new( - Self::default_sp_component_caboose( - v.to_string(), - ), - )); + *new = Some(Arc::new(Self::default_sp_caboose( + v.to_string(), + ))); } } } @@ -1422,11 +1420,9 @@ impl Sled { Arc::make_mut(caboose).version = v.to_string() } new @ None => { - *new = Some(Arc::new( - Self::default_rot_component_caboose( - v.to_string(), - ), - )); + *new = Some(Arc::new(Self::default_rot_caboose( + v.to_string(), + ))); } } } @@ -1444,11 +1440,9 @@ impl Sled { Arc::make_mut(caboose).version = v.to_string() } new @ None => { - *new = Some(Arc::new( - Self::default_rot_component_caboose( - v.to_string(), - ), - )); + *new = Some(Arc::new(Self::default_rot_caboose( + v.to_string(), + ))); } } } @@ -1456,7 +1450,7 @@ impl Sled { } } - fn default_sp_component_caboose(version: String) -> Caboose { + fn default_sp_caboose(version: String) -> Caboose { let board = sp_sim::SIM_GIMLET_BOARD.to_string(); Caboose { board: board.clone(), @@ -1467,7 +1461,7 @@ impl Sled { } } - fn default_rot_component_caboose(version: String) -> Caboose { + fn default_rot_caboose(version: String) -> Caboose { let board = sp_sim::SIM_ROT_BOARD.to_string(); Caboose { board: board.clone(), From 370bcebb2bef4650e522afb449965e49f900bd6d Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 17 Jul 2025 21:00:27 +1200 Subject: [PATCH 27/42] add some more tests --- .../reconfigurator-cli/tests/input/cmds.txt | 12 + .../tests/output/cmds-stderr | 7 + .../tests/output/cmds-stdout | 217 +++++++++++++++++- 3 files changed, 224 insertions(+), 12 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds.txt b/dev-tools/reconfigurator-cli/tests/input/cmds.txt index 671a89303b1..9bee6a18ac7 100644 --- a/dev-tools/reconfigurator-cli/tests/input/cmds.txt +++ b/dev-tools/reconfigurator-cli/tests/input/cmds.txt @@ -13,6 +13,18 @@ sled-add 90c1102a-b9f5-4d88-92a2-60d54a2d98cc sled-add 04ef3330-c682-4a08-8def-fcc4bef31bcd --policy non-provisionable sled-list +sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 1.0.0 +sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-b 2.0.0 +sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 3.0.0 +sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 4.0.0 --slot-b invalid +sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 4.0.0 --slot-b 5.0.0 +sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a + sled-update-sp dde1c0e2-b10d-4621-b420-f179f7a7a00a sled-update-sp dde1c0e2-b10d-4621-b420-f179f7a7a00a --active 1.0.0 sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stderr b/dev-tools/reconfigurator-cli/tests/output/cmds-stderr index cf184b190ab..8acb3b51a50 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stderr +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stderr @@ -1,3 +1,10 @@ +error: the following required arguments were not provided: + --slot-a + --slot-b + +Usage: sled-update-rot --slot-a --slot-b + +For more information, try '--help'. error: the following required arguments were not provided: --active --inactive diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout index 9ada230083e..82ea08cbd30 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout @@ -72,6 +72,199 @@ ID SERIAL NZPOOLS SUBNET dde1c0e2-b10d-4621-b420-f179f7a7a00a serial0 10 fd00:1122:3344:101::/64 +> sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a + +> sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 1.0.0 +set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 1.0.0 + +> sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) +serial serial0 +subnet fd00:1122:3344:101::/64 +SP active version: Some("0.0.1") +SP inactive version: None +RoT active slot: A +RoT slot A version: Some("1.0.0") +RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None +zpools (10): + 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } + 677dd944-6761-4a89-8606-4d7fe485a63c (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-677dd944-6761-4a89-8606-4d7fe485a63c" }, disk_id: fcf54220-3ff4-463e-b4a2-58447f51b68c (physical_disk), policy: InService, state: Active } + 70e81eac-6ed4-4c2d-b16a-fabe2aec56fc (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-70e81eac-6ed4-4c2d-b16a-fabe2aec56fc" }, disk_id: 42643377-e4d1-41a0-ac32-38d6e56cb22a (physical_disk), policy: InService, state: Active } + 7b26c659-bf8f-4c60-ab75-fd2dd8ef5866 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-7b26c659-bf8f-4c60-ab75-fd2dd8ef5866" }, disk_id: 5b2df08c-ea6a-4771-8363-80031249c97b (physical_disk), policy: InService, state: Active } + 8e0008d0-9313-4caf-bc20-305ccce29846 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8e0008d0-9313-4caf-bc20-305ccce29846" }, disk_id: 0f5e7fc1-8d87-45f8-a00e-f5127b7a3905 (physical_disk), policy: InService, state: Active } + 929e328a-dd25-447d-9af7-6e2216adf4aa (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-929e328a-dd25-447d-9af7-6e2216adf4aa" }, disk_id: f62e3201-e89b-4667-9707-e49f86b9df07 (physical_disk), policy: InService, state: Active } + 9a25ff89-5446-4233-bf58-20a24c80aa58 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9a25ff89-5446-4233-bf58-20a24c80aa58" }, disk_id: 49b87668-e08b-4939-91f7-a82612e2ebff (physical_disk), policy: InService, state: Active } + a9cd1fe6-f1ba-4227-bff7-978992c3d6ad (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-a9cd1fe6-f1ba-4227-bff7-978992c3d6ad" }, disk_id: dcde393a-3ac6-4e98-8833-012787e73e15 (physical_disk), policy: InService, state: Active } + b3ede1e1-3264-4b21-8c7d-9ea5d3715210 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-b3ede1e1-3264-4b21-8c7d-9ea5d3715210" }, disk_id: 4863117c-b77d-4dbc-996d-d18ddf0f5ff7 (physical_disk), policy: InService, state: Active } + e0f5c287-3296-4a35-b597-7452283ff329 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-e0f5c287-3296-4a35-b597-7452283ff329" }, disk_id: 0f13d3dd-1830-4a06-b664-e6f0473ba704 (physical_disk), policy: InService, state: Active } + + +> sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-b 2.0.0 +set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot b -> 2.0.0 + +> sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) +serial serial0 +subnet fd00:1122:3344:101::/64 +SP active version: Some("0.0.1") +SP inactive version: None +RoT active slot: A +RoT slot A version: Some("1.0.0") +RoT slot B version: Some("2.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None +zpools (10): + 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } + 677dd944-6761-4a89-8606-4d7fe485a63c (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-677dd944-6761-4a89-8606-4d7fe485a63c" }, disk_id: fcf54220-3ff4-463e-b4a2-58447f51b68c (physical_disk), policy: InService, state: Active } + 70e81eac-6ed4-4c2d-b16a-fabe2aec56fc (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-70e81eac-6ed4-4c2d-b16a-fabe2aec56fc" }, disk_id: 42643377-e4d1-41a0-ac32-38d6e56cb22a (physical_disk), policy: InService, state: Active } + 7b26c659-bf8f-4c60-ab75-fd2dd8ef5866 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-7b26c659-bf8f-4c60-ab75-fd2dd8ef5866" }, disk_id: 5b2df08c-ea6a-4771-8363-80031249c97b (physical_disk), policy: InService, state: Active } + 8e0008d0-9313-4caf-bc20-305ccce29846 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8e0008d0-9313-4caf-bc20-305ccce29846" }, disk_id: 0f5e7fc1-8d87-45f8-a00e-f5127b7a3905 (physical_disk), policy: InService, state: Active } + 929e328a-dd25-447d-9af7-6e2216adf4aa (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-929e328a-dd25-447d-9af7-6e2216adf4aa" }, disk_id: f62e3201-e89b-4667-9707-e49f86b9df07 (physical_disk), policy: InService, state: Active } + 9a25ff89-5446-4233-bf58-20a24c80aa58 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9a25ff89-5446-4233-bf58-20a24c80aa58" }, disk_id: 49b87668-e08b-4939-91f7-a82612e2ebff (physical_disk), policy: InService, state: Active } + a9cd1fe6-f1ba-4227-bff7-978992c3d6ad (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-a9cd1fe6-f1ba-4227-bff7-978992c3d6ad" }, disk_id: dcde393a-3ac6-4e98-8833-012787e73e15 (physical_disk), policy: InService, state: Active } + b3ede1e1-3264-4b21-8c7d-9ea5d3715210 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-b3ede1e1-3264-4b21-8c7d-9ea5d3715210" }, disk_id: 4863117c-b77d-4dbc-996d-d18ddf0f5ff7 (physical_disk), policy: InService, state: Active } + e0f5c287-3296-4a35-b597-7452283ff329 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-e0f5c287-3296-4a35-b597-7452283ff329" }, disk_id: 0f13d3dd-1830-4a06-b664-e6f0473ba704 (physical_disk), policy: InService, state: Active } + + +> sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 3.0.0 +set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 3.0.0 + +> sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) +serial serial0 +subnet fd00:1122:3344:101::/64 +SP active version: Some("0.0.1") +SP inactive version: None +RoT active slot: A +RoT slot A version: Some("3.0.0") +RoT slot B version: Some("2.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None +zpools (10): + 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } + 677dd944-6761-4a89-8606-4d7fe485a63c (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-677dd944-6761-4a89-8606-4d7fe485a63c" }, disk_id: fcf54220-3ff4-463e-b4a2-58447f51b68c (physical_disk), policy: InService, state: Active } + 70e81eac-6ed4-4c2d-b16a-fabe2aec56fc (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-70e81eac-6ed4-4c2d-b16a-fabe2aec56fc" }, disk_id: 42643377-e4d1-41a0-ac32-38d6e56cb22a (physical_disk), policy: InService, state: Active } + 7b26c659-bf8f-4c60-ab75-fd2dd8ef5866 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-7b26c659-bf8f-4c60-ab75-fd2dd8ef5866" }, disk_id: 5b2df08c-ea6a-4771-8363-80031249c97b (physical_disk), policy: InService, state: Active } + 8e0008d0-9313-4caf-bc20-305ccce29846 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8e0008d0-9313-4caf-bc20-305ccce29846" }, disk_id: 0f5e7fc1-8d87-45f8-a00e-f5127b7a3905 (physical_disk), policy: InService, state: Active } + 929e328a-dd25-447d-9af7-6e2216adf4aa (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-929e328a-dd25-447d-9af7-6e2216adf4aa" }, disk_id: f62e3201-e89b-4667-9707-e49f86b9df07 (physical_disk), policy: InService, state: Active } + 9a25ff89-5446-4233-bf58-20a24c80aa58 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9a25ff89-5446-4233-bf58-20a24c80aa58" }, disk_id: 49b87668-e08b-4939-91f7-a82612e2ebff (physical_disk), policy: InService, state: Active } + a9cd1fe6-f1ba-4227-bff7-978992c3d6ad (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-a9cd1fe6-f1ba-4227-bff7-978992c3d6ad" }, disk_id: dcde393a-3ac6-4e98-8833-012787e73e15 (physical_disk), policy: InService, state: Active } + b3ede1e1-3264-4b21-8c7d-9ea5d3715210 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-b3ede1e1-3264-4b21-8c7d-9ea5d3715210" }, disk_id: 4863117c-b77d-4dbc-996d-d18ddf0f5ff7 (physical_disk), policy: InService, state: Active } + e0f5c287-3296-4a35-b597-7452283ff329 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-e0f5c287-3296-4a35-b597-7452283ff329" }, disk_id: 0f13d3dd-1830-4a06-b664-e6f0473ba704 (physical_disk), policy: InService, state: Active } + + +> sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 4.0.0 --slot-b invalid +set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slot b -> invalid + +> sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) +serial serial0 +subnet fd00:1122:3344:101::/64 +SP active version: Some("0.0.1") +SP inactive version: None +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: None +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None +zpools (10): + 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } + 677dd944-6761-4a89-8606-4d7fe485a63c (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-677dd944-6761-4a89-8606-4d7fe485a63c" }, disk_id: fcf54220-3ff4-463e-b4a2-58447f51b68c (physical_disk), policy: InService, state: Active } + 70e81eac-6ed4-4c2d-b16a-fabe2aec56fc (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-70e81eac-6ed4-4c2d-b16a-fabe2aec56fc" }, disk_id: 42643377-e4d1-41a0-ac32-38d6e56cb22a (physical_disk), policy: InService, state: Active } + 7b26c659-bf8f-4c60-ab75-fd2dd8ef5866 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-7b26c659-bf8f-4c60-ab75-fd2dd8ef5866" }, disk_id: 5b2df08c-ea6a-4771-8363-80031249c97b (physical_disk), policy: InService, state: Active } + 8e0008d0-9313-4caf-bc20-305ccce29846 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8e0008d0-9313-4caf-bc20-305ccce29846" }, disk_id: 0f5e7fc1-8d87-45f8-a00e-f5127b7a3905 (physical_disk), policy: InService, state: Active } + 929e328a-dd25-447d-9af7-6e2216adf4aa (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-929e328a-dd25-447d-9af7-6e2216adf4aa" }, disk_id: f62e3201-e89b-4667-9707-e49f86b9df07 (physical_disk), policy: InService, state: Active } + 9a25ff89-5446-4233-bf58-20a24c80aa58 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9a25ff89-5446-4233-bf58-20a24c80aa58" }, disk_id: 49b87668-e08b-4939-91f7-a82612e2ebff (physical_disk), policy: InService, state: Active } + a9cd1fe6-f1ba-4227-bff7-978992c3d6ad (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-a9cd1fe6-f1ba-4227-bff7-978992c3d6ad" }, disk_id: dcde393a-3ac6-4e98-8833-012787e73e15 (physical_disk), policy: InService, state: Active } + b3ede1e1-3264-4b21-8c7d-9ea5d3715210 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-b3ede1e1-3264-4b21-8c7d-9ea5d3715210" }, disk_id: 4863117c-b77d-4dbc-996d-d18ddf0f5ff7 (physical_disk), policy: InService, state: Active } + e0f5c287-3296-4a35-b597-7452283ff329 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-e0f5c287-3296-4a35-b597-7452283ff329" }, disk_id: 0f13d3dd-1830-4a06-b664-e6f0473ba704 (physical_disk), policy: InService, state: Active } + + +> sled-update-rot dde1c0e2-b10d-4621-b420-f179f7a7a00a --slot-a 4.0.0 --slot-b 5.0.0 +set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slot b -> 5.0.0 + +> sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a +sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) +serial serial0 +subnet fd00:1122:3344:101::/64 +SP active version: Some("0.0.1") +SP inactive version: None +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None +zpools (10): + 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } + 677dd944-6761-4a89-8606-4d7fe485a63c (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-677dd944-6761-4a89-8606-4d7fe485a63c" }, disk_id: fcf54220-3ff4-463e-b4a2-58447f51b68c (physical_disk), policy: InService, state: Active } + 70e81eac-6ed4-4c2d-b16a-fabe2aec56fc (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-70e81eac-6ed4-4c2d-b16a-fabe2aec56fc" }, disk_id: 42643377-e4d1-41a0-ac32-38d6e56cb22a (physical_disk), policy: InService, state: Active } + 7b26c659-bf8f-4c60-ab75-fd2dd8ef5866 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-7b26c659-bf8f-4c60-ab75-fd2dd8ef5866" }, disk_id: 5b2df08c-ea6a-4771-8363-80031249c97b (physical_disk), policy: InService, state: Active } + 8e0008d0-9313-4caf-bc20-305ccce29846 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8e0008d0-9313-4caf-bc20-305ccce29846" }, disk_id: 0f5e7fc1-8d87-45f8-a00e-f5127b7a3905 (physical_disk), policy: InService, state: Active } + 929e328a-dd25-447d-9af7-6e2216adf4aa (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-929e328a-dd25-447d-9af7-6e2216adf4aa" }, disk_id: f62e3201-e89b-4667-9707-e49f86b9df07 (physical_disk), policy: InService, state: Active } + 9a25ff89-5446-4233-bf58-20a24c80aa58 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9a25ff89-5446-4233-bf58-20a24c80aa58" }, disk_id: 49b87668-e08b-4939-91f7-a82612e2ebff (physical_disk), policy: InService, state: Active } + a9cd1fe6-f1ba-4227-bff7-978992c3d6ad (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-a9cd1fe6-f1ba-4227-bff7-978992c3d6ad" }, disk_id: dcde393a-3ac6-4e98-8833-012787e73e15 (physical_disk), policy: InService, state: Active } + b3ede1e1-3264-4b21-8c7d-9ea5d3715210 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-b3ede1e1-3264-4b21-8c7d-9ea5d3715210" }, disk_id: 4863117c-b77d-4dbc-996d-d18ddf0f5ff7 (physical_disk), policy: InService, state: Active } + e0f5c287-3296-4a35-b597-7452283ff329 (zpool) + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-e0f5c287-3296-4a35-b597-7452283ff329" }, disk_id: 0f13d3dd-1830-4a06-b664-e6f0473ba704 (physical_disk), policy: InService, state: Active } + + + > sled-update-sp dde1c0e2-b10d-4621-b420-f179f7a7a00a > sled-update-sp dde1c0e2-b10d-4621-b420-f179f7a7a00a --active 1.0.0 @@ -84,8 +277,8 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("1.0.0") SP inactive version: None RoT active slot: A -RoT slot A version: Some("0.0.2") -RoT slot B version: None +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") RoT persistent boot preference: A RoT pending persistent boot preference: None RoT transient boot preference: None @@ -122,8 +315,8 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("1.0.0") SP inactive version: Some("2.0.0") RoT active slot: A -RoT slot A version: Some("0.0.2") -RoT slot B version: None +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") RoT persistent boot preference: A RoT pending persistent boot preference: None RoT transient boot preference: None @@ -160,8 +353,8 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("3.0.0") SP inactive version: Some("2.0.0") RoT active slot: A -RoT slot A version: Some("0.0.2") -RoT slot B version: None +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") RoT persistent boot preference: A RoT pending persistent boot preference: None RoT transient boot preference: None @@ -198,8 +391,8 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: None RoT active slot: A -RoT slot A version: Some("0.0.2") -RoT slot B version: None +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") RoT persistent boot preference: A RoT pending persistent boot preference: None RoT transient boot preference: None @@ -236,8 +429,8 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") RoT active slot: A -RoT slot A version: Some("0.0.2") -RoT slot B version: None +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") RoT persistent boot preference: A RoT pending persistent boot preference: None RoT transient boot preference: None @@ -306,8 +499,8 @@ subnet fd00:1122:3344:101::/64 SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") RoT active slot: A -RoT slot A version: Some("0.0.2") -RoT slot B version: None +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") RoT persistent boot preference: A RoT pending persistent boot preference: None RoT transient boot preference: None From cd9cea313a0c78341b297dd4a3100913633628e9 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 22 Jul 2025 13:54:14 +1200 Subject: [PATCH 28/42] fixes after merging main --- dev-tools/reconfigurator-cli/src/lib.rs | 1 + .../tests/output/cmds-example-stdout | 15 +++++++++------ .../tests/output/cmds-mupdate-update-flow-stdout | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index 81ec72ce0df..3597eaeb9ec 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -521,6 +521,7 @@ struct SledUpdateRotArgs { slot_b: Option, } +#[derive(Debug, Args)] struct SledSetMupdateOverrideArgs { #[clap(flatten)] source: SledMupdateOverrideSource, diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index f8f298adfee..2cc8a5bc254 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -1043,8 +1043,9 @@ Sled serial0 A 0101010101010101010101010101010101010101010101010101010101010101 B 0202020202020202020202020202020202020202020202020202020202020202 cabooses: - SLOT BOARD NAME VERSION GIT_COMMIT SIGN - SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + SLOT BOARD NAME VERSION GIT_COMMIT SIGN + SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + RotSlotA SimRot SimRot 0.0.2 unknown n/a RoT pages: SLOT DATA_BASE64 RoT: active slot: slot A @@ -1065,8 +1066,9 @@ Sled serial1 A 0101010101010101010101010101010101010101010101010101010101010101 B 0202020202020202020202020202020202020202020202020202020202020202 cabooses: - SLOT BOARD NAME VERSION GIT_COMMIT SIGN - SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + SLOT BOARD NAME VERSION GIT_COMMIT SIGN + SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + RotSlotA SimRot SimRot 0.0.2 unknown n/a RoT pages: SLOT DATA_BASE64 RoT: active slot: slot A @@ -1087,8 +1089,9 @@ Sled serial2 A 0101010101010101010101010101010101010101010101010101010101010101 B 0202020202020202020202020202020202020202020202020202020202020202 cabooses: - SLOT BOARD NAME VERSION GIT_COMMIT SIGN - SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + SLOT BOARD NAME VERSION GIT_COMMIT SIGN + SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + RotSlotA SimRot SimRot 0.0.2 unknown n/a RoT pages: SLOT DATA_BASE64 RoT: active slot: slot A diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index 80c88c1edfc..c69d65f6c8d 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -18,8 +18,8 @@ created repo-1.0.0.zip for system version 1.0.0 INFO extracting uploaded archive to INFO created directory to store extracted artifacts, path: INFO added artifact, name: SimGimletSp, kind: gimlet_sp, version: 1.0.0, hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, length: 747 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 INFO added artifact, name: fake-gimlet-rot-bootloader, kind: gimlet_rot_bootloader, version: 1.0.0, hash: 005ea358f1cd316df42465b1e3a0334ea22cc0c0442cf9ddf9b42fbf49780236, length: 750 INFO added artifact, name: fake-host, kind: host_phase_1, version: 1.0.0, hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, length: 524288 INFO added artifact, name: fake-host, kind: host_phase_2, version: 1.0.0, hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, length: 1048576 From b71f5b8cac3fd7c82572063a968223348051307f Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 23 Jul 2025 13:52:47 +1200 Subject: [PATCH 29/42] small fixes --- .../planning/src/mgs_updates/mod.rs | 66 +++++-------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 1fdc981fab2..7bb378666e5 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -356,50 +356,10 @@ fn mgs_update_status_sp( // still matches what we saw when we configured this update. If not, then // this update cannot proceed as currently configured. It will fail its // precondition check. - // - // This logic is more complex than for the active slot because unlike the - // active slot, it's possible for both the found contents and the expected - // contents to be missing and that's not necessarily an error. - match (found_inactive_version, expected_inactive_version) { - (Some(_), ExpectedVersion::NoValidVersion) => { - // We expected nothing in the inactive slot, but found something. - MgsUpdateStatus::Impossible - } - (Some(found), ExpectedVersion::Version(expected)) => { - if found == expected.as_str() { - // We found something in the inactive slot that matches what we - // expected. - MgsUpdateStatus::NotDone - } else { - // We found something in the inactive slot that differs from - // what we expected. - MgsUpdateStatus::Impossible - } - } - (None, ExpectedVersion::Version(_)) => { - // We expected something in the inactive slot, but found nothing. - // This case is tricky because we can't tell from the inventory - // whether we transiently failed to fetch the caboose for some - // reason or whether the caboose is actually garbage. We choose to - // assume that it's actually garbage, which would mean that this - // update as-configured is impossible. This will cause us to - // generate a new update that expects garbage in the inactive slot. - // If we're right, great. If we're wrong, then *that* update will - // be impossible to complete, but we should fix this again if the - // transient error goes away. - // - // If we instead assumed that this was a transient error, we'd do - // nothing here instead. But if the caboose was really missing, - // then we'd get stuck forever waiting for something that would - // never happen. - MgsUpdateStatus::Impossible - } - (None, ExpectedVersion::NoValidVersion) => { - // We expected nothing in the inactive slot and found nothing there. - // No problem! - MgsUpdateStatus::NotDone - } - } + mgs_update_status_inactive_versions( + found_inactive_version, + expected_inactive_version, + ) } struct RotUpdateState { @@ -468,7 +428,16 @@ fn mgs_update_status_rot( // still matches what we saw when we configured this update. If not, then // this update cannot proceed as currently configured. It will fail its // precondition check. - // + mgs_update_status_inactive_versions( + found_inactive_version, + expected_inactive_version, + ) +} + +fn mgs_update_status_inactive_versions( + found_inactive_version: Option<&str>, + expected_inactive_version: &ExpectedVersion, +) -> MgsUpdateStatus { // This logic is more complex than for the active slot because unlike the // active slot, it's possible for both the found contents and the expected // contents to be missing and that's not necessarily an error. @@ -733,6 +702,7 @@ fn try_make_update_rot( // // - "name" matching the board name (found above from caboose) // - "kind" matching one of the known RoT kinds + // - "rkth" verified against the CMPA/CFPA found in inventory if a.id.name != *board { return false; @@ -779,9 +749,9 @@ fn try_make_update_rot( if matching_artifacts.len() > 1 { // This should be impossible unless we shipped a TUF repo with more - // than 1 artifact for the same board and slot that verifies against the - // RoT's CMPA/CFPA. But it doesn't prevent us from picking one and - // proceeding. Make a note and proceed. + // than 1 artifact for the same board and root key table hash (RKTH) + // that can be verified afgainst the RoT's CMPA/CFPA. But it doesn't + // prevent us from picking one and proceeding. Make a note and proceed. error!(log, "found more than one matching artifact for RoT update"); } From 6c75bc6418a4f4feffa38a1e56a727466cf7a6e6 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 28 Jul 2025 19:53:37 +1200 Subject: [PATCH 30/42] fixes after merge --- .../tests/output/cmds-example-stdout | 21 +++++----- .../output/cmds-mupdate-update-flow-stdout | 2 +- .../output/cmds-noop-image-source-stdout | 2 +- .../tests/output/cmds-stdout | 40 +++++++++++++++++++ .../tests/output/cmds-target-release-stdout | 6 +-- 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index b9fbd57f022..b22027f7cca 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -1049,9 +1049,10 @@ Sled serial0 A 0101010101010101010101010101010101010101010101010101010101010101 B 0202020202020202020202020202020202020202020202020202020202020202 cabooses: - SLOT BOARD NAME VERSION GIT_COMMIT SIGN - SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a - RotSlotA SimRot SimRot 0.0.2 unknown n/a + SLOT BOARD NAME VERSION GIT_COMMIT SIGN + SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + RotSlotA SimRot SimRot 0.0.2 unknown n/a + Stage0 SimRotStage0 SimRotStage0 0.0.1 unknown n/a RoT pages: SLOT DATA_BASE64 RoT: active slot: slot A @@ -1072,9 +1073,10 @@ Sled serial1 A 0101010101010101010101010101010101010101010101010101010101010101 B 0202020202020202020202020202020202020202020202020202020202020202 cabooses: - SLOT BOARD NAME VERSION GIT_COMMIT SIGN - SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a - RotSlotA SimRot SimRot 0.0.2 unknown n/a + SLOT BOARD NAME VERSION GIT_COMMIT SIGN + SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + RotSlotA SimRot SimRot 0.0.2 unknown n/a + Stage0 SimRotStage0 SimRotStage0 0.0.1 unknown n/a RoT pages: SLOT DATA_BASE64 RoT: active slot: slot A @@ -1095,9 +1097,10 @@ Sled serial2 A 0101010101010101010101010101010101010101010101010101010101010101 B 0202020202020202020202020202020202020202020202020202020202020202 cabooses: - SLOT BOARD NAME VERSION GIT_COMMIT SIGN - SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a - RotSlotA SimRot SimRot 0.0.2 unknown n/a + SLOT BOARD NAME VERSION GIT_COMMIT SIGN + SpSlot0 SimGimletSp SimGimletSp 0.0.1 unknown n/a + RotSlotA SimRot SimRot 0.0.2 unknown n/a + Stage0 SimRotStage0 SimRotStage0 0.0.1 unknown n/a RoT pages: SLOT DATA_BASE64 RoT: active slot: slot A diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index 8c69fe16dca..98cfe6d7608 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -20,7 +20,7 @@ INFO created directory to store extracted artifacts, path: 1.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None RoT active slot: A @@ -121,6 +123,8 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot b -> 2.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None RoT active slot: A @@ -159,6 +163,8 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 3.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None RoT active slot: A @@ -197,6 +203,8 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slo sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None RoT active slot: A @@ -235,6 +243,8 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slo sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None RoT active slot: A @@ -440,6 +450,12 @@ RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -477,6 +493,12 @@ RoT bootloader stage 0 version: Some("1.0.0") RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -511,6 +533,12 @@ RoT bootloader stage 0 version: Some("1.0.0") RoT bootloader stage 0 next version: Some("2.0.0") SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -545,6 +573,12 @@ RoT bootloader stage 0 version: Some("3.0.0") RoT bootloader stage 0 next version: Some("2.0.0") SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } @@ -579,6 +613,12 @@ RoT bootloader stage 0 version: Some("4.0.0") RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT active slot: A +RoT slot A version: Some("4.0.0") +RoT slot B version: Some("5.0.0") +RoT persistent boot preference: A +RoT pending persistent boot preference: None +RoT transient boot preference: None zpools (10): 674c6591-11be-44f2-9df1-db3bb663ec01 (zpool) SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-674c6591-11be-44f2-9df1-db3bb663ec01" }, disk_id: a52a7c57-7fd0-4139-8293-bda299523c53 (physical_disk), policy: InService, state: Active } diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 3195255912a..69e55613404 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -35,7 +35,7 @@ INFO created directory to store extracted artifacts, path: Date: Mon, 28 Jul 2025 19:55:48 +1200 Subject: [PATCH 31/42] let's reorder output --- dev-tools/reconfigurator-cli/src/lib.rs | 4 +- .../tests/output/cmds-example-stdout | 8 +-- .../tests/output/cmds-stdout | 68 +++++++++---------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index a19ff110825..fc0886ac4e9 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -1394,14 +1394,14 @@ fn cmd_sled_show( swriteln!(s, "sled {} ({}, {})", sled_id, sled.policy, sled.state); swriteln!(s, "serial {}", sled.baseboard_id.serial_number); swriteln!(s, "subnet {}", sled_resources.subnet.net()); + swriteln!(s, "SP active version: {:?}", sp_active_version); + swriteln!(s, "SP inactive version: {:?}", sp_inactive_version); swriteln!(s, "RoT bootloader stage 0 version: {:?}", stage0_version); swriteln!( s, "RoT bootloader stage 0 next version: {:?}", stage0_next_version ); - swriteln!(s, "SP active version: {:?}", sp_active_version); - swriteln!(s, "SP inactive version: {:?}", sp_inactive_version); swriteln!(s, "RoT active slot: {}", rot_active_slot); swriteln!(s, "RoT slot A version: {:?}", rot_slot_a_version); swriteln!(s, "RoT slot B version: {:?}", rot_slot_b_version); diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index b22027f7cca..b747c3f56c4 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -42,10 +42,10 @@ T ENA ID PARENT sled 2eb69596-f081-4e2d-9425-9994926e0832 (in service, active) serial serial1 subnet fd00:1122:3344:102::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None @@ -435,10 +435,10 @@ T ENA ID PARENT sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout index 40a5c9c8aca..9507ef74108 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout @@ -28,10 +28,10 @@ dde1c0e2-b10d-4621-b420-f179f7a7a00a serial0 10 fd00:1122:3344:101::/64 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("0.0.2") RoT slot B version: None @@ -83,10 +83,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 1.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("1.0.0") RoT slot B version: None @@ -123,10 +123,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot b -> 2.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("1.0.0") RoT slot B version: Some("2.0.0") @@ -163,10 +163,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 3.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("3.0.0") RoT slot B version: Some("2.0.0") @@ -203,10 +203,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slo sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: None @@ -243,10 +243,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slo sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("0.0.1") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -286,10 +286,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 1.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("1.0.0") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -326,10 +326,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: inactive -> 2.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("1.0.0") SP inactive version: Some("2.0.0") +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -366,10 +366,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 3.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("3.0.0") SP inactive version: Some("2.0.0") +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -406,10 +406,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 4.0.0, inac sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: None +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -446,10 +446,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 4.0.0, inac sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("0.0.1") -RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -489,10 +489,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("1.0.0") -RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("1.0.0") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -529,10 +529,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0_ne sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("1.0.0") -RoT bootloader stage 0 next version: Some("2.0.0") SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("1.0.0") +RoT bootloader stage 0 next version: Some("2.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -569,10 +569,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("3.0.0") -RoT bootloader stage 0 next version: Some("2.0.0") SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("3.0.0") +RoT bootloader stage 0 next version: Some("2.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -609,10 +609,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("4.0.0") -RoT bootloader stage 0 next version: None SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("4.0.0") +RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -649,10 +649,10 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("4.0.0") -RoT bootloader stage 0 next version: Some("5.0.0") SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("4.0.0") +RoT bootloader stage 0 next version: Some("5.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") @@ -721,10 +721,10 @@ result: sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -RoT bootloader stage 0 version: Some("4.0.0") -RoT bootloader stage 0 next version: Some("5.0.0") SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") +RoT bootloader stage 0 version: Some("4.0.0") +RoT bootloader stage 0 next version: Some("5.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") RoT slot B version: Some("5.0.0") From 312543947525c2f758272be2ce13c09aafcb6a2a Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 7 Aug 2025 12:06:01 +1200 Subject: [PATCH 32/42] Fix tests after merge --- .../tests/input/cmds-target-release.txt | 1 + .../output/cmds-mupdate-update-flow-stdout | 7 ++- .../tests/output/cmds-target-release-stdout | 59 +++++++++++-------- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt b/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt index c895fd17a8b..9ac65b23274 100644 --- a/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt +++ b/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt @@ -77,6 +77,7 @@ blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316 # Now we'll change the inactive slot contents of the simulated SP. Like with the # RoT, this should make the update impossible and cause the planner to fix it. +set ignore-impossible-mgs-updates-since now sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 inventory-generate blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index b374a1cb8b7..72e48d6155b 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -821,8 +821,8 @@ INFO extracting uploaded archive to INFO created directory to store extracted artifacts, path: INFO added artifact, name: installinator_document, kind: installinator_document, version: 1.0.0, hash: c6ae866031d1183094c92cde9d9d1fd5f18356abc81a842ce31471b473fd5582, length: 367 INFO added artifact, name: SimGimletSp, kind: gimlet_sp, version: 1.0.0, hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, length: 747 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 -INFO added artifact, name: fake-gimlet-rot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_a, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 +INFO added artifact, name: SimRot, kind: gimlet_rot_image_b, version: 1.0.0, hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, length: 735 INFO added artifact, name: SimRotStage0, kind: gimlet_rot_bootloader, version: 1.0.0, hash: 005ea358f1cd316df42465b1e3a0334ea22cc0c0442cf9ddf9b42fbf49780236, length: 750 INFO added artifact, name: fake-host, kind: host_phase_1, version: 1.0.0, hash: 2053f8594971bbf0a7326c833e2ffc12b065b9d823b9c0b967d275fa595e4e89, length: 524288 INFO added artifact, name: fake-host, kind: host_phase_2, version: 1.0.0, hash: f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008, length: 1048576 @@ -2136,10 +2136,13 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 +WARN cannot configure RoT update for board (no matching artifact), serial_number: serial0, part_number: model0 WARN cannot configure SP update for board (no matching artifact), serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (no matching artifact), serial_number: serial1, part_number: model1 WARN cannot configure SP update for board (no matching artifact), serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 +WARN cannot configure RoT update for board (no matching artifact), serial_number: serial2, part_number: model2 WARN cannot configure SP update for board (no matching artifact), serial_number: serial2, part_number: model2 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index aaa7a4a4949..b83d0e72395 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -216,8 +216,8 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO reached maximum number of pending SP updates, max: 1 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 @@ -401,8 +401,8 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO reached maximum number of pending SP updates, max: 1 +INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -587,9 +587,9 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO reached maximum number of pending SP updates, max: 1 +INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 @@ -784,10 +784,10 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO skipping board for SP update, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO reached maximum number of pending SP updates, max: 1 +INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 @@ -959,7 +959,11 @@ external DNS: > # This time, make it more interesting. Change the inactive slot contents of > # the simulated RoT. This should make the configured update impossible and cause -> # the planner to fix it. +> # the planner to fix it. To test this, we also need to tell the planner not to +> # ignore this update even though it's quite new. +> set ignore-impossible-mgs-updates-since now +ignoring impossible MGS updates since + > sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot b -> 0.5.0 @@ -980,9 +984,9 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO reached maximum number of pending SP updates, max: 1 +INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 @@ -1174,9 +1178,9 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO reached maximum number of pending SP updates, max: 1 +INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba @@ -1348,6 +1352,9 @@ external DNS: > # Now we'll change the inactive slot contents of the simulated SP. Like with the > # RoT, this should make the update impossible and cause the planner to fix it. +> set ignore-impossible-mgs-updates-since now +ignoring impossible MGS updates since + > sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: inactive -> 0.5.0 @@ -1562,11 +1569,11 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO skipping board for SP update, serial_number: serial1, part_number: model1 -INFO skipping board for SP update, serial_number: serial0, part_number: model0 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO ran out of boards for SP update +INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO ran out of boards for MGS-driven update INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 @@ -1758,9 +1765,9 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO SP update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO configuring SP update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO reached maximum number of pending SP updates, max: 1 +INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 From ce4247e3a647c4aae24b1acdb6840aeb05fd8e06 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 7 Aug 2025 12:27:28 +1200 Subject: [PATCH 33/42] Fix tests after merge --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 8c080b65d9d..938050a3116 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1534,6 +1534,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::Version(ARTIFACT_VERSION_1_5), + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::Version(ARTIFACT_VERSION_1_5), ); let current_boards = &collection.baseboards; let initial_updates = PendingMgsUpdates::new(); @@ -1588,6 +1591,9 @@ mod test { ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::Version(ARTIFACT_VERSION_1_5), ); // If we plan with `ImpossibleUpdatePolicy::Keep`, we should _not_ From 374f613db278d94202079e97406d2f0f1ebe4de3 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 7 Aug 2025 15:27:57 +1200 Subject: [PATCH 34/42] refactor tests --- .../planning/src/mgs_updates/mod.rs | 145 ++++++++++++------ 1 file changed, 101 insertions(+), 44 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 4cf5b3505ce..32b5d05277c 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -725,6 +725,16 @@ fn try_make_update_rot( }; let board = &active_caboose.caboose.board; + let Some(rkth) = &active_caboose.caboose.sign else { + warn!( + log, + "cannot configure RoT update for board \ + (missing sign in caboose from inventory)"; + baseboard_id + ); + return None; + }; + let matching_artifacts: Vec<_> = current_artifacts .artifacts .iter() @@ -733,13 +743,27 @@ fn try_make_update_rot( // // - "name" matching the board name (found above from caboose) // - "kind" matching one of the known RoT kinds - // - "rkth" verified against the CMPA/CFPA found in inventory + // - "sign" matching the rkth (found above from caboose) if a.id.name != *board { return false; } - // TODO-K: Do SIGN check or verify CMPA/CFPA pages + let Some(artifact_sign) = &a.sign else { + // TODO-K: remove log + warn!(log, "MISSING ARTIFACT SIGN. RKTH: {rkth}"); + return false; + }; + let Ok(artifact_sign) = String::from_utf8(artifact_sign.to_vec()) + else { + return false; + }; + if artifact_sign != *rkth { + // TODO-K: remove log + warn!(log, "ARTIFACT_SIGN: {artifact_sign}"); + warn!(log, "RKTH: {rkth}"); + return false; + } match active_slot { RotSlot::A => { @@ -907,17 +931,21 @@ mod test { const ARTIFACT_HASH_NEXUS: ArtifactHash = ArtifactHash([34; 32]); const ARTIFACT_HASH_HOST_OS: ArtifactHash = ArtifactHash([35; 32]); - /// Hash of the RoT development signing key - // TODO-K: This sign only makes sense for one kind of artifact, we want to - // have different ones for switch and power when we implement this. - const ROT_STAGING_DEVEL_SIGN: &str = - "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf"; + /// Hash of a fake RoT signing keys + const ROT_SIGN_GIMLET: &str = + "1111111111111111111111111111111111111111111111111111111111111111"; + const ROT_SIGN_PSC: &str = + "2222222222222222222222222222222222222222222222222222222222222222"; + const ROT_SIGN_SWITCH: &str = + "3333333333333333333333333333333333333333333333333333333333333333"; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] - enum SpComponent { + enum MgsUpdateComponent { Sp, Rot, RotBootloader, + #[allow(unused)] + HostOs, } fn test_artifact_for_board(board: &str) -> ArtifactHash { @@ -968,31 +996,25 @@ mod test { /// Describes the SPs and RoTs in the environment used in these tests, but /// spearated by component for use in sequential testing fn test_config() - -> BTreeMap<(SpType, u16, SpComponent), (&'static str, &'static str)> { - BTreeMap::from([ - ((SpType::Sled, 0, SpComponent::Sp), ("sled_0", "gimlet-d")), - ((SpType::Sled, 1, SpComponent::Sp), ("sled_1", "gimlet-e")), - ((SpType::Sled, 2, SpComponent::Sp), ("sled_2", "gimlet-e")), - ((SpType::Sled, 3, SpComponent::Sp), ("sled_3", "gimlet-e")), - ((SpType::Switch, 0, SpComponent::Sp), ("switch_0", "sidecar-b")), - ((SpType::Switch, 1, SpComponent::Sp), ("switch_1", "sidecar-c")), - ((SpType::Power, 0, SpComponent::Sp), ("power_0", "psc-b")), - ((SpType::Power, 1, SpComponent::Sp), ("power_1", "psc-c")), - ((SpType::Sled, 0, SpComponent::Rot), ("sled_0", "oxide-rot-1")), - ((SpType::Sled, 1, SpComponent::Rot), ("sled_1", "oxide-rot-1")), - ((SpType::Sled, 2, SpComponent::Rot), ("sled_2", "oxide-rot-1")), - ((SpType::Sled, 3, SpComponent::Rot), ("sled_3", "oxide-rot-1")), - ( - (SpType::Switch, 0, SpComponent::Rot), - ("switch_0", "oxide-rot-1"), - ), - ( - (SpType::Switch, 1, SpComponent::Rot), - ("switch_1", "oxide-rot-1"), - ), - ((SpType::Power, 0, SpComponent::Rot), ("power_0", "oxide-rot-1")), - ((SpType::Power, 1, SpComponent::Rot), ("power_1", "oxide-rot-1")), - ]) + -> BTreeMap<(SpType, u16, MgsUpdateComponent), (&'static str, &'static str)> + { + test_collection_config() + .into_iter() + .flat_map( + |((sp_type, slot_id), (serial, sp_board_name, rot_board_name))| { + [ + ( + (sp_type, slot_id, MgsUpdateComponent::Sp), + (serial, sp_board_name), + ), + ( + (sp_type, slot_id, MgsUpdateComponent::Rot), + (serial, rot_board_name), + ), + ] + }, + ) + .collect() } /// Returns a TufRepoDescription that we can use to exercise the planning @@ -1008,58 +1030,92 @@ mod test { "control-plane", KnownArtifactKind::ControlPlane.into(), ARTIFACT_HASH_CONTROL_PLANE, + None, ), make_artifact( "nexus", KnownArtifactKind::Zone.into(), ARTIFACT_HASH_NEXUS, + None, ), make_artifact( "host-os", KnownArtifactKind::Host.into(), ARTIFACT_HASH_HOST_OS, + None, ), make_artifact( "gimlet-d", KnownArtifactKind::GimletSp.into(), test_artifact_for_board("gimlet-d"), + None, ), make_artifact( "gimlet-e", KnownArtifactKind::GimletSp.into(), test_artifact_for_board("gimlet-e"), + None, ), make_artifact( "sidecar-b", KnownArtifactKind::SwitchSp.into(), test_artifact_for_board("sidecar-b"), + None, ), make_artifact( "sidecar-c", KnownArtifactKind::SwitchSp.into(), test_artifact_for_board("sidecar-c"), + None, ), make_artifact( "psc-b", KnownArtifactKind::PscSp.into(), test_artifact_for_board("psc-b"), + None, ), make_artifact( "psc-c", KnownArtifactKind::PscSp.into(), test_artifact_for_board("psc-c"), + None, ), make_artifact( "oxide-rot-1", ArtifactKind::GIMLET_ROT_IMAGE_A, test_artifact_for_board("oxide-rot-1"), + Some(ROT_SIGN_GIMLET.into()), ), make_artifact( "oxide-rot-1", ArtifactKind::GIMLET_ROT_IMAGE_B, test_artifact_for_board("oxide-rot-1"), + Some(ROT_SIGN_GIMLET.into()), + ), + make_artifact( + "oxide-rot-1", + ArtifactKind::PSC_ROT_IMAGE_A, + test_artifact_for_board("oxide-rot-1"), + Some(ROT_SIGN_PSC.into()), + ), + make_artifact( + "oxide-rot-1", + ArtifactKind::PSC_ROT_IMAGE_B, + test_artifact_for_board("oxide-rot-1"), + Some(ROT_SIGN_PSC.into()), + ), + make_artifact( + "oxide-rot-1", + ArtifactKind::SWITCH_ROT_IMAGE_A, + test_artifact_for_board("oxide-rot-1"), + Some(ROT_SIGN_SWITCH.into()), + ), + make_artifact( + "oxide-rot-1", + ArtifactKind::SWITCH_ROT_IMAGE_B, + test_artifact_for_board("oxide-rot-1"), + Some(ROT_SIGN_SWITCH.into()), ), - // TODO-K: Make more artifacts for other RoT artifact kinds ]; TufRepoDescription { @@ -1078,6 +1134,7 @@ mod test { name: &str, kind: ArtifactKind, hash: ArtifactHash, + sign: Option>, ) -> TufArtifactMeta { TufArtifactMeta { id: ArtifactId { @@ -1086,8 +1143,8 @@ mod test { kind, }, hash, - size: 0, // unused here - sign: None, // unused here + size: 0, // unused here + sign, } } @@ -1183,7 +1240,7 @@ mod test { epoch: None, git_commit: String::from("unused"), name: caboose_rot_board.to_string(), - sign: Some(ROT_STAGING_DEVEL_SIGN.to_string()), + sign: Some(ROT_SIGN_GIMLET.to_string()), version: active_rot_version.as_str().to_string(), }, ) @@ -1222,7 +1279,7 @@ mod test { epoch: None, git_commit: String::from("unused"), name: caboose_rot_board.to_string(), - sign: Some(ROT_STAGING_DEVEL_SIGN.to_string()), + sign: Some(ROT_SIGN_GIMLET.to_string()), version: inactive_rot_version.as_str().to_string(), }, ) @@ -1824,23 +1881,23 @@ mod test { fn verify_one_sp_update( expected_updates: &mut BTreeMap< - (SpType, u16, SpComponent), + (SpType, u16, MgsUpdateComponent), (&str, ArtifactHash), >, update: &PendingMgsUpdate, ) { let sp_type = update.sp_type; let sp_slot = update.slot_id; - let sp_component = match &update.details { - PendingMgsUpdateDetails::Rot { .. } => SpComponent::Rot, + let component = match &update.details { + PendingMgsUpdateDetails::Rot { .. } => MgsUpdateComponent::Rot, PendingMgsUpdateDetails::RotBootloader { .. } => { - SpComponent::RotBootloader + MgsUpdateComponent::RotBootloader } - PendingMgsUpdateDetails::Sp { .. } => SpComponent::Sp, + PendingMgsUpdateDetails::Sp { .. } => MgsUpdateComponent::Sp, }; println!("found update: {} slot {}", sp_type, sp_slot); let (expected_serial, expected_artifact) = expected_updates - .remove(&(sp_type, sp_slot, sp_component)) + .remove(&(sp_type, sp_slot, component)) .expect("unexpected update"); assert_eq!(update.artifact_hash, expected_artifact); assert_eq!(update.artifact_version, ARTIFACT_VERSION_2); From a63233bb98e1dfa78b69ec7dcba6cfdad5c33b52 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 7 Aug 2025 17:33:55 +1200 Subject: [PATCH 35/42] Improve tests --- .../planning/src/mgs_updates/mod.rs | 133 +++++++++++++----- 1 file changed, 100 insertions(+), 33 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 32b5d05277c..1205480ad9c 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -750,8 +750,6 @@ fn try_make_update_rot( } let Some(artifact_sign) = &a.sign else { - // TODO-K: remove log - warn!(log, "MISSING ARTIFACT SIGN. RKTH: {rkth}"); return false; }; let Ok(artifact_sign) = String::from_utf8(artifact_sign.to_vec()) @@ -759,13 +757,12 @@ fn try_make_update_rot( return false; }; if artifact_sign != *rkth { - // TODO-K: remove log - warn!(log, "ARTIFACT_SIGN: {artifact_sign}"); - warn!(log, "RKTH: {rkth}"); return false; } - match active_slot { + // We'll be updating the inactive slot, so we choose the artifact + // based on the inactive slot's kind. + match active_slot.toggled() { RotSlot::A => { let slot_a_artifacts = [ ArtifactKind::GIMLET_ROT_IMAGE_A, @@ -774,6 +771,13 @@ fn try_make_update_rot( ]; if slot_a_artifacts.contains(&a.id.kind) { + warn!( + log, + "CHOSE: name {} sign {} kind {}", + a.id.name, + artifact_sign, + a.id.kind + ); return true; } } @@ -785,6 +789,13 @@ fn try_make_update_rot( ]; if slot_b_artifacts.contains(&a.id.kind) { + warn!( + log, + "CHOSE: name {} sign {} kind {}", + a.id.name, + artifact_sign, + a.id.kind + ); return true; } } @@ -922,8 +933,18 @@ mod test { const ARTIFACT_HASH_SP_PSC_B: ArtifactHash = ArtifactHash([9; 32]); /// Hash of fake artifact for fake psc-c SP const ARTIFACT_HASH_SP_PSC_C: ArtifactHash = ArtifactHash([10; 32]); - /// Hash of fake artifact for fake oxide-rot-1 RoT - const ARTIFACT_HASH_OXIDE_ROT_1: ArtifactHash = ArtifactHash([13; 32]); + /// Hash of fake artifact for fake gimlet RoT slot A + const ARTIFACT_HASH_ROT_GIMLET_A: ArtifactHash = ArtifactHash([13; 32]); + /// Hash of fake artifact for fake gimlet RoT slot B + const ARTIFACT_HASH_ROT_GIMLET_B: ArtifactHash = ArtifactHash([14; 32]); + /// Hash of fake artifact for fake psc RoT slot A + const ARTIFACT_HASH_ROT_PSC_A: ArtifactHash = ArtifactHash([17; 32]); + /// Hash of fake artifact for fake psc RoT slot B + const ARTIFACT_HASH_ROT_PSC_B: ArtifactHash = ArtifactHash([18; 32]); + /// Hash of fake artifact for fake switch RoT slot A + const ARTIFACT_HASH_ROT_SWITCH_A: ArtifactHash = ArtifactHash([21; 32]); + /// Hash of fake artifact for fake switch RoT slot B + const ARTIFACT_HASH_ROT_SWITCH_B: ArtifactHash = ArtifactHash([22; 32]); // unused artifact hashes @@ -956,13 +977,22 @@ mod test { "sidecar-c" => ARTIFACT_HASH_SP_SIDECAR_C, "psc-b" => ARTIFACT_HASH_SP_PSC_B, "psc-c" => ARTIFACT_HASH_SP_PSC_C, - "oxide-rot-1" => ARTIFACT_HASH_OXIDE_ROT_1, - // TODO-K: figure out how to have artifacts for different RoTs and - // bootloaders which have the same board name. _ => panic!("test bug: no artifact for board {board:?}"), } } + fn test_artifact_for_artifact_kind(kind: ArtifactKind) -> ArtifactHash { + match kind.as_str() { + "gimlet_rot_image_a" => ARTIFACT_HASH_ROT_GIMLET_A, + "gimlet_rot_image_b" => ARTIFACT_HASH_ROT_GIMLET_B, + "psc_rot_image_a" => ARTIFACT_HASH_ROT_PSC_A, + "psc_rot_image_b" => ARTIFACT_HASH_ROT_PSC_B, + "switch_rot_image_a" => ARTIFACT_HASH_ROT_SWITCH_A, + "switch_rot_image_b" => ARTIFACT_HASH_ROT_SWITCH_B, + _ => panic!("test bug: no artifact for artifact kind {kind:?}"), + } + } + /// Describes the SPs and RoTs in the environment used in these tests /// /// There will be: @@ -979,17 +1009,43 @@ mod test { /// - switch 1: sidecar-c, oxide-rot-1 /// - psc 0: psc-b, oxide-rot-1 /// - psc 1: psc-c, oxide-rot-1 - fn test_collection_config() - -> BTreeMap<(SpType, u16), (&'static str, &'static str, &'static str)> { + fn test_collection_config() -> BTreeMap< + (SpType, u16), + (&'static str, &'static str, &'static str, &'static str), + > { BTreeMap::from([ - ((SpType::Sled, 0), ("sled_0", "gimlet-d", "oxide-rot-1")), - ((SpType::Sled, 1), ("sled_1", "gimlet-e", "oxide-rot-1")), - ((SpType::Sled, 2), ("sled_2", "gimlet-e", "oxide-rot-1")), - ((SpType::Sled, 3), ("sled_3", "gimlet-e", "oxide-rot-1")), - ((SpType::Switch, 0), ("switch_0", "sidecar-b", "oxide-rot-1")), - ((SpType::Switch, 1), ("switch_1", "sidecar-c", "oxide-rot-1")), - ((SpType::Power, 0), ("power_0", "psc-b", "oxide-rot-1")), - ((SpType::Power, 1), ("power_1", "psc-c", "oxide-rot-1")), + ( + (SpType::Sled, 0), + ("sled_0", "gimlet-d", "oxide-rot-1", ROT_SIGN_GIMLET), + ), + ( + (SpType::Sled, 1), + ("sled_1", "gimlet-e", "oxide-rot-1", ROT_SIGN_GIMLET), + ), + ( + (SpType::Sled, 2), + ("sled_2", "gimlet-e", "oxide-rot-1", ROT_SIGN_GIMLET), + ), + ( + (SpType::Sled, 3), + ("sled_3", "gimlet-e", "oxide-rot-1", ROT_SIGN_GIMLET), + ), + ( + (SpType::Switch, 0), + ("switch_0", "sidecar-b", "oxide-rot-1", ROT_SIGN_SWITCH), + ), + ( + (SpType::Switch, 1), + ("switch_1", "sidecar-c", "oxide-rot-1", ROT_SIGN_SWITCH), + ), + ( + (SpType::Power, 0), + ("power_0", "psc-b", "oxide-rot-1", ROT_SIGN_PSC), + ), + ( + (SpType::Power, 1), + ("power_1", "psc-c", "oxide-rot-1", ROT_SIGN_PSC), + ), ]) } @@ -1001,7 +1057,10 @@ mod test { test_collection_config() .into_iter() .flat_map( - |((sp_type, slot_id), (serial, sp_board_name, rot_board_name))| { + |( + (sp_type, slot_id), + (serial, sp_board_name, rot_board_name, ..), + )| { [ ( (sp_type, slot_id, MgsUpdateComponent::Sp), @@ -1083,37 +1142,45 @@ mod test { make_artifact( "oxide-rot-1", ArtifactKind::GIMLET_ROT_IMAGE_A, - test_artifact_for_board("oxide-rot-1"), + test_artifact_for_artifact_kind( + ArtifactKind::GIMLET_ROT_IMAGE_A, + ), Some(ROT_SIGN_GIMLET.into()), ), make_artifact( "oxide-rot-1", ArtifactKind::GIMLET_ROT_IMAGE_B, - test_artifact_for_board("oxide-rot-1"), + test_artifact_for_artifact_kind( + ArtifactKind::GIMLET_ROT_IMAGE_B, + ), Some(ROT_SIGN_GIMLET.into()), ), make_artifact( "oxide-rot-1", ArtifactKind::PSC_ROT_IMAGE_A, - test_artifact_for_board("oxide-rot-1"), + test_artifact_for_artifact_kind(ArtifactKind::PSC_ROT_IMAGE_A), Some(ROT_SIGN_PSC.into()), ), make_artifact( "oxide-rot-1", ArtifactKind::PSC_ROT_IMAGE_B, - test_artifact_for_board("oxide-rot-1"), + test_artifact_for_artifact_kind(ArtifactKind::PSC_ROT_IMAGE_B), Some(ROT_SIGN_PSC.into()), ), make_artifact( "oxide-rot-1", ArtifactKind::SWITCH_ROT_IMAGE_A, - test_artifact_for_board("oxide-rot-1"), + test_artifact_for_artifact_kind( + ArtifactKind::SWITCH_ROT_IMAGE_A, + ), Some(ROT_SIGN_SWITCH.into()), ), make_artifact( "oxide-rot-1", ArtifactKind::SWITCH_ROT_IMAGE_B, - test_artifact_for_board("oxide-rot-1"), + test_artifact_for_artifact_kind( + ArtifactKind::SWITCH_ROT_IMAGE_B, + ), Some(ROT_SIGN_SWITCH.into()), ), ]; @@ -1195,7 +1262,7 @@ mod test { let test_config = test_collection_config(); for ( (sp_type, sp_slot), - (serial, caboose_sp_board, caboose_rot_board), + (serial, caboose_sp_board, caboose_rot_board, rkth), ) in test_config { let sp_state = SpState { @@ -1240,7 +1307,7 @@ mod test { epoch: None, git_commit: String::from("unused"), name: caboose_rot_board.to_string(), - sign: Some(ROT_SIGN_GIMLET.to_string()), + sign: Some(rkth.to_string()), version: active_rot_version.as_str().to_string(), }, ) @@ -1279,7 +1346,7 @@ mod test { epoch: None, git_commit: String::from("unused"), name: caboose_rot_board.to_string(), - sign: Some(ROT_SIGN_GIMLET.to_string()), + sign: Some(rkth.to_string()), version: inactive_rot_version.as_str().to_string(), }, ) @@ -1342,7 +1409,7 @@ mod test { assert_eq!(first_update.baseboard_id.serial_number, "sled_0"); assert_eq!(first_update.sp_type, SpType::Sled); assert_eq!(first_update.slot_id, 0); - assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_OXIDE_ROT_1); + assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_ROT_GIMLET_B); assert_eq!(first_update.artifact_version, ARTIFACT_VERSION_2); // Test that when an update is already pending, and nothing changes @@ -1415,7 +1482,7 @@ mod test { assert_eq!(next_update.baseboard_id.serial_number, "switch_1"); assert_eq!(next_update.sp_type, SpType::Switch); assert_eq!(next_update.slot_id, 1); - assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_OXIDE_ROT_1); + assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_ROT_SWITCH_B); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); // Finally, test that when all RoTs and SPs are in spec, then no updates From 77c24097237ed4f6eb72759ba85ab8b3988ae969 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 7 Aug 2025 18:35:18 +1200 Subject: [PATCH 36/42] Improve testing --- .../planning/src/mgs_updates/mod.rs | 95 +++++++++++++++++-- 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 1205480ad9c..55b49d13417 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1373,7 +1373,7 @@ mod test { &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ); let current_boards = &collection.baseboards; @@ -1391,9 +1391,8 @@ mod test { ); assert!(updates.is_empty()); - // Test that when a TUF repo is specified and one RoT and SP are - // outdated, then it's configured with an update (and the update looks - // correct). + // Test that when a TUF repo is specified and one SP is outdated, then + // it's configured with an update (and the update looks correct). let repo = make_tuf_repo(); let updates = plan_mgs_updates( log, @@ -1409,7 +1408,7 @@ mod test { assert_eq!(first_update.baseboard_id.serial_number, "sled_0"); assert_eq!(first_update.sp_type, SpType::Sled); assert_eq!(first_update.slot_id, 0); - assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_ROT_GIMLET_B); + assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_SP_GIMLET_D); assert_eq!(first_update.artifact_version, ARTIFACT_VERSION_2); // Test that when an update is already pending, and nothing changes @@ -1426,6 +1425,34 @@ mod test { ); assert_eq!(updates, later_updates); + // Now we make sure the RoT is updated as well + let later_collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, + ); + let later_updates = plan_mgs_updates( + log, + &later_collection, + current_boards, + &updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert_eq!(later_updates.len(), 1); + let next_update = + later_updates.iter().next().expect("at least one update"); + assert_ne!(first_update, next_update); + assert_eq!(next_update.baseboard_id.serial_number, "sled_0"); + assert_eq!(next_update.sp_type, SpType::Sled); + assert_eq!(next_update.slot_id, 0); + assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_ROT_GIMLET_B); + assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); + // Test that when two updates for two SpTypes are needed, but one is // already pending, then the other one is *not* started (because it // exceeds nmax_updates). @@ -1455,9 +1482,11 @@ mod test { assert_eq!(updates, later_updates); // At this point, we're ready to test that when the first SpType update - // completes, then the second one *is* started. This tests two - // different things: first that we noticed the first one completed, and - // second that we noticed another thing needed an update + // completes, then the second one *is* started. This tests three + // different things: first that we noticed the first one completed, + // second that we noticed another thing needed an update, and third that + // the planner schedules the updates in the correct order: first RoT, + // and second SP. let later_collection = make_collection( ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), @@ -1485,6 +1514,34 @@ mod test { assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_ROT_SWITCH_B); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); + // Same test but for SP only. + let later_collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, + ); + let later_updates = plan_mgs_updates( + log, + &later_collection, + current_boards, + &updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert_eq!(later_updates.len(), 1); + let next_update = + later_updates.iter().next().expect("at least one update"); + assert_ne!(first_update, next_update); + assert_eq!(next_update.baseboard_id.serial_number, "switch_1"); + assert_eq!(next_update.sp_type, SpType::Switch); + assert_eq!(next_update.slot_id, 1); + assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); + assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); + // Finally, test that when all RoTs and SPs are in spec, then no updates // are configured. let updated_collection = make_collection( @@ -1792,7 +1849,16 @@ mod test { let mut expected_updates: BTreeMap<_, _> = test_config() .into_iter() .map(|(k, (serial, board_name))| { - (k, (serial, test_artifact_for_board(board_name))) + if board_name == "oxide-rot-1" { + let kind = match k.0 { + SpType::Sled => ArtifactKind::GIMLET_ROT_IMAGE_B, + SpType::Power => ArtifactKind::PSC_ROT_IMAGE_B, + SpType::Switch => ArtifactKind::SWITCH_ROT_IMAGE_B, + }; + (k, (serial, test_artifact_for_artifact_kind(kind))) + } else { + (k, (serial, test_artifact_for_board(board_name))) + } }) .collect(); @@ -1890,7 +1956,16 @@ mod test { let mut expected_updates: BTreeMap<_, _> = test_config() .into_iter() .map(|(k, (serial, board_name))| { - (k, (serial, test_artifact_for_board(board_name))) + if board_name == "oxide-rot-1" { + let kind = match k.0 { + SpType::Sled => ArtifactKind::GIMLET_ROT_IMAGE_B, + SpType::Power => ArtifactKind::PSC_ROT_IMAGE_B, + SpType::Switch => ArtifactKind::SWITCH_ROT_IMAGE_B, + }; + (k, (serial, test_artifact_for_artifact_kind(kind))) + } else { + (k, (serial, test_artifact_for_board(board_name))) + } }) .collect(); From 5ecf6a5385e1351190c66de2f03ea672e7669e66 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 8 Aug 2025 10:09:08 +1200 Subject: [PATCH 37/42] expectorate --- .../output/cmds-mupdate-update-flow-stdout | 6 +- .../tests/output/cmds-target-release-stdout | 94 +++++++------------ 2 files changed, 35 insertions(+), 65 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index 72e48d6155b..8d19d5d30d5 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -2136,13 +2136,13 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -WARN cannot configure RoT update for board (no matching artifact), serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 WARN cannot configure SP update for board (no matching artifact), serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -WARN cannot configure RoT update for board (no matching artifact), serial_number: serial1, part_number: model1 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 WARN cannot configure SP update for board (no matching artifact), serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -WARN cannot configure RoT update for board (no matching artifact), serial_number: serial2, part_number: model2 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial2, part_number: model2 WARN cannot configure SP update for board (no matching artifact), serial_number: serial2, part_number: model2 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index b83d0e72395..1862e3af368 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -216,7 +216,8 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 @@ -241,10 +242,10 @@ to: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -+ sled 0 model0 serial0 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ++ sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } internal DNS: @@ -401,7 +402,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -587,8 +588,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 @@ -610,15 +610,6 @@ to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 generation: 1 (unchanged) read from:: SingleNode (unchanged) - PENDING MGS UPDATES: - - Pending MGS-managed updates (all baseboards): - --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -* sled 0 model0 serial0 - 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } - └─ + 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } - internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) @@ -785,8 +776,10 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 @@ -811,11 +804,11 @@ to: blueprint df06bb57-ad42-4431-9206-abff322896c7 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } -+ sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } ++ sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } internal DNS: @@ -984,8 +977,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 @@ -1007,15 +999,6 @@ to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba generation: 1 (unchanged) read from:: SingleNode (unchanged) - PENDING MGS UPDATES: - - Pending MGS-managed updates (all baseboards): - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -* sled 1 model1 serial1 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } - └─ + Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } - internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) @@ -1178,8 +1161,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), component: rot, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba @@ -1201,15 +1183,6 @@ to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac generation: 1 (unchanged) read from:: SingleNode (unchanged) - PENDING MGS UPDATES: - - Pending MGS-managed updates (all baseboards): - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -* sled 1 model1 serial1 - 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } - └─ + 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } - internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) @@ -1376,6 +1349,7 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 INFO MGS-driven update impossible (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify @@ -1570,9 +1544,12 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial2, part_number: model2 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 @@ -1597,11 +1574,11 @@ to: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 PENDING MGS UPDATES: Pending MGS-managed updates (all baseboards): - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) } -+ sled 2 model2 serial2 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) } ++ sled 2 model2 serial2 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } internal DNS: @@ -1765,8 +1742,7 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a, expected_transient_boot_preference: None, expected_pending_persistent_boot_preference: None, expected_persistent_boot_preference: A, expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, component: rot, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 @@ -1788,15 +1764,6 @@ to: blueprint 626487fa-7139-45ec-8416-902271fc730b generation: 1 (unchanged) read from:: SingleNode (unchanged) - PENDING MGS UPDATES: - - Pending MGS-managed updates (all baseboards): - --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -* sled 2 model2 serial2 - 04e4a7fdb84acca92c8fd3235e26d64ea61bef8a5f98202589fd346989c5720a 1.0.0 - Rot { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None } - └─ + 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 + Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } - internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) @@ -1960,8 +1927,11 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial2, part_number: model2 INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 INFO ran out of boards for MGS-driven update INFO updating zone image source in-place, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545, kind: Clickhouse, image_source: artifact: version 1.0.0 From 2456fda51c3214366c2c95a2ce77a7b4699c1b5b Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 8 Aug 2025 12:01:43 +1200 Subject: [PATCH 38/42] Address comments --- dev-tools/reconfigurator-cli/src/lib.rs | 4 +- .../tests/output/cmds-example-stdout | 8 +- .../tests/output/cmds-stdout | 68 ++++++++--------- .../planning/src/mgs_updates/mod.rs | 73 +++++++++---------- nexus/reconfigurator/planning/src/system.rs | 17 ++--- nexus/types/src/inventory.rs | 7 ++ 6 files changed, 89 insertions(+), 88 deletions(-) diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index a5fb1f8543f..3bd2909cc11 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -1416,9 +1416,9 @@ fn cmd_sled_show( swriteln!(s, "sled {} ({}, {})", sled_id, sled.policy, sled.state); swriteln!(s, "serial {}", sled.baseboard_id.serial_number); swriteln!(s, "subnet {}", sled_resources.subnet.net()); - swriteln!(s, "SP active version: {:?}", sp_active_version); + swriteln!(s, "SP active version: {:?}", sp_active_version); swriteln!(s, "SP inactive version: {:?}", sp_inactive_version); - swriteln!(s, "RoT bootloader stage 0 version: {:?}", stage0_version); + swriteln!(s, "RoT bootloader stage 0 version: {:?}", stage0_version); swriteln!( s, "RoT bootloader stage 0 next version: {:?}", diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index d6114ceeade..1360fae93e4 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -42,9 +42,9 @@ T ENA ID PARENT sled 2eb69596-f081-4e2d-9425-9994926e0832 (in service, active) serial serial1 subnet fd00:1122:3344:102::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("0.0.2") @@ -435,9 +435,9 @@ T ENA ID PARENT sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("0.0.2") diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout index 9507ef74108..52b0cc3f939 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-stdout @@ -28,9 +28,9 @@ dde1c0e2-b10d-4621-b420-f179f7a7a00a serial0 10 fd00:1122:3344:101::/64 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("0.0.2") @@ -83,9 +83,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 1.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("1.0.0") @@ -123,9 +123,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot b -> 2.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("1.0.0") @@ -163,9 +163,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 3.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("3.0.0") @@ -203,9 +203,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slo sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -243,9 +243,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT settings: slot a -> 4.0.0, slo sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("0.0.1") +SP active version: Some("0.0.1") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -286,9 +286,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 1.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("1.0.0") +SP active version: Some("1.0.0") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -326,9 +326,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: inactive -> 2.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("1.0.0") +SP active version: Some("1.0.0") SP inactive version: Some("2.0.0") -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -366,9 +366,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 3.0.0 sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("3.0.0") +SP active version: Some("3.0.0") SP inactive version: Some("2.0.0") -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -406,9 +406,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 4.0.0, inac sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: None -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -446,9 +446,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a SP versions: active -> 4.0.0, inac sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("0.0.1") +RoT bootloader stage 0 version: Some("0.0.1") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -489,9 +489,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("1.0.0") +RoT bootloader stage 0 version: Some("1.0.0") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -529,9 +529,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0_ne sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("1.0.0") +RoT bootloader stage 0 version: Some("1.0.0") RoT bootloader stage 0 next version: Some("2.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") @@ -569,9 +569,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("3.0.0") +RoT bootloader stage 0 version: Some("3.0.0") RoT bootloader stage 0 next version: Some("2.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") @@ -609,9 +609,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("4.0.0") +RoT bootloader stage 0 version: Some("4.0.0") RoT bootloader stage 0 next version: None RoT active slot: A RoT slot A version: Some("4.0.0") @@ -649,9 +649,9 @@ set sled dde1c0e2-b10d-4621-b420-f179f7a7a00a RoT bootloader versions: stage0 -> sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("4.0.0") +RoT bootloader stage 0 version: Some("4.0.0") RoT bootloader stage 0 next version: Some("5.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") @@ -721,9 +721,9 @@ result: sled dde1c0e2-b10d-4621-b420-f179f7a7a00a (in service, active) serial serial0 subnet fd00:1122:3344:101::/64 -SP active version: Some("4.0.0") +SP active version: Some("4.0.0") SP inactive version: Some("5.0.0") -RoT bootloader stage 0 version: Some("4.0.0") +RoT bootloader stage 0 version: Some("4.0.0") RoT bootloader stage 0 next version: Some("5.0.0") RoT active slot: A RoT slot A version: Some("4.0.0") diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 55b49d13417..1de711814b6 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -406,7 +406,23 @@ fn mgs_update_status_rot( expected_inactive_version: &ExpectedVersion, found_inactive_version: Option<&str>, ) -> MgsUpdateStatus { - if &found.active_slot.version() == desired_version { + let RotUpdateState { + active_slot: found_active_slot, + persistent_boot_preference: found_persistent_boot_preference, + pending_persistent_boot_preference: + found_pending_persistent_boot_preference, + transient_boot_preference: found_transient_boot_preference, + } = found; + + let RotUpdateState { + active_slot: expected_active_slot, + persistent_boot_preference: expected_persistent_boot_preference, + pending_persistent_boot_preference: + expected_pending_persistent_boot_preference, + transient_boot_preference: expected_transient_boot_preference, + } = expected; + + if &found_active_slot.version() == desired_version { // If we find the desired version in the active slot, we're done. return MgsUpdateStatus::Done; } @@ -424,15 +440,15 @@ fn mgs_update_status_rot( // Transient boot preference is not in use yet, if we find that it is set we // should not proceed. Once https://github.com/oxidecomputer/hubris/pull/2050 // is implemented, we should revist this check - if found.active_slot.version() != expected.active_slot.version() - || found.persistent_boot_preference - != expected.persistent_boot_preference - || found.pending_persistent_boot_preference - != expected.pending_persistent_boot_preference - || found.transient_boot_preference != expected.transient_boot_preference - || found.persistent_boot_preference != found.active_slot.slot - || found.transient_boot_preference.is_some() - || expected.transient_boot_preference.is_some() + if found_active_slot.version() != expected_active_slot.version() + || found_persistent_boot_preference + != expected_persistent_boot_preference + || found_pending_persistent_boot_preference + != expected_pending_persistent_boot_preference + || found_transient_boot_preference != expected_transient_boot_preference + || found_persistent_boot_preference != found_active_slot.slot + || found_transient_boot_preference.is_some() + || expected_transient_boot_preference.is_some() { return MgsUpdateStatus::Impossible; } @@ -444,13 +460,13 @@ fn mgs_update_status_rot( // https://github.com/oxidecomputer/omicron/issues/8414 for context // about when we'll be able to know whether an it's an ongoing update // or an RoT in a failed state. - if found.pending_persistent_boot_preference.is_some() { + if found_pending_persistent_boot_preference.is_some() { return MgsUpdateStatus::NotDone; } // If there is a mismatch between the expected active slot and the found // active slot then an update is not done. - if found.active_slot.slot != expected.active_slot.slot { + if found_active_slot.slot != expected_active_slot.slot { return MgsUpdateStatus::NotDone; } @@ -695,13 +711,8 @@ fn try_make_update_rot( let active_slot = rot_state.active_slot; - let active_caboose = match active_slot { - RotSlot::A => CabooseWhich::RotSlotA, - RotSlot::B => CabooseWhich::RotSlotB, - }; - - let Some(active_caboose) = - inventory.caboose_for(active_caboose, baseboard_id) + let Some(active_caboose) = inventory + .caboose_for(CabooseWhich::from_rot_slot(active_slot), baseboard_id) else { warn!( log, @@ -771,13 +782,6 @@ fn try_make_update_rot( ]; if slot_a_artifacts.contains(&a.id.kind) { - warn!( - log, - "CHOSE: name {} sign {} kind {}", - a.id.name, - artifact_sign, - a.id.kind - ); return true; } } @@ -789,13 +793,6 @@ fn try_make_update_rot( ]; if slot_b_artifacts.contains(&a.id.kind) { - warn!( - log, - "CHOSE: name {} sign {} kind {}", - a.id.name, - artifact_sign, - a.id.kind - ); return true; } } @@ -836,13 +833,11 @@ fn try_make_update_rot( }; // Begin configuring an update. - let inactive_caboose = match active_slot.toggled() { - RotSlot::A => CabooseWhich::RotSlotA, - RotSlot::B => CabooseWhich::RotSlotB, - }; - let expected_inactive_version = match inventory - .caboose_for(inactive_caboose, baseboard_id) + .caboose_for( + CabooseWhich::from_rot_slot(active_slot.toggled()), + baseboard_id, + ) .map(|c| c.caboose.version.parse::()) .transpose() { diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index dcab1020031..54fbde63c45 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -615,13 +615,13 @@ impl SystemDescription { pub fn sled_rot_active_slot( &self, sled_id: SledUuid, - ) -> anyhow::Result<&RotSlot> { + ) -> anyhow::Result { let sp_state = self.sled_sp_state(sled_id)?; sp_state .ok_or_else(|| { anyhow!("failed to retrieve SP state from sled id: {sled_id}") }) - .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + .and_then(|(_hw_slot, sp_state)| match sp_state.rot.clone() { RotState::V2 { active, .. } | RotState::V3 { active, .. } => { Ok(active) } @@ -635,13 +635,13 @@ impl SystemDescription { pub fn sled_rot_persistent_boot_preference( &self, sled_id: SledUuid, - ) -> anyhow::Result<&RotSlot> { + ) -> anyhow::Result { let sp_state = self.sled_sp_state(sled_id)?; sp_state .ok_or_else(|| { anyhow!("failed to retrieve SP state from sled id: {sled_id}") }) - .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + .and_then(|(_hw_slot, sp_state)| match sp_state.rot.clone() { RotState::V2 { persistent_boot_preference, .. } | RotState::V3 { persistent_boot_preference, .. } => { Ok(persistent_boot_preference) @@ -656,13 +656,13 @@ impl SystemDescription { pub fn sled_rot_pending_persistent_boot_preference( &self, sled_id: SledUuid, - ) -> anyhow::Result<&Option> { + ) -> anyhow::Result> { let sp_state = self.sled_sp_state(sled_id)?; sp_state .ok_or_else(|| { anyhow!("failed to retrieve SP state from sled id: {sled_id}") }) - .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + .and_then(|(_hw_slot, sp_state)| match sp_state.rot.clone() { RotState::V2 { pending_persistent_boot_preference, .. } | RotState::V3 { pending_persistent_boot_preference, .. } => { Ok(pending_persistent_boot_preference) @@ -677,13 +677,13 @@ impl SystemDescription { pub fn sled_rot_transient_boot_preference( &self, sled_id: SledUuid, - ) -> anyhow::Result<&Option> { + ) -> anyhow::Result> { let sp_state = self.sled_sp_state(sled_id)?; sp_state .ok_or_else(|| { anyhow!("failed to retrieve SP state from sled id: {sled_id}") }) - .and_then(|(_hw_slot, sp_state)| match &sp_state.rot { + .and_then(|(_hw_slot, sp_state)| match sp_state.rot.clone() { RotState::V2 { transient_boot_preference, .. } | RotState::V3 { transient_boot_preference, .. } => { Ok(transient_boot_preference) @@ -1577,7 +1577,6 @@ impl Sled { /// Update the reported RoT bootloader versions /// /// If either field is `None`, that field is _unchanged_. - // Note that this means there's no way to _unset_ the version. fn set_rot_bootloader_versions( &mut self, stage0_version: Option, diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 3bda2503396..713e074bf2b 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -457,6 +457,13 @@ impl CabooseWhich { CabooseWhich::Stage0Next => CabooseWhich::Stage0, } } + + pub fn from_rot_slot(slot: RotSlot) -> Self { + match slot { + RotSlot::A => CabooseWhich::RotSlotA, + RotSlot::B => CabooseWhich::RotSlotB, + } + } } /// Root of trust page contents found during a collection From 46526ee47aeb038839d958ce6eef5c5062150a20 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 8 Aug 2025 12:41:04 +1200 Subject: [PATCH 39/42] Extract rot code into its own submodule --- .../planning/src/mgs_updates/mod.rs | 286 +---------------- .../planning/src/mgs_updates/rot.rs | 301 ++++++++++++++++++ 2 files changed, 307 insertions(+), 280 deletions(-) create mode 100644 nexus/reconfigurator/planning/src/mgs_updates/rot.rs diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 1de711814b6..ab54387f1c9 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -4,6 +4,12 @@ //! Facilities for making choices about MGS-managed updates +mod rot; + +use crate::mgs_updates::rot::RotUpdateState; +use crate::mgs_updates::rot::mgs_update_status_rot; +use crate::mgs_updates::rot::try_make_update_rot; + use gateway_types::rot::RotSlot; use nexus_types::deployment::ExpectedActiveRotSlot; use nexus_types::deployment::ExpectedVersion; @@ -20,7 +26,6 @@ use slog_error_chain::InlineErrorChain; use std::collections::BTreeSet; use std::sync::Arc; use thiserror::Error; -use tufaceous_artifact::ArtifactKind; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::ArtifactVersionError; use tufaceous_artifact::KnownArtifactKind; @@ -392,94 +397,6 @@ fn mgs_update_status_sp( ) } -struct RotUpdateState { - active_slot: ExpectedActiveRotSlot, - persistent_boot_preference: RotSlot, - pending_persistent_boot_preference: Option, - transient_boot_preference: Option, -} - -fn mgs_update_status_rot( - desired_version: &ArtifactVersion, - expected: RotUpdateState, - found: RotUpdateState, - expected_inactive_version: &ExpectedVersion, - found_inactive_version: Option<&str>, -) -> MgsUpdateStatus { - let RotUpdateState { - active_slot: found_active_slot, - persistent_boot_preference: found_persistent_boot_preference, - pending_persistent_boot_preference: - found_pending_persistent_boot_preference, - transient_boot_preference: found_transient_boot_preference, - } = found; - - let RotUpdateState { - active_slot: expected_active_slot, - persistent_boot_preference: expected_persistent_boot_preference, - pending_persistent_boot_preference: - expected_pending_persistent_boot_preference, - transient_boot_preference: expected_transient_boot_preference, - } = expected; - - if &found_active_slot.version() == desired_version { - // If we find the desired version in the active slot, we're done. - return MgsUpdateStatus::Done; - } - - // The update hasn't completed. - // - // Check to make sure the contents of the active slot, persistent boot - // preference, pending persistent boot preference, and transient boot - // preference are still what they were when we configured this update. - // If not, then this update cannot proceed as currently configured. - // If there is a mismatch between the found persistent boot preference - // and the found active slot then this means a failed update. We cannot - // proceed. - // It will fail its precondition check. - // Transient boot preference is not in use yet, if we find that it is set we - // should not proceed. Once https://github.com/oxidecomputer/hubris/pull/2050 - // is implemented, we should revist this check - if found_active_slot.version() != expected_active_slot.version() - || found_persistent_boot_preference - != expected_persistent_boot_preference - || found_pending_persistent_boot_preference - != expected_pending_persistent_boot_preference - || found_transient_boot_preference != expected_transient_boot_preference - || found_persistent_boot_preference != found_active_slot.slot - || found_transient_boot_preference.is_some() - || expected_transient_boot_preference.is_some() - { - return MgsUpdateStatus::Impossible; - } - - // If found pending persistent boot preference is not empty, then an update - // is not done. - // - // TODO: Alternatively, this could also mean a failed update. See - // https://github.com/oxidecomputer/omicron/issues/8414 for context - // about when we'll be able to know whether an it's an ongoing update - // or an RoT in a failed state. - if found_pending_persistent_boot_preference.is_some() { - return MgsUpdateStatus::NotDone; - } - - // If there is a mismatch between the expected active slot and the found - // active slot then an update is not done. - if found_active_slot.slot != expected_active_slot.slot { - return MgsUpdateStatus::NotDone; - } - - // Similarly, check the contents of the inactive slot to determine if it - // still matches what we saw when we configured this update. If not, then - // this update cannot proceed as currently configured. It will fail its - // precondition check. - mgs_update_status_inactive_versions( - found_inactive_version, - expected_inactive_version, - ) -} - fn mgs_update_status_inactive_versions( found_inactive_version: Option<&str>, expected_inactive_version: &ExpectedVersion, @@ -682,197 +599,6 @@ fn try_make_update_sp( }) } -/// Determine if the given baseboard needs an SP update and, if so, returns it. -fn try_make_update_rot( - log: &slog::Logger, - baseboard_id: &Arc, - inventory: &Collection, - current_artifacts: &TufRepoDescription, -) -> Option { - let Some(sp_info) = inventory.sps.get(baseboard_id) else { - warn!( - log, - "cannot configure RoT update for board \ - (missing SP info from inventory)"; - baseboard_id - ); - return None; - }; - - let Some(rot_state) = inventory.rots.get(baseboard_id) else { - warn!( - log, - "cannot configure RoT update for board \ - (missing RoT state from inventory)"; - baseboard_id - ); - return None; - }; - - let active_slot = rot_state.active_slot; - - let Some(active_caboose) = inventory - .caboose_for(CabooseWhich::from_rot_slot(active_slot), baseboard_id) - else { - warn!( - log, - "cannot configure RoT update for board \ - (missing active slot {active_slot} caboose from inventory)"; - baseboard_id, - ); - return None; - }; - - let Ok(expected_active_version) = active_caboose.caboose.version.parse() - else { - warn!( - log, - "cannot configure RoT update for board \ - (cannot parse current active version as an ArtifactVersion)"; - baseboard_id, - "found_version" => &active_caboose.caboose.version, - ); - return None; - }; - - let board = &active_caboose.caboose.board; - let Some(rkth) = &active_caboose.caboose.sign else { - warn!( - log, - "cannot configure RoT update for board \ - (missing sign in caboose from inventory)"; - baseboard_id - ); - return None; - }; - - let matching_artifacts: Vec<_> = current_artifacts - .artifacts - .iter() - .filter(|a| { - // A matching RoT artifact will have: - // - // - "name" matching the board name (found above from caboose) - // - "kind" matching one of the known RoT kinds - // - "sign" matching the rkth (found above from caboose) - - if a.id.name != *board { - return false; - } - - let Some(artifact_sign) = &a.sign else { - return false; - }; - let Ok(artifact_sign) = String::from_utf8(artifact_sign.to_vec()) - else { - return false; - }; - if artifact_sign != *rkth { - return false; - } - - // We'll be updating the inactive slot, so we choose the artifact - // based on the inactive slot's kind. - match active_slot.toggled() { - RotSlot::A => { - let slot_a_artifacts = [ - ArtifactKind::GIMLET_ROT_IMAGE_A, - ArtifactKind::PSC_ROT_IMAGE_A, - ArtifactKind::SWITCH_ROT_IMAGE_A, - ]; - - if slot_a_artifacts.contains(&a.id.kind) { - return true; - } - } - RotSlot::B => { - let slot_b_artifacts = [ - ArtifactKind::GIMLET_ROT_IMAGE_B, - ArtifactKind::PSC_ROT_IMAGE_B, - ArtifactKind::SWITCH_ROT_IMAGE_B, - ]; - - if slot_b_artifacts.contains(&a.id.kind) { - return true; - } - } - } - - false - }) - .collect(); - if matching_artifacts.is_empty() { - warn!( - log, - "cannot configure RoT update for board (no matching artifact)"; - baseboard_id, - ); - return None; - } - - if matching_artifacts.len() > 1 { - // This should be impossible unless we shipped a TUF repo with more - // than 1 artifact for the same board and root key table hash (RKTH) - // that can be verified afgainst the RoT's CMPA/CFPA. But it doesn't - // prevent us from picking one and proceeding. Make a note and proceed. - error!(log, "found more than one matching artifact for RoT update"); - } - - let artifact = matching_artifacts[0]; - - // If the artifact's version matches what's deployed, then no update is - // needed. - if artifact.id.version == expected_active_version { - debug!(log, "no RoT update needed for board"; baseboard_id); - return None; - } - - let expected_active_slot = ExpectedActiveRotSlot { - slot: active_slot, - version: expected_active_version, - }; - - // Begin configuring an update. - let expected_inactive_version = match inventory - .caboose_for( - CabooseWhich::from_rot_slot(active_slot.toggled()), - baseboard_id, - ) - .map(|c| c.caboose.version.parse::()) - .transpose() - { - Ok(None) => ExpectedVersion::NoValidVersion, - Ok(Some(v)) => ExpectedVersion::Version(v), - Err(_) => { - warn!( - log, - "cannot configure RoT update for board \ - (found inactive slot contents but version was not valid)"; - baseboard_id - ); - return None; - } - }; - - Some(PendingMgsUpdate { - baseboard_id: baseboard_id.clone(), - sp_type: sp_info.sp_type, - slot_id: sp_info.sp_slot, - details: PendingMgsUpdateDetails::Rot { - expected_active_slot, - expected_inactive_version, - expected_persistent_boot_preference: rot_state - .persistent_boot_preference, - expected_pending_persistent_boot_preference: rot_state - .pending_persistent_boot_preference, - expected_transient_boot_preference: rot_state - .transient_boot_preference, - }, - artifact_hash: artifact.hash, - artifact_version: artifact.id.version.clone(), - }) -} - #[cfg(test)] mod test { use super::ImpossibleUpdatePolicy; diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs new file mode 100644 index 00000000000..cfcf0535537 --- /dev/null +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -0,0 +1,301 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Facilities for making choices about RoT updates + +use super::MgsUpdateStatus; +use super::mgs_update_status_inactive_versions; + +use gateway_types::rot::RotSlot; +use nexus_types::deployment::ExpectedActiveRotSlot; +use nexus_types::deployment::ExpectedVersion; +use nexus_types::deployment::PendingMgsUpdate; +use nexus_types::deployment::PendingMgsUpdateDetails; +use nexus_types::inventory::BaseboardId; +use nexus_types::inventory::CabooseWhich; +use nexus_types::inventory::Collection; +use omicron_common::api::external::TufRepoDescription; +use slog::{debug, error, warn}; +use std::sync::Arc; +use tufaceous_artifact::ArtifactKind; +use tufaceous_artifact::ArtifactVersion; + +pub struct RotUpdateState { + pub active_slot: ExpectedActiveRotSlot, + pub persistent_boot_preference: RotSlot, + pub pending_persistent_boot_preference: Option, + pub transient_boot_preference: Option, +} + +pub fn mgs_update_status_rot( + desired_version: &ArtifactVersion, + expected: RotUpdateState, + found: RotUpdateState, + expected_inactive_version: &ExpectedVersion, + found_inactive_version: Option<&str>, +) -> MgsUpdateStatus { + let RotUpdateState { + active_slot: found_active_slot, + persistent_boot_preference: found_persistent_boot_preference, + pending_persistent_boot_preference: + found_pending_persistent_boot_preference, + transient_boot_preference: found_transient_boot_preference, + } = found; + + let RotUpdateState { + active_slot: expected_active_slot, + persistent_boot_preference: expected_persistent_boot_preference, + pending_persistent_boot_preference: + expected_pending_persistent_boot_preference, + transient_boot_preference: expected_transient_boot_preference, + } = expected; + + if &found_active_slot.version() == desired_version { + // If we find the desired version in the active slot, we're done. + return MgsUpdateStatus::Done; + } + + // The update hasn't completed. + // + // Check to make sure the contents of the active slot, persistent boot + // preference, pending persistent boot preference, and transient boot + // preference are still what they were when we configured this update. + // If not, then this update cannot proceed as currently configured. + // If there is a mismatch between the found persistent boot preference + // and the found active slot then this means a failed update. We cannot + // proceed. + // It will fail its precondition check. + // Transient boot preference is not in use yet, if we find that it is set we + // should not proceed. Once https://github.com/oxidecomputer/hubris/pull/2050 + // is implemented, we should revist this check + if found_active_slot.version() != expected_active_slot.version() + || found_persistent_boot_preference + != expected_persistent_boot_preference + || found_pending_persistent_boot_preference + != expected_pending_persistent_boot_preference + || found_transient_boot_preference != expected_transient_boot_preference + || found_persistent_boot_preference != found_active_slot.slot + || found_transient_boot_preference.is_some() + || expected_transient_boot_preference.is_some() + { + return MgsUpdateStatus::Impossible; + } + + // If found pending persistent boot preference is not empty, then an update + // is not done. + // + // TODO: Alternatively, this could also mean a failed update. See + // https://github.com/oxidecomputer/omicron/issues/8414 for context + // about when we'll be able to know whether an it's an ongoing update + // or an RoT in a failed state. + if found_pending_persistent_boot_preference.is_some() { + return MgsUpdateStatus::NotDone; + } + + // If there is a mismatch between the expected active slot and the found + // active slot then an update is not done. + if found_active_slot.slot != expected_active_slot.slot { + return MgsUpdateStatus::NotDone; + } + + // Similarly, check the contents of the inactive slot to determine if it + // still matches what we saw when we configured this update. If not, then + // this update cannot proceed as currently configured. It will fail its + // precondition check. + mgs_update_status_inactive_versions( + found_inactive_version, + expected_inactive_version, + ) +} + +/// Determine if the given baseboard needs an RoT update and, if so, returns it. +pub fn try_make_update_rot( + log: &slog::Logger, + baseboard_id: &Arc, + inventory: &Collection, + current_artifacts: &TufRepoDescription, +) -> Option { + let Some(sp_info) = inventory.sps.get(baseboard_id) else { + warn!( + log, + "cannot configure RoT update for board \ + (missing SP info from inventory)"; + baseboard_id + ); + return None; + }; + + let Some(rot_state) = inventory.rots.get(baseboard_id) else { + warn!( + log, + "cannot configure RoT update for board \ + (missing RoT state from inventory)"; + baseboard_id + ); + return None; + }; + + let active_slot = rot_state.active_slot; + + let Some(active_caboose) = inventory + .caboose_for(CabooseWhich::from_rot_slot(active_slot), baseboard_id) + else { + warn!( + log, + "cannot configure RoT update for board \ + (missing active slot {active_slot} caboose from inventory)"; + baseboard_id, + ); + return None; + }; + + let Ok(expected_active_version) = active_caboose.caboose.version.parse() + else { + warn!( + log, + "cannot configure RoT update for board \ + (cannot parse current active version as an ArtifactVersion)"; + baseboard_id, + "found_version" => &active_caboose.caboose.version, + ); + return None; + }; + + let board = &active_caboose.caboose.board; + let Some(rkth) = &active_caboose.caboose.sign else { + warn!( + log, + "cannot configure RoT update for board \ + (missing sign in caboose from inventory)"; + baseboard_id + ); + return None; + }; + + let matching_artifacts: Vec<_> = current_artifacts + .artifacts + .iter() + .filter(|a| { + // A matching RoT artifact will have: + // + // - "name" matching the board name (found above from caboose) + // - "kind" matching one of the known RoT kinds + // - "sign" matching the rkth (found above from caboose) + + if a.id.name != *board { + return false; + } + + let Some(artifact_sign) = &a.sign else { + return false; + }; + let Ok(artifact_sign) = String::from_utf8(artifact_sign.to_vec()) + else { + return false; + }; + if artifact_sign != *rkth { + return false; + } + + // We'll be updating the inactive slot, so we choose the artifact + // based on the inactive slot's kind. + match active_slot.toggled() { + RotSlot::A => { + let slot_a_artifacts = [ + ArtifactKind::GIMLET_ROT_IMAGE_A, + ArtifactKind::PSC_ROT_IMAGE_A, + ArtifactKind::SWITCH_ROT_IMAGE_A, + ]; + + if slot_a_artifacts.contains(&a.id.kind) { + return true; + } + } + RotSlot::B => { + let slot_b_artifacts = [ + ArtifactKind::GIMLET_ROT_IMAGE_B, + ArtifactKind::PSC_ROT_IMAGE_B, + ArtifactKind::SWITCH_ROT_IMAGE_B, + ]; + + if slot_b_artifacts.contains(&a.id.kind) { + return true; + } + } + } + + false + }) + .collect(); + if matching_artifacts.is_empty() { + warn!( + log, + "cannot configure RoT update for board (no matching artifact)"; + baseboard_id, + ); + return None; + } + + if matching_artifacts.len() > 1 { + // This should be impossible unless we shipped a TUF repo with more + // than 1 artifact for the same board and root key table hash (RKTH) + // that can be verified afgainst the RoT's CMPA/CFPA. But it doesn't + // prevent us from picking one and proceeding. Make a note and proceed. + error!(log, "found more than one matching artifact for RoT update"); + } + + let artifact = matching_artifacts[0]; + + // If the artifact's version matches what's deployed, then no update is + // needed. + if artifact.id.version == expected_active_version { + debug!(log, "no RoT update needed for board"; baseboard_id); + return None; + } + + let expected_active_slot = ExpectedActiveRotSlot { + slot: active_slot, + version: expected_active_version, + }; + + // Begin configuring an update. + let expected_inactive_version = match inventory + .caboose_for( + CabooseWhich::from_rot_slot(active_slot.toggled()), + baseboard_id, + ) + .map(|c| c.caboose.version.parse::()) + .transpose() + { + Ok(None) => ExpectedVersion::NoValidVersion, + Ok(Some(v)) => ExpectedVersion::Version(v), + Err(_) => { + warn!( + log, + "cannot configure RoT update for board \ + (found inactive slot contents but version was not valid)"; + baseboard_id + ); + return None; + } + }; + + Some(PendingMgsUpdate { + baseboard_id: baseboard_id.clone(), + sp_type: sp_info.sp_type, + slot_id: sp_info.sp_slot, + details: PendingMgsUpdateDetails::Rot { + expected_active_slot, + expected_inactive_version, + expected_persistent_boot_preference: rot_state + .persistent_boot_preference, + expected_pending_persistent_boot_preference: rot_state + .pending_persistent_boot_preference, + expected_transient_boot_preference: rot_state + .transient_boot_preference, + }, + artifact_hash: artifact.hash, + artifact_version: artifact.id.version.clone(), + }) +} From d4f31f93104167c703807b786db80bae983e9f91 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 8 Aug 2025 13:13:42 +1200 Subject: [PATCH 40/42] roll back cmds-target-release rot testing --- .../tests/input/cmds-target-release.txt | 60 +- .../tests/output/cmds-target-release-stdout | 840 +----------------- 2 files changed, 63 insertions(+), 837 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt b/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt index 9ac65b23274..ffc3e8dcfa0 100644 --- a/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt +++ b/dev-tools/reconfigurator-cli/tests/input/cmds-target-release.txt @@ -31,7 +31,7 @@ sled-list blueprint-list inventory-list -# First step: upgrade one RoT. +# First step: upgrade one SP. blueprint-plan dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 f45ba181-4b56-42cc-a762-874d90184a43 blueprint-diff dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -39,71 +39,39 @@ blueprint-diff dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f64 blueprint-plan 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 f45ba181-4b56-42cc-a762-874d90184a43 blueprint-diff 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 58d5e830-0884-47d8-a7cd-b2b3751adeb4 -# Now, update the simulated RoT to reflect that the update completed. +# Now, update the simulated SP to reflect that the update completed. # Collect inventory from it and use that collection for another planning step. # This should report that the update completed, remove that update, and add one -# for an SP on the same sled. -sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 -inventory-generate -blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 -blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 - -# After the RoT update has completed, we update the simulated SP to reflect that -# update has completed as well. -# Like before, collect inventory from it and use that collection for the next step. -# This should report that the update completed, remove that update, and add one # for another sled. sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 inventory-generate -blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 -blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 +blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 +blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 # This time, make it more interesting. Change the inactive slot contents of -# the simulated RoT. This should make the configured update impossible and cause +# the simulated SP. This should make the configured update impossible and cause # the planner to fix it. To test this, we also need to tell the planner not to # ignore this update even though it's quite new. set ignore-impossible-mgs-updates-since now -sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 -inventory-generate -blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 -blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba - -# Now simulate the update completing successfully. -# Like before, we should see a pending SP update for this sled. -sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 -inventory-generate -blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 -blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac - -# Now we'll change the inactive slot contents of the simulated SP. Like with the -# RoT, this should make the update impossible and cause the planner to fix it. -set ignore-impossible-mgs-updates-since now sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --inactive 0.5.0 inventory-generate -blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 -blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 +blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 +blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 -# Let's simulate the successful SP update as well. -# Another couple of planning steps should try to update the last sled. +# Now simulate the update completing successfully. +# Another planning step should try to update the last sled. sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 inventory-generate -blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 -blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 - -# Update the RoT on the last sled. -# There should be one last pending SP update. -sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 -inventory-generate -blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 -blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b +blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 +blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba # Finish updating the last sled and do one more planning run. # This should update one control plane zone. sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 inventory-generate -blueprint-plan 626487fa-7139-45ec-8416-902271fc730b 04bc9001-0836-4fec-b9cb-9d4760caf8b4 -blueprint-diff 626487fa-7139-45ec-8416-902271fc730b c1a0d242-9160-40f4-96ae-61f8f40a0b1b +blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 +blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac # We should continue walking through the update. We need to build out a # reconfigurator-cli subcommand to simulate updated zone image sources (just -# like we have sled-update-sp for simulated SP updates). +# like we have sled-update-sp for simulated SP updates). \ No newline at end of file diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 1862e3af368..8f9b044396b 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -201,7 +201,7 @@ ID NERRORS TIME_DONE f45ba181-4b56-42cc-a762-874d90184a43 0 -> # First step: upgrade one RoT. +> # First step: upgrade one SP. > blueprint-plan dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 f45ba181-4b56-42cc-a762-874d90184a43 INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 @@ -564,590 +564,17 @@ external DNS: -> # Now, update the simulated RoT to reflect that the update completed. +> # Now, update the simulated SP to reflect that the update completed. > # Collect inventory from it and use that collection for another planning step. > # This should report that the update completed, remove that update, and add one -> # for an SP on the same sled. -> sled-update-rot 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-a 1.0.0 -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 RoT settings: slot a -> 1.0.0 - -> inventory-generate -generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configured sleds - -> blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 -INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 -INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 -INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 -INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO reached maximum number of pending MGS-driven updates, max: 1 -INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 - -> blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 -from: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 -to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) - AAAA fd00:1122:3344:101::27 - name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) - AAAA fd00:1122:3344:101::22 - name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) - AAAA fd00:1122:3344:102::1 - name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) - AAAA fd00:1122:3344:102::23 - name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) - AAAA fd00:1122:3344:103::22 - name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) - AAAA fd00:1122:3344:2::1 - name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) - AAAA fd00:1122:3344:102::22 - name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) - AAAA fd00:1122:3344:101::25 - name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) - AAAA fd00:1122:3344:102::21 - name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) - AAAA fd00:1122:3344:101::21 - name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) - AAAA fd00:1122:3344:103::26 - name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) - AAAA fd00:1122:3344:102::24 - name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) - AAAA fd00:1122:3344:103::24 - name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) - AAAA fd00:1122:3344:103::27 - name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) - AAAA fd00:1122:3344:101::23 - name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) - AAAA fd00:1122:3344:102::28 - name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) - AAAA fd00:1122:3344:101::1 - name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) - AAAA fd00:1122:3344:1::1 - name: @ (records: 3) - NS ns1.control-plane.oxide.internal - NS ns2.control-plane.oxide.internal - NS ns3.control-plane.oxide.internal - name: _clickhouse-admin-single-server._tcp (records: 1) - SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse-native._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse._tcp (records: 1) - SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _crucible-pantry._tcp (records: 3) - SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal - SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal - SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal - name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) - SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal - name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) - SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal - name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) - SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal - name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) - SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal - name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) - SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal - name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) - SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal - name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) - SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal - name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) - SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal - name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) - SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal - name: _external-dns._tcp (records: 3) - SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal - SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal - SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - name: _internal-ntp._tcp (records: 3) - SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal - SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal - SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal - name: _nameservice._tcp (records: 3) - SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal - SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal - SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal - name: _nexus._tcp (records: 3) - SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal - SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal - SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal - name: _oximeter-reader._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _repo-depot._tcp (records: 3) - SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal - SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal - SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal - name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) - AAAA fd00:1122:3344:102::25 - name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) - AAAA fd00:1122:3344:101::24 - name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) - AAAA fd00:1122:3344:102::26 - name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) - AAAA fd00:1122:3344:103::1 - name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) - AAAA fd00:1122:3344:101::26 - name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) - AAAA fd00:1122:3344:102::27 - name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) - AAAA fd00:1122:3344:3::1 - name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) - AAAA fd00:1122:3344:103::21 - name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) - AAAA fd00:1122:3344:103::25 - name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) - AAAA fd00:1122:3344:103::23 - name: ns1 (records: 1) - AAAA fd00:1122:3344:1::1 - name: ns2 (records: 1) - AAAA fd00:1122:3344:2::1 - name: ns3 (records: 1) - AAAA fd00:1122:3344:3::1 - -external DNS: - DNS zone: "oxide.example" (unchanged) - name: @ (records: 3) - NS ns1.oxide.example - NS ns2.oxide.example - NS ns3.oxide.example - name: example-silo.sys (records: 3) - A 192.0.2.2 - A 192.0.2.3 - A 192.0.2.4 - name: ns1 (records: 1) - A 198.51.100.1 - name: ns2 (records: 1) - A 198.51.100.2 - name: ns3 (records: 1) - A 198.51.100.3 - - - - -> # After the RoT update has completed, we update the simulated SP to reflect that -> # update has completed as well. -> # Like before, collect inventory from it and use that collection for the next step. -> # This should report that the update completed, remove that update, and add one -> # for another sled. -> sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 - -> inventory-generate -generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds - -> blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 -INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 -INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 -INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 -INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 -INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO reached maximum number of pending MGS-driven updates, max: 1 -INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 - -> blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 -from: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 -to: blueprint df06bb57-ad42-4431-9206-abff322896c7 - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - PENDING MGS UPDATES: - - Pending MGS-managed updates (all baseboards): - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - sp_type slot part_number serial_number artifact_hash artifact_version details - ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -- sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } -+ sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) - AAAA fd00:1122:3344:101::27 - name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) - AAAA fd00:1122:3344:101::22 - name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) - AAAA fd00:1122:3344:102::1 - name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) - AAAA fd00:1122:3344:102::23 - name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) - AAAA fd00:1122:3344:103::22 - name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) - AAAA fd00:1122:3344:2::1 - name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) - AAAA fd00:1122:3344:102::22 - name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) - AAAA fd00:1122:3344:101::25 - name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) - AAAA fd00:1122:3344:102::21 - name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) - AAAA fd00:1122:3344:101::21 - name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) - AAAA fd00:1122:3344:103::26 - name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) - AAAA fd00:1122:3344:102::24 - name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) - AAAA fd00:1122:3344:103::24 - name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) - AAAA fd00:1122:3344:103::27 - name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) - AAAA fd00:1122:3344:101::23 - name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) - AAAA fd00:1122:3344:102::28 - name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) - AAAA fd00:1122:3344:101::1 - name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) - AAAA fd00:1122:3344:1::1 - name: @ (records: 3) - NS ns1.control-plane.oxide.internal - NS ns2.control-plane.oxide.internal - NS ns3.control-plane.oxide.internal - name: _clickhouse-admin-single-server._tcp (records: 1) - SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse-native._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse._tcp (records: 1) - SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _crucible-pantry._tcp (records: 3) - SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal - SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal - SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal - name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) - SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal - name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) - SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal - name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) - SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal - name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) - SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal - name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) - SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal - name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) - SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal - name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) - SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal - name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) - SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal - name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) - SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal - name: _external-dns._tcp (records: 3) - SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal - SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal - SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - name: _internal-ntp._tcp (records: 3) - SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal - SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal - SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal - name: _nameservice._tcp (records: 3) - SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal - SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal - SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal - name: _nexus._tcp (records: 3) - SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal - SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal - SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal - name: _oximeter-reader._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _repo-depot._tcp (records: 3) - SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal - SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal - SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal - name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) - AAAA fd00:1122:3344:102::25 - name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) - AAAA fd00:1122:3344:101::24 - name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) - AAAA fd00:1122:3344:102::26 - name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) - AAAA fd00:1122:3344:103::1 - name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) - AAAA fd00:1122:3344:101::26 - name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) - AAAA fd00:1122:3344:102::27 - name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) - AAAA fd00:1122:3344:3::1 - name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) - AAAA fd00:1122:3344:103::21 - name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) - AAAA fd00:1122:3344:103::25 - name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) - AAAA fd00:1122:3344:103::23 - name: ns1 (records: 1) - AAAA fd00:1122:3344:1::1 - name: ns2 (records: 1) - AAAA fd00:1122:3344:2::1 - name: ns3 (records: 1) - AAAA fd00:1122:3344:3::1 - -external DNS: - DNS zone: "oxide.example" (unchanged) - name: @ (records: 3) - NS ns1.oxide.example - NS ns2.oxide.example - NS ns3.oxide.example - name: example-silo.sys (records: 3) - A 192.0.2.2 - A 192.0.2.3 - A 192.0.2.4 - name: ns1 (records: 1) - A 198.51.100.1 - name: ns2 (records: 1) - A 198.51.100.2 - name: ns3 (records: 1) - A 198.51.100.3 - - - - -> # This time, make it more interesting. Change the inactive slot contents of -> # the simulated RoT. This should make the configured update impossible and cause -> # the planner to fix it. To test this, we also need to tell the planner not to -> # ignore this update even though it's quite new. -> set ignore-impossible-mgs-updates-since now -ignoring impossible MGS updates since - -> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-b 0.5.0 -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot b -> 0.5.0 - -> inventory-generate -generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds - -> blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 -INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 -INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 -INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 -INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO reached maximum number of pending MGS-driven updates, max: 1 -INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 - -> blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -from: blueprint df06bb57-ad42-4431-9206-abff322896c7 -to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) - AAAA fd00:1122:3344:101::27 - name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) - AAAA fd00:1122:3344:101::22 - name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) - AAAA fd00:1122:3344:102::1 - name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) - AAAA fd00:1122:3344:102::23 - name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) - AAAA fd00:1122:3344:103::22 - name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) - AAAA fd00:1122:3344:2::1 - name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) - AAAA fd00:1122:3344:102::22 - name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) - AAAA fd00:1122:3344:101::25 - name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) - AAAA fd00:1122:3344:102::21 - name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) - AAAA fd00:1122:3344:101::21 - name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) - AAAA fd00:1122:3344:103::26 - name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) - AAAA fd00:1122:3344:102::24 - name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) - AAAA fd00:1122:3344:103::24 - name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) - AAAA fd00:1122:3344:103::27 - name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) - AAAA fd00:1122:3344:101::23 - name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) - AAAA fd00:1122:3344:102::28 - name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) - AAAA fd00:1122:3344:101::1 - name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) - AAAA fd00:1122:3344:1::1 - name: @ (records: 3) - NS ns1.control-plane.oxide.internal - NS ns2.control-plane.oxide.internal - NS ns3.control-plane.oxide.internal - name: _clickhouse-admin-single-server._tcp (records: 1) - SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse-native._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse._tcp (records: 1) - SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _crucible-pantry._tcp (records: 3) - SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal - SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal - SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal - name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) - SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal - name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) - SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal - name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) - SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal - name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) - SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal - name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) - SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal - name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) - SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal - name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) - SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal - name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) - SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal - name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) - SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal - name: _external-dns._tcp (records: 3) - SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal - SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal - SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - name: _internal-ntp._tcp (records: 3) - SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal - SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal - SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal - name: _nameservice._tcp (records: 3) - SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal - SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal - SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal - name: _nexus._tcp (records: 3) - SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal - SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal - SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal - name: _oximeter-reader._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _repo-depot._tcp (records: 3) - SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal - SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal - SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal - name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) - AAAA fd00:1122:3344:102::25 - name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) - AAAA fd00:1122:3344:101::24 - name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) - AAAA fd00:1122:3344:102::26 - name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) - AAAA fd00:1122:3344:103::1 - name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) - AAAA fd00:1122:3344:101::26 - name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) - AAAA fd00:1122:3344:102::27 - name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) - AAAA fd00:1122:3344:3::1 - name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) - AAAA fd00:1122:3344:103::21 - name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) - AAAA fd00:1122:3344:103::25 - name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) - AAAA fd00:1122:3344:103::23 - name: ns1 (records: 1) - AAAA fd00:1122:3344:1::1 - name: ns2 (records: 1) - AAAA fd00:1122:3344:2::1 - name: ns3 (records: 1) - AAAA fd00:1122:3344:3::1 - -external DNS: - DNS zone: "oxide.example" (unchanged) - name: @ (records: 3) - NS ns1.oxide.example - NS ns2.oxide.example - NS ns3.oxide.example - name: example-silo.sys (records: 3) - A 192.0.2.2 - A 192.0.2.3 - A 192.0.2.4 - name: ns1 (records: 1) - A 198.51.100.1 - name: ns2 (records: 1) - A 198.51.100.2 - name: ns3 (records: 1) - A 198.51.100.3 - - - - -> # Now simulate the update completing successfully. -> # Like before, we should see a pending SP update for this sled. -> sled-update-rot 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --slot-a 1.0.0 -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c RoT settings: slot a -> 1.0.0 +> # for another sled. +> sled-update-sp 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active 1.0.0 +set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 SP versions: active -> 1.0.0 > inventory-generate -generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds +generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configured sleds -> blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 +> blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 @@ -1161,14 +588,18 @@ INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 +INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +WARN cannot configure RoT update for board (missing sign in caboose from inventory), serial_number: serial1, part_number: model1 +INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 -> blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac -from: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac +> blueprint-diff 58d5e830-0884-47d8-a7cd-b2b3751adeb4 af934083-59b5-4bf6-8966-6fb5292c29e1 +from: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 +to: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -1183,6 +614,15 @@ to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac generation: 1 (unchanged) read from:: SingleNode (unchanged) + PENDING MGS UPDATES: + + Pending MGS-managed updates (all baseboards): + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + sp_type slot part_number serial_number artifact_hash artifact_version details + ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +- sled 0 model0 serial0 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } ++ sled 1 model1 serial1 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670 1.0.0 Sp { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion } + internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) @@ -1323,8 +763,10 @@ external DNS: -> # Now we'll change the inactive slot contents of the simulated SP. Like with the -> # RoT, this should make the update impossible and cause the planner to fix it. +> # This time, make it more interesting. Change the inactive slot contents of +> # the simulated SP. This should make the configured update impossible and cause +> # the planner to fix it. To test this, we also need to tell the planner not to +> # ignore this update even though it's quite new. > set ignore-impossible-mgs-updates-since now ignoring impossible MGS updates since @@ -1332,9 +774,9 @@ ignoring impossible MGS updates since set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: inactive -> 0.5.0 > inventory-generate -generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds +generated inventory collection 61f451b3-2121-4ed6-91c7-a550054f6c21 from configured sleds -> blueprint-plan 9034c710-3e57-45f3-99e5-4316145e87ac 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 +> blueprint-plan af934083-59b5-4bf6-8966-6fb5292c29e1 61f451b3-2121-4ed6-91c7-a550054f6c21 INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 @@ -1353,11 +795,11 @@ WARN cannot configure RoT update for board (missing sign in caboose from invento INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_active_version: 0.0.1, component: sp, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 based on parent blueprint 9034c710-3e57-45f3-99e5-4316145e87ac +generated blueprint df06bb57-ad42-4431-9206-abff322896c7 based on parent blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 -> blueprint-diff 9034c710-3e57-45f3-99e5-4316145e87ac d60afc57-f15d-476c-bd0f-b1071e2bb976 -from: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac -to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 +> blueprint-diff af934083-59b5-4bf6-8966-6fb5292c29e1 df06bb57-ad42-4431-9206-abff322896c7 +from: blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 +to: blueprint df06bb57-ad42-4431-9206-abff322896c7 COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -1521,15 +963,15 @@ external DNS: -> # Let's simulate the successful SP update as well. -> # Another couple of planning steps should try to update the last sled. +> # Now simulate the update completing successfully. +> # Another planning step should try to update the last sled. > sled-update-sp 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c --active 1.0.0 set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c SP versions: active -> 1.0.0 > inventory-generate -generated inventory collection 78f72e8d-46a9-40a9-8618-602f54454d80 from configured sleds +generated inventory collection b1bda47d-2c19-4fba-96e3-d9df28db7436 from configured sleds -> blueprint-plan d60afc57-f15d-476c-bd0f-b1071e2bb976 78f72e8d-46a9-40a9-8618-602f54454d80 +> blueprint-plan df06bb57-ad42-4431-9206-abff322896c7 b1bda47d-2c19-4fba-96e3-d9df28db7436 INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 @@ -1552,11 +994,11 @@ WARN cannot configure RoT update for board (missing sign in caboose from invento INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 +generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 -> blueprint-diff d60afc57-f15d-476c-bd0f-b1071e2bb976 a5a8f242-ffa5-473c-8efd-2acf2dc0b736 -from: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 -to: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 +> blueprint-diff df06bb57-ad42-4431-9206-abff322896c7 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +from: blueprint df06bb57-ad42-4431-9206-abff322896c7 +to: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba COCKROACHDB SETTINGS: state fingerprint::::::::::::::::: (none) (unchanged) @@ -1720,199 +1162,15 @@ external DNS: -> # Update the RoT on the last sled. -> # There should be one last pending SP update. -> sled-update-rot d81c6a84-79b8-4958-ae41-ea46c9b19763 --slot-a 1.0.0 -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 RoT settings: slot a -> 1.0.0 - -> inventory-generate -generated inventory collection 39363465-89ae-4ac2-9be1-099068da9d45 from configured sleds - -> blueprint-plan a5a8f242-ffa5-473c-8efd-2acf2dc0b736 39363465-89ae-4ac2-9be1-099068da9d45 -INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 -INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO sufficient BoundaryNtp zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient Clickhouse zones exist in plan, desired_count: 1, current_count: 1 -INFO sufficient ClickhouseKeeper zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient ClickhouseServer zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CockroachDb zones exist in plan, desired_count: 0, current_count: 0 -INFO sufficient CruciblePantry zones exist in plan, desired_count: 0, current_count: 3 -INFO sufficient InternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3 -INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0 -INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: 7e6667e646ad001b54c8365a3d309c03f89c59102723d38d01697ee8079fe670, expected_inactive_version: NoValidVersion, expected_active_version: 0.0.1, component: sp, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO reached maximum number of pending MGS-driven updates, max: 1 -INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 - -> blueprint-diff a5a8f242-ffa5-473c-8efd-2acf2dc0b736 626487fa-7139-45ec-8416-902271fc730b -from: blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 -to: blueprint 626487fa-7139-45ec-8416-902271fc730b - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - name: 058fd5f9-60a8-4e11-9302-15172782e17d.host (records: 1) - AAAA fd00:1122:3344:101::27 - name: 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host (records: 1) - AAAA fd00:1122:3344:101::22 - name: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled (records: 1) - AAAA fd00:1122:3344:102::1 - name: 353b3b65-20f7-48c3-88f7-495bd5d31545.host (records: 1) - AAAA fd00:1122:3344:102::23 - name: 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host (records: 1) - AAAA fd00:1122:3344:103::22 - name: 427ec88f-f467-42fa-9bbb-66a91a36103c.host (records: 1) - AAAA fd00:1122:3344:2::1 - name: 466a9f29-62bf-4e63-924a-b9efdb86afec.host (records: 1) - AAAA fd00:1122:3344:102::22 - name: 5199c033-4cf9-4ab6-8ae7-566bd7606363.host (records: 1) - AAAA fd00:1122:3344:101::25 - name: 62620961-fc4a-481e-968b-f5acbac0dc63.host (records: 1) - AAAA fd00:1122:3344:102::21 - name: 6444f8a5-6465-4f0b-a549-1993c113569c.host (records: 1) - AAAA fd00:1122:3344:101::21 - name: 694bd14f-cb24-4be4-bb19-876e79cda2c8.host (records: 1) - AAAA fd00:1122:3344:103::26 - name: 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host (records: 1) - AAAA fd00:1122:3344:102::24 - name: 75b220ba-a0f4-4872-8202-dc7c87f062d0.host (records: 1) - AAAA fd00:1122:3344:103::24 - name: 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host (records: 1) - AAAA fd00:1122:3344:103::27 - name: 803bfb63-c246-41db-b0da-d3b87ddfc63d.host (records: 1) - AAAA fd00:1122:3344:101::23 - name: 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host (records: 1) - AAAA fd00:1122:3344:102::28 - name: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled (records: 1) - AAAA fd00:1122:3344:101::1 - name: 99e2f30b-3174-40bf-a78a-90da8abba8ca.host (records: 1) - AAAA fd00:1122:3344:1::1 - name: @ (records: 3) - NS ns1.control-plane.oxide.internal - NS ns2.control-plane.oxide.internal - NS ns3.control-plane.oxide.internal - name: _clickhouse-admin-single-server._tcp (records: 1) - SRV port 8888 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse-native._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _clickhouse._tcp (records: 1) - SRV port 8123 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _crucible-pantry._tcp (records: 3) - SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal - SRV port 17000 ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host.control-plane.oxide.internal - SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal - name: _crucible._tcp.058fd5f9-60a8-4e11-9302-15172782e17d (records: 1) - SRV port 32345 058fd5f9-60a8-4e11-9302-15172782e17d.host.control-plane.oxide.internal - name: _crucible._tcp.5199c033-4cf9-4ab6-8ae7-566bd7606363 (records: 1) - SRV port 32345 5199c033-4cf9-4ab6-8ae7-566bd7606363.host.control-plane.oxide.internal - name: _crucible._tcp.694bd14f-cb24-4be4-bb19-876e79cda2c8 (records: 1) - SRV port 32345 694bd14f-cb24-4be4-bb19-876e79cda2c8.host.control-plane.oxide.internal - name: _crucible._tcp.7c252b64-c5af-4ec1-989e-9a03f3b0f111 (records: 1) - SRV port 32345 7c252b64-c5af-4ec1-989e-9a03f3b0f111.host.control-plane.oxide.internal - name: _crucible._tcp.86a22a56-0168-453d-9df1-cb2a7c64b5d3 (records: 1) - SRV port 32345 86a22a56-0168-453d-9df1-cb2a7c64b5d3.host.control-plane.oxide.internal - name: _crucible._tcp.bd354eef-d8a6-4165-9124-283fb5e46d77 (records: 1) - SRV port 32345 bd354eef-d8a6-4165-9124-283fb5e46d77.host.control-plane.oxide.internal - name: _crucible._tcp.dfac80b4-a887-430a-ae87-a4e065dba787 (records: 1) - SRV port 32345 dfac80b4-a887-430a-ae87-a4e065dba787.host.control-plane.oxide.internal - name: _crucible._tcp.e2fdefe7-95b2-4fd2-ae37-56929a06d58c (records: 1) - SRV port 32345 e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host.control-plane.oxide.internal - name: _crucible._tcp.f55647d4-5500-4ad3-893a-df45bd50d622 (records: 1) - SRV port 32345 f55647d4-5500-4ad3-893a-df45bd50d622.host.control-plane.oxide.internal - name: _external-dns._tcp (records: 3) - SRV port 5353 6c3ae381-04f7-41ea-b0ac-74db387dbc3a.host.control-plane.oxide.internal - SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal - SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - name: _internal-ntp._tcp (records: 3) - SRV port 123 62620961-fc4a-481e-968b-f5acbac0dc63.host.control-plane.oxide.internal - SRV port 123 6444f8a5-6465-4f0b-a549-1993c113569c.host.control-plane.oxide.internal - SRV port 123 f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host.control-plane.oxide.internal - name: _nameservice._tcp (records: 3) - SRV port 5353 427ec88f-f467-42fa-9bbb-66a91a36103c.host.control-plane.oxide.internal - SRV port 5353 99e2f30b-3174-40bf-a78a-90da8abba8ca.host.control-plane.oxide.internal - SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal - name: _nexus._tcp (records: 3) - SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal - SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal - SRV port 12221 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal - name: _oximeter-reader._tcp (records: 1) - SRV port 9000 353b3b65-20f7-48c3-88f7-495bd5d31545.host.control-plane.oxide.internal - name: _repo-depot._tcp (records: 3) - SRV port 12348 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c.sled.control-plane.oxide.internal - SRV port 12348 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6.sled.control-plane.oxide.internal - SRV port 12348 d81c6a84-79b8-4958-ae41-ea46c9b19763.sled.control-plane.oxide.internal - name: ad6a3a03-8d0f-4504-99a4-cbf73d69b973.host (records: 1) - AAAA fd00:1122:3344:102::25 - name: ba4994a8-23f9-4b1a-a84f-a08d74591389.host (records: 1) - AAAA fd00:1122:3344:101::24 - name: bd354eef-d8a6-4165-9124-283fb5e46d77.host (records: 1) - AAAA fd00:1122:3344:102::26 - name: d81c6a84-79b8-4958-ae41-ea46c9b19763.sled (records: 1) - AAAA fd00:1122:3344:103::1 - name: dfac80b4-a887-430a-ae87-a4e065dba787.host (records: 1) - AAAA fd00:1122:3344:101::26 - name: e2fdefe7-95b2-4fd2-ae37-56929a06d58c.host (records: 1) - AAAA fd00:1122:3344:102::27 - name: ea5b4030-b52f-44b2-8d70-45f15f987d01.host (records: 1) - AAAA fd00:1122:3344:3::1 - name: f10a4fb9-759f-4a65-b25e-5794ad2d07d8.host (records: 1) - AAAA fd00:1122:3344:103::21 - name: f55647d4-5500-4ad3-893a-df45bd50d622.host (records: 1) - AAAA fd00:1122:3344:103::25 - name: f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host (records: 1) - AAAA fd00:1122:3344:103::23 - name: ns1 (records: 1) - AAAA fd00:1122:3344:1::1 - name: ns2 (records: 1) - AAAA fd00:1122:3344:2::1 - name: ns3 (records: 1) - AAAA fd00:1122:3344:3::1 - -external DNS: - DNS zone: "oxide.example" (unchanged) - name: @ (records: 3) - NS ns1.oxide.example - NS ns2.oxide.example - NS ns3.oxide.example - name: example-silo.sys (records: 3) - A 192.0.2.2 - A 192.0.2.3 - A 192.0.2.4 - name: ns1 (records: 1) - A 198.51.100.1 - name: ns2 (records: 1) - A 198.51.100.2 - name: ns3 (records: 1) - A 198.51.100.3 - - - - > # Finish updating the last sled and do one more planning run. > # This should update one control plane zone. > sled-update-sp d81c6a84-79b8-4958-ae41-ea46c9b19763 --active 1.0.0 set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 SP versions: active -> 1.0.0 > inventory-generate -generated inventory collection 04bc9001-0836-4fec-b9cb-9d4760caf8b4 from configured sleds +generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds -> blueprint-plan 626487fa-7139-45ec-8416-902271fc730b 04bc9001-0836-4fec-b9cb-9d4760caf8b4 +> blueprint-plan 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba a71f7a73-35a6-45e8-acbe-f1c5925eed69 INFO performed noop image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 INFO performed noop image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO performed noop image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 @@ -1936,11 +1194,11 @@ INFO skipping board for MGS-driven update, serial_number: serial1, part_number: INFO ran out of boards for MGS-driven update INFO updating zone image source in-place, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone_id: 353b3b65-20f7-48c3-88f7-495bd5d31545, kind: Clickhouse, image_source: artifact: version 1.0.0 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify -generated blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b based on parent blueprint 626487fa-7139-45ec-8416-902271fc730b +generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -> blueprint-diff 626487fa-7139-45ec-8416-902271fc730b c1a0d242-9160-40f4-96ae-61f8f40a0b1b -from: blueprint 626487fa-7139-45ec-8416-902271fc730b -to: blueprint c1a0d242-9160-40f4-96ae-61f8f40a0b1b +> blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba 9034c710-3e57-45f3-99e5-4316145e87ac +from: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba +to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac MODIFIED SLEDS: From 5da11b0eb24cea7e7232a1bdc2b7f71eca6af417 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 11 Aug 2025 17:00:18 +1200 Subject: [PATCH 41/42] Address comments --- .../planning/src/mgs_updates/mod.rs | 344 +++++++++++++++--- .../planning/src/mgs_updates/rot.rs | 48 +-- nexus/reconfigurator/planning/src/system.rs | 17 +- 3 files changed, 310 insertions(+), 99 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index ab54387f1c9..417fff43964 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -703,15 +703,23 @@ mod test { } fn test_artifact_for_artifact_kind(kind: ArtifactKind) -> ArtifactHash { - match kind.as_str() { - "gimlet_rot_image_a" => ARTIFACT_HASH_ROT_GIMLET_A, - "gimlet_rot_image_b" => ARTIFACT_HASH_ROT_GIMLET_B, - "psc_rot_image_a" => ARTIFACT_HASH_ROT_PSC_A, - "psc_rot_image_b" => ARTIFACT_HASH_ROT_PSC_B, - "switch_rot_image_a" => ARTIFACT_HASH_ROT_SWITCH_A, - "switch_rot_image_b" => ARTIFACT_HASH_ROT_SWITCH_B, - _ => panic!("test bug: no artifact for artifact kind {kind:?}"), - } + let artifact_hash = if kind == ArtifactKind::GIMLET_ROT_IMAGE_A { + ARTIFACT_HASH_ROT_GIMLET_A + } else if kind == ArtifactKind::GIMLET_ROT_IMAGE_B { + ARTIFACT_HASH_ROT_GIMLET_B + } else if kind == ArtifactKind::PSC_ROT_IMAGE_A { + ARTIFACT_HASH_ROT_PSC_A + } else if kind == ArtifactKind::PSC_ROT_IMAGE_B { + ARTIFACT_HASH_ROT_PSC_B + } else if kind == ArtifactKind::SWITCH_ROT_IMAGE_A { + ARTIFACT_HASH_ROT_SWITCH_A + } else if kind == ArtifactKind::SWITCH_ROT_IMAGE_B { + ARTIFACT_HASH_ROT_SWITCH_B + } else { + panic!("test bug: no artifact for artifact kind {kind:?}") + }; + + return artifact_hash; } /// Describes the SPs and RoTs in the environment used in these tests @@ -1078,11 +1086,12 @@ mod test { builder.build() } - // Short hand-rolled update sequence that exercises some basic behavior. + // Short hand-rolled update sequence that exercises some basic behavior for + // SP updates. #[test] - fn test_basic() { + fn test_basic_sp() { let logctx = LogContext::new( - "planning_mgs_updates_basic", + "planning_mgs_updates_basic_sp", &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, ); let log = &logctx.log; @@ -1146,13 +1155,41 @@ mod test { ); assert_eq!(updates, later_updates); - // Now we make sure the RoT is updated as well + // Test that when two updates are needed, but one is already pending, + // then the other one is *not* started (because it exceeds + // nmax_updates). let later_collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::from([ + ((SpType::Sled, 0), ARTIFACT_VERSION_1), + ((SpType::Switch, 1), ARTIFACT_VERSION_1), + ]), + ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, &BTreeMap::new(), ExpectedVersion::NoValidVersion, + ); + let later_updates = plan_mgs_updates( + log, + &later_collection, + current_boards, + &updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert_eq!(updates, later_updates); + + // At this point, we're ready to test that when the first update + // completes, then the second one *is* started. This tests two + // different things: first that we noticed the first one completed, and + // second that we noticed another thing needed an update + let later_collection = make_collection( ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( @@ -1168,32 +1205,230 @@ mod test { let next_update = later_updates.iter().next().expect("at least one update"); assert_ne!(first_update, next_update); - assert_eq!(next_update.baseboard_id.serial_number, "sled_0"); - assert_eq!(next_update.sp_type, SpType::Sled); - assert_eq!(next_update.slot_id, 0); - assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_ROT_GIMLET_B); + assert_eq!(next_update.baseboard_id.serial_number, "switch_1"); + assert_eq!(next_update.sp_type, SpType::Switch); + assert_eq!(next_update.slot_id, 1); + assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); - // Test that when two updates for two SpTypes are needed, but one is - // already pending, then the other one is *not* started (because it - // exceeds nmax_updates). - let later_collection = make_collection( + // Finally, test that when all SPs are in spec, then no updates are + // configured. + let updated_collection = make_collection( ARTIFACT_VERSION_2, - &BTreeMap::from([ - ((SpType::Sled, 0), ARTIFACT_VERSION_1), - ((SpType::Switch, 1), ARTIFACT_VERSION_1), - ]), + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, - &BTreeMap::from([ - ((SpType::Sled, 0), ARTIFACT_VERSION_1), - ((SpType::Switch, 1), ARTIFACT_VERSION_1), - ]), + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( log, - &later_collection, + &updated_collection, + current_boards, + &later_updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert!(later_updates.is_empty()); + + // Test that we don't try to update boards that aren't in + // `current_boards`, even if they're in inventory and outdated. + let collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, + ); + let updates = plan_mgs_updates( + log, + &collection, + &BTreeSet::new(), + &PendingMgsUpdates::new(), + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert!(updates.is_empty()); + let updates = plan_mgs_updates( + log, + &collection, + &collection.baseboards, + &PendingMgsUpdates::new(), + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + // We verified most of the details above. Here we're just double + // checking that the baseboard being missing is the only reason that no + // update was generated. + assert_eq!(updates.len(), 1); + + // Verify the precondition details of an ordinary update. + let old_update = + updates.into_iter().next().expect("at least one update"); + let PendingMgsUpdateDetails::Sp { + expected_active_version: old_expected_active_version, + expected_inactive_version: old_expected_inactive_version, + } = &old_update.details + else { + panic!("expected SP update"); + }; + assert_eq!(ARTIFACT_VERSION_1, *old_expected_active_version); + assert_eq!( + ExpectedVersion::NoValidVersion, + *old_expected_inactive_version + ); + + // Test that if the inactive slot contents have changed, then we'll get + // a new update reflecting that. + let collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::Version(ARTIFACT_VERSION_1), + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, + ); + let new_updates = plan_mgs_updates( + log, + &collection, + &collection.baseboards, + &updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert_ne!(updates, new_updates); + assert_eq!(new_updates.len(), 1); + let new_update = + new_updates.into_iter().next().expect("at least one update"); + assert_eq!(old_update.baseboard_id, new_update.baseboard_id); + assert_eq!(old_update.sp_type, new_update.sp_type); + assert_eq!(old_update.slot_id, new_update.slot_id); + assert_eq!(old_update.artifact_hash, new_update.artifact_hash); + assert_eq!(old_update.artifact_version, new_update.artifact_version); + let PendingMgsUpdateDetails::Sp { + expected_active_version: new_expected_active_version, + expected_inactive_version: new_expected_inactive_version, + } = &new_update.details + else { + panic!("expected SP update"); + }; + assert_eq!(ARTIFACT_VERSION_1, *new_expected_active_version); + assert_eq!( + ExpectedVersion::Version(ARTIFACT_VERSION_1), + *new_expected_inactive_version + ); + + // Test that if instead it's the active slot whose contents have changed + // to something other than the new expected version, then we'll also get + // a new update reflecting that. + let collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1_5)]), + ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, + ); + let new_updates = plan_mgs_updates( + log, + &collection, + &collection.baseboards, + &updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert_ne!(updates, new_updates); + assert_eq!(new_updates.len(), 1); + let new_update = + new_updates.into_iter().next().expect("at least one update"); + assert_eq!(old_update.baseboard_id, new_update.baseboard_id); + assert_eq!(old_update.sp_type, new_update.sp_type); + assert_eq!(old_update.slot_id, new_update.slot_id); + assert_eq!(old_update.artifact_hash, new_update.artifact_hash); + assert_eq!(old_update.artifact_version, new_update.artifact_version); + let PendingMgsUpdateDetails::Sp { + expected_active_version: new_expected_active_version, + expected_inactive_version: new_expected_inactive_version, + } = &new_update.details + else { + panic!("expected SP update"); + }; + assert_eq!(ARTIFACT_VERSION_1_5, *new_expected_active_version); + assert_eq!( + ExpectedVersion::NoValidVersion, + *new_expected_inactive_version + ); + + logctx.cleanup_successful(); + } + + // Short hand-rolled update sequence that exercises some basic behavior for + // RoT updates. + #[test] + fn test_basic_rot() { + let logctx = LogContext::new( + "planning_mgs_updates_basic_rot", + &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, + ); + let log = &logctx.log; + + // Test that with no updates pending and no TUF repo specified, there + // will remain no updates pending. + let collection = make_collection( + ARTIFACT_VERSION_2, + &BTreeMap::new(), + ExpectedVersion::NoValidVersion, + ARTIFACT_VERSION_2, + &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + ExpectedVersion::NoValidVersion, + ); + let current_boards = &collection.baseboards; + let initial_updates = PendingMgsUpdates::new(); + let nmax_updates = 1; + let impossible_update_policy = ImpossibleUpdatePolicy::Reevaluate; + let updates = plan_mgs_updates( + log, + &collection, + current_boards, + &initial_updates, + &TargetReleaseDescription::Initial, + nmax_updates, + impossible_update_policy, + ); + assert!(updates.is_empty()); + + // Test that when a TUF repo is specified and one RoT is outdated, then + // it's configured with an update (and the update looks correct). + let repo = make_tuf_repo(); + let updates = plan_mgs_updates( + log, + &collection, + current_boards, + &initial_updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + assert_eq!(updates.len(), 1); + let first_update = updates.iter().next().expect("at least one update"); + assert_eq!(first_update.baseboard_id.serial_number, "sled_0"); + assert_eq!(first_update.sp_type, SpType::Sled); + assert_eq!(first_update.slot_id, 0); + assert_eq!(first_update.artifact_hash, ARTIFACT_HASH_ROT_GIMLET_B); + assert_eq!(first_update.artifact_version, ARTIFACT_VERSION_2); + + // Test that when an update is already pending, and nothing changes + // about the state of the world (i.e., the inventory), then the planner + // makes no changes. + let later_updates = plan_mgs_updates( + log, + &collection, current_boards, &updates, &TargetReleaseDescription::TufRepo(repo.clone()), @@ -1202,18 +1437,18 @@ mod test { ); assert_eq!(updates, later_updates); - // At this point, we're ready to test that when the first SpType update - // completes, then the second one *is* started. This tests three - // different things: first that we noticed the first one completed, - // second that we noticed another thing needed an update, and third that - // the planner schedules the updates in the correct order: first RoT, - // and second SP. + // Test that when two updates are needed, but one is already pending, + // then the other one is *not* started (because it exceeds + // nmax_updates). let later_collection = make_collection( ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), + &BTreeMap::from([ + ((SpType::Sled, 0), ARTIFACT_VERSION_1), + ((SpType::Switch, 1), ARTIFACT_VERSION_1), + ]), ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( @@ -1225,23 +1460,20 @@ mod test { nmax_updates, impossible_update_policy, ); - assert_eq!(later_updates.len(), 1); - let next_update = - later_updates.iter().next().expect("at least one update"); - assert_ne!(first_update, next_update); - assert_eq!(next_update.baseboard_id.serial_number, "switch_1"); - assert_eq!(next_update.sp_type, SpType::Switch); - assert_eq!(next_update.slot_id, 1); - assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_ROT_SWITCH_B); - assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); + assert_eq!(updates, later_updates); - // Same test but for SP only. + // At this point, we're ready to test that when the first SpType update + // completes, then the second one *is* started. This tests three + // different things: first that we noticed the first one completed, + // second that we noticed another thing needed an update, and third that + // the planner schedules the updates in the correct order: first RoT, + // and second SP. let later_collection = make_collection( ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, - &BTreeMap::new(), + &BTreeMap::from([((SpType::Switch, 1), ARTIFACT_VERSION_1)]), ExpectedVersion::NoValidVersion, ); let later_updates = plan_mgs_updates( @@ -1260,10 +1492,10 @@ mod test { assert_eq!(next_update.baseboard_id.serial_number, "switch_1"); assert_eq!(next_update.sp_type, SpType::Switch); assert_eq!(next_update.slot_id, 1); - assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_SP_SIDECAR_C); + assert_eq!(next_update.artifact_hash, ARTIFACT_HASH_ROT_SWITCH_B); assert_eq!(next_update.artifact_version, ARTIFACT_VERSION_2); - // Finally, test that when all RoTs and SPs are in spec, then no updates + // Finally, test that when all components are in spec, then no updates // are configured. let updated_collection = make_collection( ARTIFACT_VERSION_2, @@ -1288,7 +1520,7 @@ mod test { // `current_boards`, even if they're in inventory and outdated. let collection = make_collection( ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), @@ -1318,7 +1550,7 @@ mod test { // update was generated. assert_eq!(updates.len(), 1); - // Verify the precondition details of an ordinary update. + // Verify the precondition details of an ordinary RoT update. let old_update = updates.into_iter().next().expect("at least one update"); let PendingMgsUpdateDetails::Rot { @@ -1339,7 +1571,7 @@ mod test { // a new update reflecting that. let collection = make_collection( ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), + &BTreeMap::new(), ExpectedVersion::Version(ARTIFACT_VERSION_1), ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1)]), @@ -1382,7 +1614,7 @@ mod test { // a new update reflecting that. let collection = make_collection( ARTIFACT_VERSION_2, - &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1_5)]), + &BTreeMap::new(), ExpectedVersion::NoValidVersion, ARTIFACT_VERSION_2, &BTreeMap::from([((SpType::Sled, 0), ARTIFACT_VERSION_1_5)]), diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index cfcf0535537..510d7cd320d 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -16,11 +16,12 @@ use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; use omicron_common::api::external::TufRepoDescription; -use slog::{debug, error, warn}; +use slog::{debug, warn}; use std::sync::Arc; use tufaceous_artifact::ArtifactKind; use tufaceous_artifact::ArtifactVersion; +/// RoT state that gets checked against preconditions in a `PendingMgsUpdate` pub struct RotUpdateState { pub active_slot: ExpectedActiveRotSlot, pub persistent_boot_preference: RotSlot, @@ -62,6 +63,16 @@ pub fn mgs_update_status_rot( // preference, pending persistent boot preference, and transient boot // preference are still what they were when we configured this update. // If not, then this update cannot proceed as currently configured. + if found_active_slot.version() != expected_active_slot.version() + || found_persistent_boot_preference + != expected_persistent_boot_preference + || found_pending_persistent_boot_preference + != expected_pending_persistent_boot_preference + || found_transient_boot_preference != expected_transient_boot_preference + { + return MgsUpdateStatus::Impossible; + } + // If there is a mismatch between the found persistent boot preference // and the found active slot then this means a failed update. We cannot // proceed. @@ -69,36 +80,13 @@ pub fn mgs_update_status_rot( // Transient boot preference is not in use yet, if we find that it is set we // should not proceed. Once https://github.com/oxidecomputer/hubris/pull/2050 // is implemented, we should revist this check - if found_active_slot.version() != expected_active_slot.version() - || found_persistent_boot_preference - != expected_persistent_boot_preference - || found_pending_persistent_boot_preference - != expected_pending_persistent_boot_preference - || found_transient_boot_preference != expected_transient_boot_preference - || found_persistent_boot_preference != found_active_slot.slot + if found_persistent_boot_preference != found_active_slot.slot || found_transient_boot_preference.is_some() || expected_transient_boot_preference.is_some() { return MgsUpdateStatus::Impossible; } - // If found pending persistent boot preference is not empty, then an update - // is not done. - // - // TODO: Alternatively, this could also mean a failed update. See - // https://github.com/oxidecomputer/omicron/issues/8414 for context - // about when we'll be able to know whether an it's an ongoing update - // or an RoT in a failed state. - if found_pending_persistent_boot_preference.is_some() { - return MgsUpdateStatus::NotDone; - } - - // If there is a mismatch between the expected active slot and the found - // active slot then an update is not done. - if found_active_slot.slot != expected_active_slot.slot { - return MgsUpdateStatus::NotDone; - } - // Similarly, check the contents of the inactive slot to determine if it // still matches what we saw when we configured this update. If not, then // this update cannot proceed as currently configured. It will fail its @@ -238,11 +226,11 @@ pub fn try_make_update_rot( } if matching_artifacts.len() > 1 { - // This should be impossible unless we shipped a TUF repo with more - // than 1 artifact for the same board and root key table hash (RKTH) - // that can be verified afgainst the RoT's CMPA/CFPA. But it doesn't - // prevent us from picking one and proceeding. Make a note and proceed. - error!(log, "found more than one matching artifact for RoT update"); + // This should be impossible unless we shipped a TUF repo with more than + // 1 artifact for the same board and root key table hash (RKTH) as + // `SIGN`. But it doesn't prevent us from picking one and proceeding. + // Make a note and proceed. + warn!(log, "found more than one matching artifact for RoT update"); } let artifact = matching_artifacts[0]; diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 54fbde63c45..f9a7a45a141 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -589,9 +589,7 @@ impl SystemDescription { &self, sled_id: SledUuid, ) -> anyhow::Result> { - let sled = self.sleds.get(&sled_id).with_context(|| { - format!("attempted to access sled {} not found in system", sled_id) - })?; + let sled = self.get_sled(sled_id)?; Ok(sled.sp_state()) } @@ -604,10 +602,7 @@ impl SystemDescription { slot_a_version: Option, slot_b_version: Option, ) -> anyhow::Result<&mut Self> { - let sled = self.sleds.get_mut(&sled_id).with_context(|| { - format!("attempted to access sled {} not found in system", sled_id) - })?; - let sled = Arc::make_mut(sled); + let sled = self.get_sled_mut(sled_id)?; sled.set_rot_versions(slot_a_version, slot_b_version); Ok(self) } @@ -699,9 +694,7 @@ impl SystemDescription { &self, sled_id: SledUuid, ) -> anyhow::Result> { - let sled = self.sleds.get(&sled_id).with_context(|| { - format!("attempted to access sled {} not found in system", sled_id) - })?; + let sled = self.get_sled(sled_id)?; Ok(sled.rot_slot_a_caboose().map(|c| c.version.as_ref())) } @@ -709,9 +702,7 @@ impl SystemDescription { &self, sled_id: SledUuid, ) -> anyhow::Result> { - let sled = self.sleds.get(&sled_id).with_context(|| { - format!("attempted to access sled {} not found in system", sled_id) - })?; + let sled = self.get_sled(sled_id)?; Ok(sled.rot_slot_b_caboose().map(|c| c.version.as_ref())) } From 82b45e09a542f13ac01910e315447647625d1c93 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 12 Aug 2025 08:31:56 +1200 Subject: [PATCH 42/42] typo --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 417fff43964..885e302dd00 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1942,7 +1942,7 @@ mod test { impossible_update_policy, ); // `all_updates` counts each update per SpType. This means an update for - // SP and RoT for the same SpType count as a sinlge update. For + // SP and RoT for the same SpType count as a single update. For // `expected_updates`, each component update counts as an update, so the // amount of `all_updates` should be half of `expected_updates`. assert_eq!(all_updates.len(), expected_updates.len() / 2);