From 5b0d884794dc178acd5a3f4bf7853fd2f69ab57d Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 29 Aug 2025 16:04:39 -0600 Subject: [PATCH 01/55] scaffolding --- .../planning/src/mgs_updates/test_helpers.rs | 22 +----- nexus/reconfigurator/planning/src/planner.rs | 5 ++ nexus/types/src/deployment.rs | 2 +- nexus/types/src/deployment/planning_report.rs | 72 ++++++++++++++++++- 4 files changed, 77 insertions(+), 24 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs index abef9ce70fd..500c7728747 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs @@ -38,6 +38,7 @@ use nexus_types::deployment::PendingMgsUpdateHostPhase1Details; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; +use nexus_types::deployment::planning_report::MgsUpdateComponent; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; use omicron_common::api::external::Generation; @@ -140,27 +141,6 @@ const ROT_SIGN_PSC: &str = const ROT_SIGN_SWITCH: &str = "3333333333333333333333333333333333333333333333333333333333333333"; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] -pub(super) enum MgsUpdateComponent { - Sp, - Rot, - RotBootloader, - HostOs, -} - -impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { - fn from(value: &'_ PendingMgsUpdateDetails) -> Self { - match value { - PendingMgsUpdateDetails::Rot { .. } => Self::Rot, - PendingMgsUpdateDetails::RotBootloader { .. } => { - Self::RotBootloader - } - PendingMgsUpdateDetails::Sp { .. } => Self::Sp, - PendingMgsUpdateDetails::HostPhase1(_) => Self::HostOs, - } - } -} - /// Description of a single fake board (sled, switch, or PSC). #[derive(Debug)] pub(super) struct TestBoard { diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 0731ba5350e..895fa30f2e2 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -191,6 +191,7 @@ impl<'a> Planner<'a> { // Only plan MGS-based updates updates if there are no outstanding // MUPdate overrides. + // TODO-K: mgs updates here let mgs_updates = if add_update_blocked_reasons.is_empty() { self.do_plan_mgs_updates()? } else { @@ -216,10 +217,14 @@ impl<'a> Planner<'a> { PlanningZoneUpdatesStepReport::waiting_on( ZoneUpdatesWaitingOn::DiscretionaryZones, ) + // TODO-K: mgs_updates can be empty if some updates failed. Make sure to + // keep track of failed updates and wait until they are on the expected + // new version befor proceeding to update zones } else if !mgs_updates.is_empty() { // ... or if there are still pending updates for the RoT / SP / // Host OS / etc. ... // TODO This is not quite right. See oxidecomputer/omicron#8285. + // TODO-K: change here? PlanningZoneUpdatesStepReport::waiting_on( ZoneUpdatesWaitingOn::PendingMgsUpdates, ) diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 5928daf9363..421a89ff23b 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -72,7 +72,7 @@ mod clickhouse; pub mod execution; mod network_resources; mod planning_input; -mod planning_report; +pub mod planning_report; mod zone_type; use crate::inventory::BaseboardId; diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 1ed30442abc..664e7c08e34 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -10,6 +10,8 @@ use super::BlueprintZoneImageSource; use super::CockroachDbPreserveDowngrade; use super::PendingMgsUpdates; use super::PlannerChickenSwitches; +use crate::deployment::PendingMgsUpdateDetails; +use crate::inventory::BaseboardId; use daft::Diffable; use indent_write::fmt::IndentWriter; @@ -473,16 +475,81 @@ impl PlanningMupdateOverrideStepReport { } } +#[derive( + Debug, + Deserialize, + Serialize, + PartialEq, + Eq, + Diffable, + PartialOrd, + JsonSchema, + Ord, + Clone, + Copy, +)] +pub enum MgsUpdateComponent { + Sp, + Rot, + RotBootloader, + HostOs, +} + +impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { + fn from(value: &'_ PendingMgsUpdateDetails) -> Self { + match value { + PendingMgsUpdateDetails::Rot { .. } => Self::Rot, + PendingMgsUpdateDetails::RotBootloader { .. } => { + Self::RotBootloader + } + PendingMgsUpdateDetails::Sp { .. } => Self::Sp, + PendingMgsUpdateDetails::HostPhase1(_) => Self::HostOs, + } + } +} + +#[derive( + Debug, + Deserialize, + Serialize, + PartialEq, + Eq, + Diffable, + PartialOrd, + JsonSchema, + Ord, + Clone, + Copy, +)] +pub enum FailedMgsUpdateReason { + MissingArtifact, + // Add more +} + +#[derive( + Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, +)] +pub struct FailedMgsUpdate { + pub baseboard_id: BaseboardId, + pub component: MgsUpdateComponent, + pub reason: FailedMgsUpdateReason, +} + #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] pub struct PlanningMgsUpdatesStepReport { pub pending_mgs_updates: PendingMgsUpdates, + // TODO-K: keep the component here that we were unable to update + // and the reason why + // add a comment + pub failed_mgs_update: Option, } impl PlanningMgsUpdatesStepReport { pub fn new(pending_mgs_updates: PendingMgsUpdates) -> Self { - Self { pending_mgs_updates } + // TODO-K: actually include the failed update + Self { pending_mgs_updates, failed_mgs_update: None } } pub fn is_empty(&self) -> bool { @@ -492,7 +559,8 @@ impl PlanningMgsUpdatesStepReport { impl fmt::Display for PlanningMgsUpdatesStepReport { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let Self { pending_mgs_updates } = self; + // TODO-K: implement display + let Self { pending_mgs_updates, .. } = self; if !pending_mgs_updates.is_empty() { let n = pending_mgs_updates.len(); let s = plural(n); From 477cb9b3910319b9a9c5e7d47f7c5478132961fc Mon Sep 17 00:00:00 2001 From: karencfv Date: Sat, 30 Aug 2025 00:53:00 -0600 Subject: [PATCH 02/55] Some thoughts --- nexus/types/src/deployment/planning_report.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 664e7c08e34..f4278d9d633 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -543,6 +543,11 @@ pub struct PlanningMgsUpdatesStepReport { // TODO-K: keep the component here that we were unable to update // and the reason why // add a comment + // + // TODO-K: Maybe we want to check that all updates are in the version + // we want, instead of keeping track of failed ones? Because an in-flight + // update may have been removed from pending updates and won't be on + // failed updates either pub failed_mgs_update: Option, } From 9f4968da4a6aca2ffcfeda81e8ada5222eb18f6e Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 2 Sep 2025 18:10:55 -0600 Subject: [PATCH 03/55] Implement functionality for RoT bootloader --- .../planning/src/mgs_updates/host_phase_1.rs | 5 +- .../planning/src/mgs_updates/mod.rs | 81 ++++++++++++++++--- .../src/mgs_updates/rot_bootloader.rs | 23 +++--- nexus/types/src/deployment/planning_report.rs | 51 ++++++++++-- 4 files changed, 130 insertions(+), 30 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index ba84e858ea6..2bdb01d1183 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -12,6 +12,7 @@ use nexus_types::deployment::BlueprintHostPhase2DesiredContents; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateHostPhase1Details; +use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use omicron_common::api::external::TufArtifactMeta; @@ -270,7 +271,7 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes)> { +) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( log, @@ -506,6 +507,8 @@ pub(super) fn try_make_update( artifact_version: phase_1_artifact.id.version.clone(), }, pending_host_phase_2_changes, + // TODO-K: This is wrong, fix + SkippedMgsUpdates::empty(), )) } diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index e614ef2d402..f0bbad83c5c 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -27,6 +27,9 @@ use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; +use nexus_types::deployment::planning_report::MgsUpdateComponent; +use nexus_types::deployment::planning_report::SkippedMgsUpdate; +use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -64,6 +67,8 @@ pub(crate) struct PlannedMgsUpdates { /// Pending changes to sleds' host phase 2 contents; each of these should /// result in a change to the respective sled's `BlueprintSledConfig`. pub(crate) pending_host_phase_2_changes: PendingHostPhase2Changes, + + // TODO-K: Add skipped updates here? } /// Generates a new set of `PendingMgsUpdates` based on: @@ -198,7 +203,8 @@ pub(crate) fn plan_mgs_updates( } match try_make_update(log, board, inventory, current_artifacts) { - Some((update, mut host_phase_2)) => { + // TODO-K: use skipped_updates + Some((update, mut host_phase_2, _skipped_updates)) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); @@ -495,27 +501,76 @@ fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes)> { +) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { + let mut skipped_mgs_updates = SkippedMgsUpdates::empty(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". - if let Some(update) = try_make_update_rot_bootloader( + match try_make_update_rot_bootloader( log, baseboard_id, inventory, current_artifacts, - ) - .or_else(|| { - try_make_update_rot(log, baseboard_id, inventory, current_artifacts) - }) - .or_else(|| { - try_make_update_sp(log, baseboard_id, inventory, current_artifacts) - }) { - // We have a non-host update; there are no pending host phase 2 changes - // necessary. - return Some((update, PendingHostPhase2Changes::empty())); + ) { + // TODO-K: Will have to clean this up, the nesting will become horrible + Ok(p) => { + if let Some(update) = p + .or_else(|| { + try_make_update_rot( + log, + baseboard_id, + inventory, + current_artifacts, + ) + }) + .or_else(|| { + try_make_update_sp( + log, + baseboard_id, + inventory, + current_artifacts, + ) + }) + { + // We have a non-host update; there are no pending host phase 2 changes + // necessary. + return Some(( + update, + PendingHostPhase2Changes::empty(), + skipped_mgs_updates, + )); + } + } + Err(e) => { + // TODO-K: Remove unwrap + skipped_mgs_updates + .by_baseboard + .insert_unique(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::RotBootloader, + reason: e, + }) + .unwrap(); + } } + // if let Some(update) = try_make_update_rot_bootloader( + // log, + // baseboard_id, + // inventory, + // current_artifacts, + // ) + // .or_else(|| { + // try_make_update_rot(log, baseboard_id, inventory, current_artifacts) + // }) + // .or_else(|| { + // try_make_update_sp(log, baseboard_id, inventory, current_artifacts) + // }) { + // // We have a non-host update; there are no pending host phase 2 changes + // // necessary. + // return Some((update, PendingHostPhase2Changes::empty())); + // } + host_phase_1::try_make_update( log, baseboard_id, diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index a82484ae6ba..659c7692b5f 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -11,6 +11,7 @@ use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; +use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -60,7 +61,9 @@ pub fn try_make_update_rot_bootloader( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> Option { + // TODO-K: This needs to stay as an option because Ok(None) means there is + // update needed +) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( log, @@ -68,7 +71,7 @@ pub fn try_make_update_rot_bootloader( (missing SP info from inventory)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::SpNotInInventory); }; let Some(stage0_caboose) = @@ -80,7 +83,7 @@ pub fn try_make_update_rot_bootloader( (missing stage0 caboose from inventory)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::CabooseNotInInventory); }; let Ok(expected_stage0_version) = stage0_caboose.caboose.version.parse() @@ -92,7 +95,7 @@ pub fn try_make_update_rot_bootloader( baseboard_id, "found_version" => &stage0_caboose.caboose.version, ); - return None; + return Err(FailedMgsUpdateReason::FailedVersionParse); }; let board = &stage0_caboose.caboose.board; @@ -103,7 +106,7 @@ pub fn try_make_update_rot_bootloader( (missing sign in stage0 caboose from inventory)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::CabooseMissingSign); }; let matching_artifacts: Vec<_> = current_artifacts @@ -160,7 +163,7 @@ pub fn try_make_update_rot_bootloader( "cannot configure RoT bootloader update for board (no matching artifact)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } if matching_artifacts.len() > 1 { @@ -180,7 +183,7 @@ pub fn try_make_update_rot_bootloader( // needed. if artifact.id.version == expected_stage0_version { debug!(log, "no RoT bootloader update needed for board"; baseboard_id); - return None; + return Ok(None); } // Begin configuring an update. @@ -198,11 +201,11 @@ pub fn try_make_update_rot_bootloader( (found stage0 next contents but version was not valid)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::FailedVersionParse); } }; - Some(PendingMgsUpdate { + Ok(Some(PendingMgsUpdate { baseboard_id: baseboard_id.clone(), sp_type: sp_info.sp_type, slot_id: sp_info.sp_slot, @@ -214,5 +217,5 @@ pub fn try_make_update_rot_bootloader( ), artifact_hash: artifact.hash, artifact_version: artifact.id.version.clone(), - }) + })) } diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index f4278d9d633..32eeb0d442a 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -14,6 +14,9 @@ use crate::deployment::PendingMgsUpdateDetails; use crate::inventory::BaseboardId; use daft::Diffable; +use iddqd::IdOrdItem; +use iddqd::IdOrdMap; +use iddqd::id_upcast; use indent_write::fmt::IndentWriter; use omicron_common::policy::COCKROACHDB_REDUNDANCY; use omicron_uuid_kinds::BlueprintUuid; @@ -25,11 +28,11 @@ use omicron_uuid_kinds::ZpoolUuid; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; - use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; use std::fmt::Write; +use std::sync::Arc; /// A full blueprint planning report. Other than the blueprint ID, each /// field corresponds to a step in the update planner, i.e., a subroutine @@ -475,6 +478,7 @@ impl PlanningMupdateOverrideStepReport { } } +// TODO-K: Moce this to deplyment.rs #[derive( Debug, Deserialize, @@ -522,19 +526,54 @@ impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { Copy, )] pub enum FailedMgsUpdateReason { - MissingArtifact, + // TODO-K: add some nice errors with information + NoMatchingArtifactFound, + // SP details + SpNotInInventory, + // Include caboose of what + CabooseNotInInventory, + // Retrieve where the verion parse comes from + FailedVersionParse, + // Include caboose of what + CabooseMissingSign, // Add more } #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] -pub struct FailedMgsUpdate { - pub baseboard_id: BaseboardId, +pub struct SkippedMgsUpdate { + pub baseboard_id: Arc, pub component: MgsUpdateComponent, pub reason: FailedMgsUpdateReason, } +impl IdOrdItem for SkippedMgsUpdate { + type Key<'a> = &'a BaseboardId; + fn key(&self) -> Self::Key<'_> { + &*self.baseboard_id + } + id_upcast!(); +} + +#[derive( + Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, +)] +pub struct SkippedMgsUpdates { + // The IdOrdMap key is the baseboard_id. Only one outstanding MGS-managed + // update is allowed for a given baseboard. + // + // Note that keys aren't strings so this can't be serialized as a JSON map, + // but IdOrdMap serializes as an array. + pub by_baseboard: IdOrdMap, +} + +impl SkippedMgsUpdates { + pub fn empty() -> Self { + Self { by_baseboard: IdOrdMap::new() } + } +} + #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] @@ -547,8 +586,8 @@ pub struct PlanningMgsUpdatesStepReport { // TODO-K: Maybe we want to check that all updates are in the version // we want, instead of keeping track of failed ones? Because an in-flight // update may have been removed from pending updates and won't be on - // failed updates either - pub failed_mgs_update: Option, + // failed updates either - No because the prechecks verify component version + pub failed_mgs_update: Option, } impl PlanningMgsUpdatesStepReport { From b8766dfc52ef08e7a85bddb79435ee8073960ec7 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 2 Sep 2025 18:38:58 -0600 Subject: [PATCH 04/55] Plumb skipped updates through --- .../planning/src/mgs_updates/host_phase_1.rs | 2 +- .../planning/src/mgs_updates/mod.rs | 21 +++++++++++++++--- nexus/reconfigurator/planning/src/planner.rs | 7 +++--- nexus/types/src/deployment/planning_report.rs | 22 +++++++------------ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index 2bdb01d1183..595ede4a163 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -508,7 +508,7 @@ pub(super) fn try_make_update( }, pending_host_phase_2_changes, // TODO-K: This is wrong, fix - SkippedMgsUpdates::empty(), + SkippedMgsUpdates::new(), )) } diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index f0bbad83c5c..c06731d74d1 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -69,6 +69,7 @@ pub(crate) struct PlannedMgsUpdates { pub(crate) pending_host_phase_2_changes: PendingHostPhase2Changes, // TODO-K: Add skipped updates here? + pub(crate) skipped_mgs_updates: SkippedMgsUpdates, } /// Generates a new set of `PendingMgsUpdates` based on: @@ -98,6 +99,7 @@ pub(crate) fn plan_mgs_updates( let mut pending_updates = PendingMgsUpdates::new(); let mut pending_host_phase_2_changes = PendingHostPhase2Changes::empty(); let mut boards_preferred = BTreeSet::new(); + let skipped_mgs_updates = SkippedMgsUpdates::new(); // Determine the status of all currently pending updates by comparing what // they were trying to do (and their preconditions) against the current @@ -140,6 +142,7 @@ pub(crate) fn plan_mgs_updates( (will remove it and re-evaluate board)"; update ); + // TODO-K: Add a skipped update here? boards_preferred.insert(update.baseboard_id.clone()); } }, @@ -171,9 +174,12 @@ pub(crate) fn plan_mgs_updates( log, "cannot issue more MGS-driven updates (no current artifacts)", ); + // TODO-K: Add a skipped update here, but this is an issue because there is no specific baseboard_id + // skipped_mgs_updates.by_baseboard.insert_unique(SkippedMgsUpdate { baseboard_id: (), component: (), reason: () }) return PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, + skipped_mgs_updates, }; } TargetReleaseDescription::TufRepo(description) => description, @@ -199,15 +205,18 @@ pub(crate) fn plan_mgs_updates( return PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, + skipped_mgs_updates, }; } match try_make_update(log, board, inventory, current_artifacts) { - // TODO-K: use skipped_updates + // TODO-K: use skipped_updates, this is where the skipped updates are collected Some((update, mut host_phase_2, _skipped_updates)) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); + // TODO-K: change so that we actually add the skipped updates + // skipped_mgs_updates.by_baseboard.insert_unique(skipped_mgs_updates.by_baseboard); } None => { info!(log, "skipping board for MGS-driven update"; board); @@ -216,7 +225,7 @@ pub(crate) fn plan_mgs_updates( } info!(log, "ran out of boards for MGS-driven update"); - PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes } + PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, skipped_mgs_updates } } #[derive(Debug)] @@ -502,7 +511,7 @@ fn try_make_update( inventory: &Collection, current_artifacts: &TufRepoDescription, ) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { - let mut skipped_mgs_updates = SkippedMgsUpdates::empty(); + let mut skipped_mgs_updates = SkippedMgsUpdates::new(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". @@ -1634,6 +1643,7 @@ mod test { let PlannedMgsUpdates { pending_updates: new_updates, mut pending_host_phase_2_changes, + .. } = plan_mgs_updates( log, &collection, @@ -1762,6 +1772,7 @@ mod test { let PlannedMgsUpdates { pending_updates: all_updates, mut pending_host_phase_2_changes, + .. } = plan_mgs_updates( log, &collection, @@ -1806,6 +1817,7 @@ mod test { let PlannedMgsUpdates { pending_updates: all_updates, mut pending_host_phase_2_changes, + .. } = plan_mgs_updates( log, &collection, @@ -1848,6 +1860,7 @@ mod test { let PlannedMgsUpdates { pending_updates: all_updates, mut pending_host_phase_2_changes, + .. } = plan_mgs_updates( log, &collection, @@ -1889,6 +1902,7 @@ mod test { let PlannedMgsUpdates { pending_updates: all_updates, mut pending_host_phase_2_changes, + .. } = plan_mgs_updates( log, &collection, @@ -1922,6 +1936,7 @@ mod test { let PlannedMgsUpdates { pending_updates: all_updates_done, pending_host_phase_2_changes, + .. } = plan_mgs_updates( log, &collection, diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 895fa30f2e2..53f651d4259 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -26,6 +26,7 @@ use nexus_sled_agent_shared::inventory::ConfigReconcilerInventoryResult; use nexus_sled_agent_shared::inventory::OmicronZoneImageSource; use nexus_sled_agent_shared::inventory::OmicronZoneType; use nexus_sled_agent_shared::inventory::ZoneKind; +use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintPhysicalDiskDisposition; use nexus_types::deployment::BlueprintZoneConfig; @@ -195,7 +196,7 @@ impl<'a> Planner<'a> { let mgs_updates = if add_update_blocked_reasons.is_empty() { self.do_plan_mgs_updates()? } else { - PlanningMgsUpdatesStepReport::new(PendingMgsUpdates::new()) + PlanningMgsUpdatesStepReport::new(PendingMgsUpdates::new(), SkippedMgsUpdates::new()) }; // Likewise for zone additions, unless overridden with the chicken switch. @@ -1241,7 +1242,7 @@ impl<'a> Planner<'a> { } else { ImpossibleUpdatePolicy::Reevaluate }; - let PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes } = + let PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, skipped_mgs_updates } = plan_mgs_updates( &self.log, &self.inventory, @@ -1264,7 +1265,7 @@ impl<'a> Planner<'a> { .apply_pending_host_phase_2_changes(pending_host_phase_2_changes)?; self.blueprint.pending_mgs_updates_replace_all(pending_updates.clone()); - Ok(PlanningMgsUpdatesStepReport::new(pending_updates)) + Ok(PlanningMgsUpdatesStepReport::new(pending_updates, skipped_mgs_updates)) } /// Update at most one existing zone to use a new image source. diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 32eeb0d442a..a3f965d66a1 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -80,7 +80,7 @@ impl PlanningReport { decommission: PlanningDecommissionStepReport::new(), noop_image_source: PlanningNoopImageSourceStepReport::new(), mgs_updates: PlanningMgsUpdatesStepReport::new( - PendingMgsUpdates::new(), + PendingMgsUpdates::new(), SkippedMgsUpdates::new(), ), add: PlanningAddStepReport::new(), zone_updates: PlanningZoneUpdatesStepReport::new(), @@ -560,16 +560,13 @@ impl IdOrdItem for SkippedMgsUpdate { Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, )] pub struct SkippedMgsUpdates { - // The IdOrdMap key is the baseboard_id. Only one outstanding MGS-managed - // update is allowed for a given baseboard. - // - // Note that keys aren't strings so this can't be serialized as a JSON map, - // but IdOrdMap serializes as an array. + // This may have to change to be a BtreeMap, it's causing problems to merge + // two IdOrMap pub by_baseboard: IdOrdMap, } impl SkippedMgsUpdates { - pub fn empty() -> Self { + pub fn new() -> Self { Self { by_baseboard: IdOrdMap::new() } } } @@ -583,17 +580,14 @@ pub struct PlanningMgsUpdatesStepReport { // and the reason why // add a comment // - // TODO-K: Maybe we want to check that all updates are in the version - // we want, instead of keeping track of failed ones? Because an in-flight - // update may have been removed from pending updates and won't be on - // failed updates either - No because the prechecks verify component version - pub failed_mgs_update: Option, + // TODO-K: Maybe use SkippedMgsUpdates instead of Option + pub skipped_mgs_updates: SkippedMgsUpdates, } impl PlanningMgsUpdatesStepReport { - pub fn new(pending_mgs_updates: PendingMgsUpdates) -> Self { + pub fn new(pending_mgs_updates: PendingMgsUpdates, skipped_mgs_updates: SkippedMgsUpdates) -> Self { // TODO-K: actually include the failed update - Self { pending_mgs_updates, failed_mgs_update: None } + Self { pending_mgs_updates, skipped_mgs_updates } } pub fn is_empty(&self) -> bool { From 0e5bd579184f3dd8929d759c7f99bd141e3330fb Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 2 Sep 2025 19:03:36 -0600 Subject: [PATCH 05/55] populate SkippedMgsUpdates --- .../planning/src/mgs_updates/host_phase_1.rs | 2 ++ .../planning/src/mgs_updates/mod.rs | 24 +++++++++---- nexus/reconfigurator/planning/src/planner.rs | 35 ++++++++++++------- nexus/types/src/deployment/planning_report.rs | 24 ++++++++++--- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index 595ede4a163..d6b9e8d4d28 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -271,6 +271,8 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, + // TODO-K: Perhaps it's better to return a struct here instead of tuple? + // this way I don't have to change the SkippedMgsUpdates type to BtreeMap ) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index c06731d74d1..5f1ba351a17 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -99,7 +99,7 @@ pub(crate) fn plan_mgs_updates( let mut pending_updates = PendingMgsUpdates::new(); let mut pending_host_phase_2_changes = PendingHostPhase2Changes::empty(); let mut boards_preferred = BTreeSet::new(); - let skipped_mgs_updates = SkippedMgsUpdates::new(); + let mut skipped_mgs_updates = SkippedMgsUpdates::new(); // Determine the status of all currently pending updates by comparing what // they were trying to do (and their preconditions) against the current @@ -211,12 +211,19 @@ pub(crate) fn plan_mgs_updates( match try_make_update(log, board, inventory, current_artifacts) { // TODO-K: use skipped_updates, this is where the skipped updates are collected - Some((update, mut host_phase_2, _skipped_updates)) => { + Some((update, mut host_phase_2, skipped_updates)) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); // TODO-K: change so that we actually add the skipped updates - // skipped_mgs_updates.by_baseboard.insert_unique(skipped_mgs_updates.by_baseboard); + for skipped in &skipped_updates { + // TODO-K: Handle unwrap, or wrap the insert_unique method + // into another helper method, the second sounds better + skipped_mgs_updates + .by_baseboard + .insert_unique(skipped.clone()) + .unwrap(); + } } None => { info!(log, "skipping board for MGS-driven update"; board); @@ -225,7 +232,11 @@ pub(crate) fn plan_mgs_updates( } info!(log, "ran out of boards for MGS-driven update"); - PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, skipped_mgs_updates } + PlannedMgsUpdates { + pending_updates, + pending_host_phase_2_changes, + skipped_mgs_updates, + } } #[derive(Debug)] @@ -515,13 +526,14 @@ fn try_make_update( // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". + // + // TODO-K: Will have to clean this up, the nesting will become horrible match try_make_update_rot_bootloader( log, baseboard_id, inventory, current_artifacts, ) { - // TODO-K: Will have to clean this up, the nesting will become horrible Ok(p) => { if let Some(update) = p .or_else(|| { @@ -551,7 +563,7 @@ fn try_make_update( } } Err(e) => { - // TODO-K: Remove unwrap + // TODO-K: Handle unwrap skipped_mgs_updates .by_baseboard .insert_unique(SkippedMgsUpdate { diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 53f651d4259..32dc0e0049c 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -26,7 +26,6 @@ use nexus_sled_agent_shared::inventory::ConfigReconcilerInventoryResult; use nexus_sled_agent_shared::inventory::OmicronZoneImageSource; use nexus_sled_agent_shared::inventory::OmicronZoneType; use nexus_sled_agent_shared::inventory::ZoneKind; -use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintPhysicalDiskDisposition; use nexus_types::deployment::BlueprintZoneConfig; @@ -42,6 +41,7 @@ use nexus_types::deployment::SledDetails; use nexus_types::deployment::SledFilter; use nexus_types::deployment::TufRepoContentsError; use nexus_types::deployment::ZpoolFilter; +use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::deployment::{ CockroachdbUnsafeToShutdown, PlanningAddStepReport, PlanningCockroachdbSettingsStepReport, PlanningDecommissionStepReport, @@ -196,7 +196,10 @@ impl<'a> Planner<'a> { let mgs_updates = if add_update_blocked_reasons.is_empty() { self.do_plan_mgs_updates()? } else { - PlanningMgsUpdatesStepReport::new(PendingMgsUpdates::new(), SkippedMgsUpdates::new()) + PlanningMgsUpdatesStepReport::new( + PendingMgsUpdates::new(), + SkippedMgsUpdates::new(), + ) }; // Likewise for zone additions, unless overridden with the chicken switch. @@ -1242,16 +1245,19 @@ impl<'a> Planner<'a> { } else { ImpossibleUpdatePolicy::Reevaluate }; - let PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, skipped_mgs_updates } = - plan_mgs_updates( - &self.log, - &self.inventory, - &included_baseboards, - current_updates, - current_artifacts, - NUM_CONCURRENT_MGS_UPDATES, - impossible_update_policy, - ); + let PlannedMgsUpdates { + pending_updates, + pending_host_phase_2_changes, + skipped_mgs_updates, + } = plan_mgs_updates( + &self.log, + &self.inventory, + &included_baseboards, + current_updates, + current_artifacts, + NUM_CONCURRENT_MGS_UPDATES, + impossible_update_policy, + ); if pending_updates != *current_updates { // This will only add comments if our set of updates changed _and_ // we have at least one update. If we went from "some updates" to @@ -1265,7 +1271,10 @@ impl<'a> Planner<'a> { .apply_pending_host_phase_2_changes(pending_host_phase_2_changes)?; self.blueprint.pending_mgs_updates_replace_all(pending_updates.clone()); - Ok(PlanningMgsUpdatesStepReport::new(pending_updates, skipped_mgs_updates)) + Ok(PlanningMgsUpdatesStepReport::new( + pending_updates, + skipped_mgs_updates, + )) } /// Update at most one existing zone to use a new image source. diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index a3f965d66a1..b801cca69cc 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -80,7 +80,8 @@ impl PlanningReport { decommission: PlanningDecommissionStepReport::new(), noop_image_source: PlanningNoopImageSourceStepReport::new(), mgs_updates: PlanningMgsUpdatesStepReport::new( - PendingMgsUpdates::new(), SkippedMgsUpdates::new(), + PendingMgsUpdates::new(), + SkippedMgsUpdates::new(), ), add: PlanningAddStepReport::new(), zone_updates: PlanningZoneUpdatesStepReport::new(), @@ -478,7 +479,7 @@ impl PlanningMupdateOverrideStepReport { } } -// TODO-K: Moce this to deplyment.rs +// TODO-K: Move this to deplyment.rs #[derive( Debug, Deserialize, @@ -560,8 +561,6 @@ impl IdOrdItem for SkippedMgsUpdate { Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, )] pub struct SkippedMgsUpdates { - // This may have to change to be a BtreeMap, it's causing problems to merge - // two IdOrMap pub by_baseboard: IdOrdMap, } @@ -569,6 +568,18 @@ impl SkippedMgsUpdates { pub fn new() -> Self { Self { by_baseboard: IdOrdMap::new() } } + + pub fn iter(&self) -> impl Iterator { + self.into_iter() + } +} + +impl<'a> IntoIterator for &'a SkippedMgsUpdates { + type Item = &'a SkippedMgsUpdate; + type IntoIter = iddqd::id_ord_map::Iter<'a, SkippedMgsUpdate>; + fn into_iter(self) -> Self::IntoIter { + self.by_baseboard.iter() + } } #[derive( @@ -585,7 +596,10 @@ pub struct PlanningMgsUpdatesStepReport { } impl PlanningMgsUpdatesStepReport { - pub fn new(pending_mgs_updates: PendingMgsUpdates, skipped_mgs_updates: SkippedMgsUpdates) -> Self { + pub fn new( + pending_mgs_updates: PendingMgsUpdates, + skipped_mgs_updates: SkippedMgsUpdates, + ) -> Self { // TODO-K: actually include the failed update Self { pending_mgs_updates, skipped_mgs_updates } } From 0c5197aef304ceedaf89d16df153df95441a6a6c Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 3 Sep 2025 10:42:21 -0600 Subject: [PATCH 06/55] remove unnecessary checks --- .../planning/src/mgs_updates/host_phase_1.rs | 3 +- .../planning/src/mgs_updates/mod.rs | 4 +- nexus/types/src/deployment/planning_report.rs | 61 +++++++++++++++++-- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index d6b9e8d4d28..3c6931a3775 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -271,8 +271,7 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, - // TODO-K: Perhaps it's better to return a struct here instead of tuple? - // this way I don't have to change the SkippedMgsUpdates type to BtreeMap + // TODO-K: Return the result here like RoT bootloader ) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 5f1ba351a17..3a830661e2e 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -142,7 +142,6 @@ pub(crate) fn plan_mgs_updates( (will remove it and re-evaluate board)"; update ); - // TODO-K: Add a skipped update here? boards_preferred.insert(update.baseboard_id.clone()); } }, @@ -170,12 +169,11 @@ pub(crate) fn plan_mgs_updates( // containing artifacts), then we cannot configure more updates. let current_artifacts = match current_artifacts { TargetReleaseDescription::Initial => { + // TODO-K: change comment to no updates needed and change to info warn!( log, "cannot issue more MGS-driven updates (no current artifacts)", ); - // TODO-K: Add a skipped update here, but this is an issue because there is no specific baseboard_id - // skipped_mgs_updates.by_baseboard.insert_unique(SkippedMgsUpdate { baseboard_id: (), component: (), reason: () }) return PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index b801cca69cc..d1832a94c55 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -31,6 +31,7 @@ use serde::Serialize; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; +use std::fmt::Display; use std::fmt::Write; use std::sync::Arc; @@ -513,6 +514,18 @@ impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { } } +impl Display for MgsUpdateComponent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + MgsUpdateComponent::HostOs => "Host OS", + MgsUpdateComponent::Rot => "RoT", + MgsUpdateComponent::RotBootloader => "RoT Bootloader", + MgsUpdateComponent::Sp => "SP", + }; + write!(f, "{s}") + } +} + #[derive( Debug, Deserialize, @@ -540,6 +553,30 @@ pub enum FailedMgsUpdateReason { // Add more } +// TODO-K: Do I need display? or are the error bits enough +impl Display for FailedMgsUpdateReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + FailedMgsUpdateReason::CabooseMissingSign => { + "caboose is missing sign" + } + FailedMgsUpdateReason::CabooseNotInInventory => { + "caboose is not in inventory" + } + FailedMgsUpdateReason::FailedVersionParse => { + "version could not be parsed" + } + FailedMgsUpdateReason::NoMatchingArtifactFound => { + "no matching artifact was found" + } + FailedMgsUpdateReason::SpNotInInventory => { + "corresponding SP is not in inventory" + } + }; + write!(f, "{s}") + } +} + #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] @@ -572,6 +609,10 @@ impl SkippedMgsUpdates { pub fn iter(&self) -> impl Iterator { self.into_iter() } + + pub fn is_empty(&self) -> bool { + self.by_baseboard.is_empty() + } } impl<'a> IntoIterator for &'a SkippedMgsUpdates { @@ -587,11 +628,7 @@ impl<'a> IntoIterator for &'a SkippedMgsUpdates { )] pub struct PlanningMgsUpdatesStepReport { pub pending_mgs_updates: PendingMgsUpdates, - // TODO-K: keep the component here that we were unable to update - // and the reason why - // add a comment - // - // TODO-K: Maybe use SkippedMgsUpdates instead of Option + // TODO-K: Add a nice comment here about what is happening pub skipped_mgs_updates: SkippedMgsUpdates, } @@ -612,7 +649,7 @@ impl PlanningMgsUpdatesStepReport { impl fmt::Display for PlanningMgsUpdatesStepReport { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // TODO-K: implement display - let Self { pending_mgs_updates, .. } = self; + let Self { pending_mgs_updates, skipped_mgs_updates } = self; if !pending_mgs_updates.is_empty() { let n = pending_mgs_updates.len(); let s = plural(n); @@ -625,6 +662,18 @@ impl fmt::Display for PlanningMgsUpdatesStepReport { )?; } } + if !skipped_mgs_updates.is_empty() { + let n = pending_mgs_updates.len(); + let s = plural(n); + writeln!(f, "* {n} skipped MGS update{s}:")?; + for update in skipped_mgs_updates.iter() { + writeln!( + f, + " * {}: {} {}", + update.baseboard_id, update.component, update.reason + )?; + } + } Ok(()) } } From 5a1a74fd86f1b842da73d3b54561de255a1fdebd Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 3 Sep 2025 16:27:43 -0600 Subject: [PATCH 07/55] do the todos --- .../planning/src/mgs_updates/mod.rs | 40 ++++----- .../src/mgs_updates/rot_bootloader.rs | 2 - .../planning/src/mgs_updates/test_helpers.rs | 2 +- nexus/reconfigurator/planning/src/planner.rs | 16 ++-- nexus/types/src/deployment.rs | 47 ++++++++++ nexus/types/src/deployment/planning_report.rs | 90 +++++++------------ 6 files changed, 106 insertions(+), 91 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 3a830661e2e..6a60ac30c6a 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -20,6 +20,7 @@ use crate::mgs_updates::sp::try_make_update_sp; use gateway_types::rot::RotSlot; use nexus_types::deployment::ExpectedActiveRotSlot; use nexus_types::deployment::ExpectedVersion; +use nexus_types::deployment::MgsUpdateComponent; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; @@ -27,7 +28,6 @@ use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; -use nexus_types::deployment::planning_report::MgsUpdateComponent; use nexus_types::deployment::planning_report::SkippedMgsUpdate; use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; @@ -68,7 +68,7 @@ pub(crate) struct PlannedMgsUpdates { /// result in a change to the respective sled's `BlueprintSledConfig`. pub(crate) pending_host_phase_2_changes: PendingHostPhase2Changes, - // TODO-K: Add skipped updates here? + // Updates to components that failed for some reason and have been skipped. pub(crate) skipped_mgs_updates: SkippedMgsUpdates, } @@ -169,10 +169,10 @@ pub(crate) fn plan_mgs_updates( // containing artifacts), then we cannot configure more updates. let current_artifacts = match current_artifacts { TargetReleaseDescription::Initial => { - // TODO-K: change comment to no updates needed and change to info - warn!( + info!( log, - "cannot issue more MGS-driven updates (no current artifacts)", + "system in initial release state \ + no update artifacts available (no update necessary)", ); return PlannedMgsUpdates { pending_updates, @@ -208,19 +208,19 @@ pub(crate) fn plan_mgs_updates( } match try_make_update(log, board, inventory, current_artifacts) { - // TODO-K: use skipped_updates, this is where the skipped updates are collected Some((update, mut host_phase_2, skipped_updates)) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); - // TODO-K: change so that we actually add the skipped updates for skipped in &skipped_updates { - // TODO-K: Handle unwrap, or wrap the insert_unique method - // into another helper method, the second sounds better - skipped_mgs_updates - .by_baseboard - .insert_unique(skipped.clone()) - .unwrap(); + warn!( + log, + "update to {} {} has been skipped: {}", + skipped.baseboard_id, + skipped.component, + skipped.reason + ); + skipped_mgs_updates.insert(skipped.clone()); } } None => { @@ -561,15 +561,11 @@ fn try_make_update( } } Err(e) => { - // TODO-K: Handle unwrap - skipped_mgs_updates - .by_baseboard - .insert_unique(SkippedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component: MgsUpdateComponent::RotBootloader, - reason: e, - }) - .unwrap(); + skipped_mgs_updates.insert(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::RotBootloader, + reason: e, + }); } } diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index 659c7692b5f..a1fda4a12d4 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -61,8 +61,6 @@ pub fn try_make_update_rot_bootloader( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, - // TODO-K: This needs to stay as an option because Ok(None) means there is - // update needed ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( diff --git a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs index 500c7728747..53d903e641f 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs @@ -32,13 +32,13 @@ use nexus_sled_agent_shared::inventory::ZoneImageResolverInventory; use nexus_types::deployment::BlueprintArtifactVersion; use nexus_types::deployment::BlueprintHostPhase2DesiredContents; use nexus_types::deployment::ExpectedVersion; +use nexus_types::deployment::MgsUpdateComponent; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateHostPhase1Details; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; -use nexus_types::deployment::planning_report::MgsUpdateComponent; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; use omicron_common::api::external::Generation; diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 32dc0e0049c..8b7dcd9d1db 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -224,14 +224,18 @@ impl<'a> Planner<'a> { // TODO-K: mgs_updates can be empty if some updates failed. Make sure to // keep track of failed updates and wait until they are on the expected // new version befor proceeding to update zones - } else if !mgs_updates.is_empty() { + } else if !mgs_updates.pending_mgs_updates.is_empty() { // ... or if there are still pending updates for the RoT / SP / // Host OS / etc. ... - // TODO This is not quite right. See oxidecomputer/omicron#8285. - // TODO-K: change here? PlanningZoneUpdatesStepReport::waiting_on( ZoneUpdatesWaitingOn::PendingMgsUpdates, ) + } else if !mgs_updates.skipped_mgs_updates.is_empty() { + // ... or if there are skipped updates for the RoT / SP / Host OS / + // RoT bootloader. + PlanningZoneUpdatesStepReport::waiting_on( + ZoneUpdatesWaitingOn::SkippedMgsUpdates, + ) } else if !add.add_update_blocked_reasons.is_empty() { // ... or if there are pending zone add blockers. PlanningZoneUpdatesStepReport::waiting_on( @@ -1885,7 +1889,7 @@ impl<'a> Planner<'a> { source_repo.zone_image_source(zone_kind) } - /// Return `true` iff a zone of the given kind is ready to be updated; + /// Return `true` if a zone of the given kind is ready to be updated; /// i.e., its dependencies have been updated. fn is_zone_ready_for_update( &self, @@ -1893,8 +1897,8 @@ impl<'a> Planner<'a> { mgs_updates: &PlanningMgsUpdatesStepReport, ) -> Result { // We return false regardless of `zone_kind` if there are still - // pending updates for components earlier in the update ordering - // than zones: RoT bootloader / RoT / SP / Host OS. + // pending or skipped updates for components earlier in the update + // ordering than zones: RoT bootloader / RoT / SP / Host OS. if !mgs_updates.is_empty() { return Ok(false); } diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 421a89ff23b..44171b3a1e5 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -57,6 +57,7 @@ use serde::Serialize; use slog::Key; use std::collections::BTreeMap; use std::fmt; +use std::fmt::Display; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use std::sync::Arc; @@ -1293,6 +1294,52 @@ impl fmt::Display for BlueprintHostPhase2DesiredContents { } } +#[derive( + Debug, + Deserialize, + Serialize, + PartialEq, + Eq, + Diffable, + PartialOrd, + JsonSchema, + Ord, + Clone, + Copy, +)] +pub enum MgsUpdateComponent { + Sp, + Rot, + RotBootloader, + HostOs, +} + +// TODO-K: Implement into() instead +impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { + fn from(value: &'_ PendingMgsUpdateDetails) -> Self { + match value { + PendingMgsUpdateDetails::Rot { .. } => Self::Rot, + PendingMgsUpdateDetails::RotBootloader { .. } => { + Self::RotBootloader + } + PendingMgsUpdateDetails::Sp { .. } => Self::Sp, + PendingMgsUpdateDetails::HostPhase1(_) => Self::HostOs, + } + } +} + +impl Display for MgsUpdateComponent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + MgsUpdateComponent::HostOs => "Host OS", + MgsUpdateComponent::Rot => "RoT", + MgsUpdateComponent::RotBootloader => "RoT Bootloader", + MgsUpdateComponent::Sp => "SP", + }; + write!(f, "{s}") + } +} + #[derive( Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, )] diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index d1832a94c55..3bead50757c 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -10,7 +10,7 @@ use super::BlueprintZoneImageSource; use super::CockroachDbPreserveDowngrade; use super::PendingMgsUpdates; use super::PlannerChickenSwitches; -use crate::deployment::PendingMgsUpdateDetails; +use crate::deployment::MgsUpdateComponent; use crate::inventory::BaseboardId; use daft::Diffable; @@ -480,52 +480,7 @@ impl PlanningMupdateOverrideStepReport { } } -// TODO-K: Move this to deplyment.rs -#[derive( - Debug, - Deserialize, - Serialize, - PartialEq, - Eq, - Diffable, - PartialOrd, - JsonSchema, - Ord, - Clone, - Copy, -)] -pub enum MgsUpdateComponent { - Sp, - Rot, - RotBootloader, - HostOs, -} - -impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { - fn from(value: &'_ PendingMgsUpdateDetails) -> Self { - match value { - PendingMgsUpdateDetails::Rot { .. } => Self::Rot, - PendingMgsUpdateDetails::RotBootloader { .. } => { - Self::RotBootloader - } - PendingMgsUpdateDetails::Sp { .. } => Self::Sp, - PendingMgsUpdateDetails::HostPhase1(_) => Self::HostOs, - } - } -} - -impl Display for MgsUpdateComponent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - MgsUpdateComponent::HostOs => "Host OS", - MgsUpdateComponent::Rot => "RoT", - MgsUpdateComponent::RotBootloader => "RoT Bootloader", - MgsUpdateComponent::Sp => "SP", - }; - write!(f, "{s}") - } -} - +/// Describes the reason why an SP component failed to update #[derive( Debug, Deserialize, @@ -540,20 +495,18 @@ impl Display for MgsUpdateComponent { Copy, )] pub enum FailedMgsUpdateReason { - // TODO-K: add some nice errors with information + /// No artifact with the required conditions for the component was found NoMatchingArtifactFound, - // SP details + /// The component's corresponding SP was not found in the inventory SpNotInInventory, - // Include caboose of what + /// The component's caboose was not found in the inventory CabooseNotInInventory, - // Retrieve where the verion parse comes from + /// The version in the caboose or artifact was not able to be parsed FailedVersionParse, - // Include caboose of what + /// The component's caboose was missing a value for "sign" CabooseMissingSign, - // Add more } -// TODO-K: Do I need display? or are the error bits enough impl Display for FailedMgsUpdateReason { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { @@ -581,8 +534,11 @@ impl Display for FailedMgsUpdateReason { Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] pub struct SkippedMgsUpdate { + /// id of the baseboard that we attempted to update pub baseboard_id: Arc, + /// type of SP component that we attempted to update pub component: MgsUpdateComponent, + /// reason why the update failed pub reason: FailedMgsUpdateReason, } @@ -598,6 +554,9 @@ impl IdOrdItem for SkippedMgsUpdate { Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, )] pub struct SkippedMgsUpdates { + // The IdOrdMap key is the baseboard_id. We only need to know which devices + // were skipped, so only one skipped MGS-managed update is allowed for a + // given baseboard. pub by_baseboard: IdOrdMap, } @@ -613,6 +572,13 @@ impl SkippedMgsUpdates { pub fn is_empty(&self) -> bool { self.by_baseboard.is_empty() } + + pub fn insert( + &mut self, + update: SkippedMgsUpdate, + ) -> Option { + self.by_baseboard.insert_overwrite(update) + } } impl<'a> IntoIterator for &'a SkippedMgsUpdates { @@ -628,7 +594,6 @@ impl<'a> IntoIterator for &'a SkippedMgsUpdates { )] pub struct PlanningMgsUpdatesStepReport { pub pending_mgs_updates: PendingMgsUpdates, - // TODO-K: Add a nice comment here about what is happening pub skipped_mgs_updates: SkippedMgsUpdates, } @@ -637,18 +602,17 @@ impl PlanningMgsUpdatesStepReport { pending_mgs_updates: PendingMgsUpdates, skipped_mgs_updates: SkippedMgsUpdates, ) -> Self { - // TODO-K: actually include the failed update Self { pending_mgs_updates, skipped_mgs_updates } } pub fn is_empty(&self) -> bool { self.pending_mgs_updates.is_empty() + && self.skipped_mgs_updates.is_empty() } } impl fmt::Display for PlanningMgsUpdatesStepReport { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO-K: implement display let Self { pending_mgs_updates, skipped_mgs_updates } = self; if !pending_mgs_updates.is_empty() { let n = pending_mgs_updates.len(); @@ -669,7 +633,7 @@ impl fmt::Display for PlanningMgsUpdatesStepReport { for update in skipped_mgs_updates.iter() { writeln!( f, - " * {}: {} {}", + " * {} {}: {}", update.baseboard_id, update.component, update.reason )?; } @@ -1142,9 +1106,12 @@ pub enum ZoneUpdatesWaitingOn { /// Waiting on discretionary zone placement. DiscretionaryZones, - /// Waiting on updates to RoT / SP / Host OS / etc. + /// Waiting on updates to RoT bootloader / RoT / SP / Host OS. PendingMgsUpdates, + /// Waiting on skipped updates to RoT bootloader / RoT / SP / Host OS. + SkippedMgsUpdates, + /// Waiting on the same set of blockers zone adds are waiting on. ZoneAddBlockers, } @@ -1154,7 +1121,10 @@ impl ZoneUpdatesWaitingOn { match self { Self::DiscretionaryZones => "discretionary zones", Self::PendingMgsUpdates => { - "pending MGS updates (RoT / SP / Host OS / etc.)" + "pending MGS updates (RoT bootloader / RoT / SP / Host OS)" + } + Self::SkippedMgsUpdates => { + "skipped MGS updates (RoT bootloader / RoT / SP / Host OS)" } Self::ZoneAddBlockers => "zone add blockers", } From 60240f2a84c337150d51769a8fda8b7c0c3d5fe5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 3 Sep 2025 17:07:59 -0600 Subject: [PATCH 08/55] expectorate --- .../output/cmds-add-sled-no-disks-stdout | 2 +- .../tests/output/cmds-example-stdout | 4 +- ...ds-expunge-newly-added-external-dns-stdout | 2 +- ...ds-expunge-newly-added-internal-dns-stdout | 2 +- .../output/cmds-mupdate-update-flow-stdout | 4 +- .../tests/output/cmds-target-release-stdout | 46 +++++++++---------- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout index 9591408f905..14a0c96e0bf 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout @@ -37,7 +37,7 @@ generated inventory collection eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 from configu > # we added has no disks. > blueprint-plan dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 eb0796d5-ab8a-4f7b-a884-b4aeacb8ab51 INFO skipping noop image source check for all sleds, reason: no target release is currently set -WARN cannot issue more MGS-driven updates (no current artifacts) +INFO system in initial release state no update artifacts available (no update necessary) generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 planning report for blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1: chicken switches: diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index 92232e4b5e6..40e9ac713c2 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -550,7 +550,7 @@ T ENA ID PARENT > blueprint-plan ade5749d-bdf3-4fab-a8ae-00bea01b3a5a INFO skipping noop image source check for all sleds, reason: no target release is currently set -WARN cannot issue more MGS-driven updates (no current artifacts) +INFO system in initial release state no update artifacts available (no update necessary) INFO some zones not yet up-to-date, sled_id: 89d02b1b-478c-401a-8e28-7a26f74fa41b, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: b3c9c041-d2f0-4767-bdaf-0e52e9d7a013 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: InstallDataset } }] generated blueprint 86db3308-f817-4626-8838-4085949a6a41 based on parent blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a planning report for blueprint 86db3308-f817-4626-8838-4085949a6a41: @@ -1574,7 +1574,7 @@ INTERNAL DNS STATUS > # sled to be expunged. > blueprint-plan latest INFO skipping noop image source check for all sleds, reason: no target release is currently set -WARN cannot issue more MGS-driven updates (no current artifacts) +INFO system in initial release state no update artifacts available (no update necessary) generated blueprint 86db3308-f817-4626-8838-4085949a6a41 based on parent blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a empty planning report for blueprint 86db3308-f817-4626-8838-4085949a6a41. diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout index cf22a460ed9..87bc805c0d9 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout @@ -836,7 +836,7 @@ empty planning report for blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0. > # blueprint-plan will place a new external DNS zone, diff DNS to see the new zone has `ns` and NS records. > blueprint-plan 366b0b68-d80e-4bc1-abd3-dc69837847e0 INFO skipping noop image source check for all sleds, reason: no target release is currently set -WARN cannot issue more MGS-driven updates (no current artifacts) +INFO system in initial release state no update artifacts available (no update necessary) generated blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 based on parent blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0 planning report for blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2: chicken switches: diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout index d18a5821897..0375418a0fe 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout @@ -644,7 +644,7 @@ external DNS: > # Planning a new blueprint will now replace the expunged zone, with new records for its replacement. > blueprint-plan 58d5e830-0884-47d8-a7cd-b2b3751adeb4 INFO skipping noop image source check for all sleds, reason: no target release is currently set -WARN cannot issue more MGS-driven updates (no current artifacts) +INFO system in initial release state no update artifacts available (no update necessary) generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 planning report for blueprint af934083-59b5-4bf6-8966-6fb5292c29e1: chicken switches: diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index 39c1f30e8b9..98844e8e518 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -1708,7 +1708,7 @@ chicken switches: * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-show latest @@ -1890,7 +1890,7 @@ chicken switches: * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 6 zones are already from artifacts * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 7938d183877..f7d6ca602ce 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -221,7 +221,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -281,7 +281,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -343,7 +343,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -414,7 +414,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -486,7 +486,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -607,7 +607,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -668,7 +668,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -728,7 +728,7 @@ chicken switches: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -790,7 +790,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -863,7 +863,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: Version(ArtifactVersion("0.5.0")) }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -931,7 +931,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1003,7 +1003,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1071,7 +1071,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1142,7 +1142,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1211,7 +1211,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1347,7 +1347,7 @@ chicken switches: * 1 pending MGS update: * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1421,7 +1421,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1488,7 +1488,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1559,7 +1559,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("1.0.0")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1632,7 +1632,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: Some(B), expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1702,7 +1702,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: Some(B) }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1770,7 +1770,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest @@ -1837,7 +1837,7 @@ chicken switches: * 1 pending MGS update: * model2:serial2: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) > blueprint-diff latest From ad2e83e4b50162bd7553622dcc9e58a05a0ce629 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 3 Sep 2025 20:35:02 -0600 Subject: [PATCH 09/55] Make SkippedMgsUpdates a vec because we need all records --- .../planning/src/mgs_updates/host_phase_1.rs | 40 ++++---- .../planning/src/mgs_updates/mod.rs | 52 ++++++++-- nexus/types/src/deployment/planning_report.rs | 98 +++++++++++++------ 3 files changed, 131 insertions(+), 59 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index 3c6931a3775..eda3a37419d 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -7,12 +7,12 @@ use super::MgsUpdateStatus; use super::MgsUpdateStatusError; use gateway_client::types::SpType; +use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::deployment::BlueprintArtifactVersion; use nexus_types::deployment::BlueprintHostPhase2DesiredContents; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateHostPhase1Details; -use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use omicron_common::api::external::TufArtifactMeta; @@ -271,8 +271,7 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, - // TODO-K: Return the result here like RoT bootloader -) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { +) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( log, @@ -280,16 +279,17 @@ pub(super) fn try_make_update( (missing SP info from inventory)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::SpNotInInventory); }; // Only configure host OS updates for sleds. // // We don't bother logging a return value of `None` for non-sleds, because // we will never attempt to configure an update for them (nor should we). + // For the same reason, we do not return an error. match sp_info.sp_type { SpType::Sled => (), - SpType::Power | SpType::Switch => return None, + SpType::Power | SpType::Switch => return Ok(None), } let Some(sled_agent) = inventory.sled_agents.iter().find(|sled_agent| { @@ -301,7 +301,7 @@ pub(super) fn try_make_update( (missing sled-agent info from inventory)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::SledAgentInfoNotInInventory); }; let Some(last_reconciliation) = sled_agent.last_reconciliation.as_ref() else { @@ -311,7 +311,7 @@ pub(super) fn try_make_update( (missing last reconciliation details from inventory)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::LastReconciliationNotInInventory); }; let boot_disk = match &last_reconciliation.boot_partitions.boot_disk { Ok(boot_disk) => *boot_disk, @@ -325,7 +325,7 @@ pub(super) fn try_make_update( baseboard_id, "err" => err, ); - return None; + return Err(FailedMgsUpdateReason::UnableToDetermineBootDisk(err.to_string())); } }; let active_phase_2_hash = @@ -342,7 +342,7 @@ pub(super) fn try_make_update( "boot_disk" => ?boot_disk, "err" => err, ); - return None; + return Err(FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image(err.to_string())); } }; @@ -355,7 +355,7 @@ pub(super) fn try_make_update( (inventory missing current active host phase 1 slot)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::ActiveHostPhase1SlotNotInInventory); }; // TODO-correctness What should we do if the active phase 1 slot doesn't @@ -378,7 +378,7 @@ pub(super) fn try_make_update( "active_phase_1_slot" => ?active_phase_1_slot, "boot_disk" => ?boot_disk, ); - return None; + return Err(FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch); } let Some(active_phase_1_hash) = inventory @@ -392,7 +392,7 @@ pub(super) fn try_make_update( baseboard_id, "slot" => ?active_phase_1_slot, ); - return None; + return Err(FailedMgsUpdateReason::ActiveHostPhase1HashNotInInventory); }; let Some(inactive_phase_1_hash) = inventory @@ -409,7 +409,7 @@ pub(super) fn try_make_update( baseboard_id, "slot" => ?active_phase_1_slot.toggled(), ); - return None; + return Err(FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory); }; let mut phase_1_artifacts = Vec::with_capacity(1); @@ -436,7 +436,7 @@ pub(super) fn try_make_update( (no phase 1 artifact)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } (_, []) => { warn!( @@ -445,7 +445,7 @@ pub(super) fn try_make_update( (no phase 2 artifact)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } // "TUF is broken" cases: have multiple of one or the other. This // should be impossible unless we shipped a TUF repo with multiple @@ -460,7 +460,7 @@ pub(super) fn try_make_update( "num-phase-1-images" => phase_1_artifacts.len(), "num-phase-2-images" => phase_2_artifacts.len(), ); - return None; + return Err(FailedMgsUpdateReason::TooManyMatchingArtifacts); } }; @@ -472,7 +472,7 @@ pub(super) fn try_make_update( // this sled will fail to boot if it were rebooted now.) if active_phase_2_hash == phase_2_artifact.hash { debug!(log, "no host OS update needed for board"; baseboard_id); - return None; + return Ok(None); } // Before we can proceed with the phase 1 update, we need sled-agent to @@ -488,7 +488,7 @@ pub(super) fn try_make_update( phase_2_artifact, ); - Some(( + Ok(Some(( PendingMgsUpdate { baseboard_id: baseboard_id.clone(), sp_type: sp_info.sp_type, @@ -508,9 +508,7 @@ pub(super) fn try_make_update( artifact_version: phase_1_artifact.id.version.clone(), }, pending_host_phase_2_changes, - // TODO-K: This is wrong, fix - SkippedMgsUpdates::new(), - )) + ))) } #[cfg(test)] diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 6a60ac30c6a..00cdb616d95 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -17,6 +17,7 @@ use crate::mgs_updates::rot_bootloader::try_make_update_rot_bootloader; use crate::mgs_updates::sp::mgs_update_status_sp; use crate::mgs_updates::sp::try_make_update_sp; +use gateway_client::types::SpType; use gateway_types::rot::RotSlot; use nexus_types::deployment::ExpectedActiveRotSlot; use nexus_types::deployment::ExpectedVersion; @@ -37,6 +38,7 @@ use omicron_common::api::external::TufRepoDescription; use omicron_common::disk::M2Slot; use slog::{error, info, warn}; use slog_error_chain::InlineErrorChain; +use tufaceous_artifact::ArtifactHash; use std::collections::BTreeSet; use std::sync::Arc; use thiserror::Error; @@ -212,7 +214,8 @@ pub(crate) fn plan_mgs_updates( info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); - for skipped in &skipped_updates { + // TODO-K: Are the logs necessary? + for skipped in &skipped_updates.updates { warn!( log, "update to {} {} has been skipped: {}", @@ -220,7 +223,7 @@ pub(crate) fn plan_mgs_updates( skipped.component, skipped.reason ); - skipped_mgs_updates.insert(skipped.clone()); + skipped_mgs_updates.push(skipped.clone()); } } None => { @@ -519,6 +522,8 @@ fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, + // TODO-K: This is wrong, there could be skipped mgs updates and no + // pending update ) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { let mut skipped_mgs_updates = SkippedMgsUpdates::new(); // We try MGS-driven update components in a hardcoded priority order until @@ -561,11 +566,13 @@ fn try_make_update( } } Err(e) => { - skipped_mgs_updates.insert(SkippedMgsUpdate { + skipped_mgs_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::RotBootloader, reason: e, }); + // TODO-K: remove debugging log + warn!(log, "HERE: {:?}", skipped_mgs_updates); } } @@ -586,12 +593,33 @@ fn try_make_update( // return Some((update, PendingHostPhase2Changes::empty())); // } - host_phase_1::try_make_update( + match host_phase_1::try_make_update( log, baseboard_id, inventory, current_artifacts, - ) + ) { + Ok(p) => { + match p { + Some((u, c)) => return Some((u, c, skipped_mgs_updates)), + // TODO-K: This is wrong, there could be skipped updates + None => return None, + } + }, + Err(e) => { + skipped_mgs_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::HostOs, + reason: e, + }); + // TODO-K: remove debugging log + warn!(log, "HERE2: {:?}", skipped_mgs_updates); + // TODO-K: This is wrong there is no pending update + const ARTIFACT_HASH_SP_GIMLET_E: ArtifactHash = + ArtifactHash([1; 32]); + return Some((PendingMgsUpdate{sp_type: SpType::Power, baseboard_id: baseboard_id.clone(), slot_id: 22, details: PendingMgsUpdateDetails::Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion::new("100.0.0").unwrap(), expected_inactive_version: ExpectedVersion::Version(ArtifactVersion::new("100.0.0").unwrap()) }), artifact_hash: ARTIFACT_HASH_SP_GIMLET_E, artifact_version: ArtifactVersion::new("1000.0.0").unwrap()}, PendingHostPhase2Changes::empty(), skipped_mgs_updates)); + } + } } #[cfg(test)] @@ -620,6 +648,7 @@ mod test { use dropshot::ConfigLoggingLevel; use gateway_client::types::SpType; use gateway_types::rot::RotSlot; + use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; @@ -627,7 +656,9 @@ mod test { use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; + use nexus_types::inventory::BaseboardId; use omicron_test_utils::dev::LogContext; + use std::sync::Arc; use std::collections::BTreeSet; use strum::IntoEnumIterator; @@ -1327,20 +1358,27 @@ mod test { // Test that we don't try to update boards that aren't in // `current_boards`, even if they're in inventory and outdated. + + // TODO-K: Remove fake boards? + let mut fake_boards = BTreeSet::new(); + fake_boards.insert(Arc::new(BaseboardId{part_number: "bob".to_string(), serial_number: "ob".to_string()})); let collection = test_boards .collection_builder() .stage0_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) .build(); - let PlannedMgsUpdates { pending_updates: updates, .. } = + let PlannedMgsUpdates { pending_updates: updates, skipped_mgs_updates, .. } = plan_mgs_updates( log, &collection, - &BTreeSet::new(), + &fake_boards, + //&BTreeSet::new(), &PendingMgsUpdates::new(), &TargetReleaseDescription::TufRepo(repo.clone()), nmax_updates, impossible_update_policy, ); + // TODO-K: Remove this assertion, or improve + assert_eq!(skipped_mgs_updates, SkippedMgsUpdates::new()); assert!(updates.is_empty()); let PlannedMgsUpdates { pending_updates: updates, .. } = plan_mgs_updates( diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 3bead50757c..8aab5dc3689 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -15,7 +15,6 @@ use crate::inventory::BaseboardId; use daft::Diffable; use iddqd::IdOrdItem; -use iddqd::IdOrdMap; use iddqd::id_upcast; use indent_write::fmt::IndentWriter; use omicron_common::policy::COCKROACHDB_REDUNDANCY; @@ -492,24 +491,53 @@ impl PlanningMupdateOverrideStepReport { JsonSchema, Ord, Clone, - Copy, )] pub enum FailedMgsUpdateReason { - /// No artifact with the required conditions for the component was found - NoMatchingArtifactFound, - /// The component's corresponding SP was not found in the inventory - SpNotInInventory, + /// The active host phase 1 slot does not match the boot disk + ActiveHostPhase1SlotBootDiskMismatch, + /// The active host phase 1 hash was not found in inventory + ActiveHostPhase1HashNotInInventory, + /// The active host phase 1 slot was not found in inventory + ActiveHostPhase1SlotNotInInventory, + /// The component's caboose was missing a value for "sign" + CabooseMissingSign, /// The component's caboose was not found in the inventory CabooseNotInInventory, /// The version in the caboose or artifact was not able to be parsed FailedVersionParse, - /// The component's caboose was missing a value for "sign" - CabooseMissingSign, + /// The inactive host phase 1 hash was not found in inventory + InactiveHostPhase1HashNotInInventory, + /// Last reconciliation details were not found in inventory + LastReconciliationNotInInventory, + /// No artifact with the required conditions for the component was found + NoMatchingArtifactFound, + /// Sled agent info was not found in inventory + SledAgentInfoNotInInventory, + /// The component's corresponding SP was not found in the inventory + SpNotInInventory, + /// Too many artifacts with the required conditions for the component were + /// found + TooManyMatchingArtifacts, + /// The sled agent reported an error determining the boot disk + UnableToDetermineBootDisk(String), + /// The sled agent reported an error retrieving boot disk phase 2 image + /// details + UnableToRetrieveBootDiskPhase2Image(String), } +// TODO-K: Keep these or the current logs or treat this more like a real error? impl Display for FailedMgsUpdateReason { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { + FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch => { + "active phase 1 slot does not match boot disk" + } + FailedMgsUpdateReason::ActiveHostPhase1HashNotInInventory => { + "active host phase 1 hash is not in inventory" + } + FailedMgsUpdateReason::ActiveHostPhase1SlotNotInInventory => { + "active host phase 1 slot is not in inventory" + } FailedMgsUpdateReason::CabooseMissingSign => { "caboose is missing sign" } @@ -519,12 +547,33 @@ impl Display for FailedMgsUpdateReason { FailedMgsUpdateReason::FailedVersionParse => { "version could not be parsed" } + FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory => { + "inactive host phase 1 hash is not in inventory" + } + FailedMgsUpdateReason::LastReconciliationNotInInventory => { + "sled agent last reconciliation is not in inventory" + } FailedMgsUpdateReason::NoMatchingArtifactFound => { "no matching artifact was found" } FailedMgsUpdateReason::SpNotInInventory => { "corresponding SP is not in inventory" } + FailedMgsUpdateReason::SledAgentInfoNotInInventory => { + "sled agent info is not in inventory" + } + FailedMgsUpdateReason::TooManyMatchingArtifacts => { + "too many matching artifacts were found" + } + FailedMgsUpdateReason::UnableToDetermineBootDisk(err) => { + &format!("sled agent was unable to determine the boot disk: {}", err) + } + FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image(err) => { + &format!( + "sled agent was unable to retrieve boot disk phase 2 image: {}", + err + ) + } }; write!(f, "{s}") } @@ -553,39 +602,26 @@ impl IdOrdItem for SkippedMgsUpdate { #[derive( Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, )] +// TODO-K: DO I really need this wrapper function? pub struct SkippedMgsUpdates { - // The IdOrdMap key is the baseboard_id. We only need to know which devices - // were skipped, so only one skipped MGS-managed update is allowed for a - // given baseboard. - pub by_baseboard: IdOrdMap, + pub updates: Vec } impl SkippedMgsUpdates { - pub fn new() -> Self { - Self { by_baseboard: IdOrdMap::new() } - } - - pub fn iter(&self) -> impl Iterator { - self.into_iter() + pub fn new() -> Self { + Self { updates: Vec::new() } } + // TODO-K: I might need iter and into_iter pub fn is_empty(&self) -> bool { - self.by_baseboard.is_empty() + self.updates.is_empty() } - pub fn insert( + pub fn push( &mut self, update: SkippedMgsUpdate, - ) -> Option { - self.by_baseboard.insert_overwrite(update) - } -} - -impl<'a> IntoIterator for &'a SkippedMgsUpdates { - type Item = &'a SkippedMgsUpdate; - type IntoIter = iddqd::id_ord_map::Iter<'a, SkippedMgsUpdate>; - fn into_iter(self) -> Self::IntoIter { - self.by_baseboard.iter() + ) { + self.updates.push(update) } } @@ -630,7 +666,7 @@ impl fmt::Display for PlanningMgsUpdatesStepReport { let n = pending_mgs_updates.len(); let s = plural(n); writeln!(f, "* {n} skipped MGS update{s}:")?; - for update in skipped_mgs_updates.iter() { + for update in &skipped_mgs_updates.updates { writeln!( f, " * {} {}: {}", From 36c788ecc9282eb39b30da169b7d6ad067048453 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 10:07:44 -0600 Subject: [PATCH 10/55] fix no pending updates bug --- .../planning/src/mgs_updates/host_phase_1.rs | 25 ++++- .../planning/src/mgs_updates/mod.rs | 105 ++++++++++++------ nexus/types/src/deployment/planning_report.rs | 16 ++- 3 files changed, 97 insertions(+), 49 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index eda3a37419d..c86017185ee 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -7,12 +7,12 @@ use super::MgsUpdateStatus; use super::MgsUpdateStatusError; use gateway_client::types::SpType; -use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::deployment::BlueprintArtifactVersion; use nexus_types::deployment::BlueprintHostPhase2DesiredContents; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateHostPhase1Details; +use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use omicron_common::api::external::TufArtifactMeta; @@ -271,7 +271,10 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> Result, FailedMgsUpdateReason> { +) -> Result< + Option<(PendingMgsUpdate, PendingHostPhase2Changes)>, + FailedMgsUpdateReason, +> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( log, @@ -325,7 +328,9 @@ pub(super) fn try_make_update( baseboard_id, "err" => err, ); - return Err(FailedMgsUpdateReason::UnableToDetermineBootDisk(err.to_string())); + return Err(FailedMgsUpdateReason::UnableToDetermineBootDisk( + err.to_string(), + )); } }; let active_phase_2_hash = @@ -342,7 +347,11 @@ pub(super) fn try_make_update( "boot_disk" => ?boot_disk, "err" => err, ); - return Err(FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image(err.to_string())); + return Err( + FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image( + err.to_string(), + ), + ); } }; @@ -378,7 +387,9 @@ pub(super) fn try_make_update( "active_phase_1_slot" => ?active_phase_1_slot, "boot_disk" => ?boot_disk, ); - return Err(FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch); + return Err( + FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch, + ); } let Some(active_phase_1_hash) = inventory @@ -409,7 +420,9 @@ pub(super) fn try_make_update( baseboard_id, "slot" => ?active_phase_1_slot.toggled(), ); - return Err(FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory); + return Err( + FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory, + ); }; let mut phase_1_artifacts = Vec::with_capacity(1); diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 00cdb616d95..392d63f23f1 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -17,7 +17,6 @@ use crate::mgs_updates::rot_bootloader::try_make_update_rot_bootloader; use crate::mgs_updates::sp::mgs_update_status_sp; use crate::mgs_updates::sp::try_make_update_sp; -use gateway_client::types::SpType; use gateway_types::rot::RotSlot; use nexus_types::deployment::ExpectedActiveRotSlot; use nexus_types::deployment::ExpectedVersion; @@ -38,7 +37,6 @@ use omicron_common::api::external::TufRepoDescription; use omicron_common::disk::M2Slot; use slog::{error, info, warn}; use slog_error_chain::InlineErrorChain; -use tufaceous_artifact::ArtifactHash; use std::collections::BTreeSet; use std::sync::Arc; use thiserror::Error; @@ -101,6 +99,7 @@ pub(crate) fn plan_mgs_updates( let mut pending_updates = PendingMgsUpdates::new(); let mut pending_host_phase_2_changes = PendingHostPhase2Changes::empty(); let mut boards_preferred = BTreeSet::new(); + // TODO-K: Do I need this here? let mut skipped_mgs_updates = SkippedMgsUpdates::new(); // Determine the status of all currently pending updates by comparing what @@ -210,7 +209,7 @@ pub(crate) fn plan_mgs_updates( } match try_make_update(log, board, inventory, current_artifacts) { - Some((update, mut host_phase_2, skipped_updates)) => { + (Some((update, mut host_phase_2)), skipped_updates) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); @@ -226,8 +225,18 @@ pub(crate) fn plan_mgs_updates( skipped_mgs_updates.push(skipped.clone()); } } - None => { + (None, skipped_updates) => { info!(log, "skipping board for MGS-driven update"; board); + for skipped in &skipped_updates.updates { + warn!( + log, + "update to {} {} has been skipped: {}", + skipped.baseboard_id, + skipped.component, + skipped.reason + ); + skipped_mgs_updates.push(skipped.clone()); + } } } } @@ -522,9 +531,8 @@ fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, - // TODO-K: This is wrong, there could be skipped mgs updates and no - // pending update -) -> Option<(PendingMgsUpdate, PendingHostPhase2Changes, SkippedMgsUpdates)> { + // TODO-K: This looks ugly, a struct should be better +) -> (Option<(PendingMgsUpdate, PendingHostPhase2Changes)>, SkippedMgsUpdates) { let mut skipped_mgs_updates = SkippedMgsUpdates::new(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section @@ -558,11 +566,10 @@ fn try_make_update( { // We have a non-host update; there are no pending host phase 2 changes // necessary. - return Some(( - update, - PendingHostPhase2Changes::empty(), + return ( + Some((update, PendingHostPhase2Changes::empty())), skipped_mgs_updates, - )); + ); } } Err(e) => { @@ -599,12 +606,9 @@ fn try_make_update( inventory, current_artifacts, ) { - Ok(p) => { - match p { - Some((u, c)) => return Some((u, c, skipped_mgs_updates)), - // TODO-K: This is wrong, there could be skipped updates - None => return None, - } + Ok(p) => match p { + Some((u, c)) => return (Some((u, c)), skipped_mgs_updates), + None => return (None, skipped_mgs_updates), }, Err(e) => { skipped_mgs_updates.push(SkippedMgsUpdate { @@ -615,9 +619,36 @@ fn try_make_update( // TODO-K: remove debugging log warn!(log, "HERE2: {:?}", skipped_mgs_updates); // TODO-K: This is wrong there is no pending update - const ARTIFACT_HASH_SP_GIMLET_E: ArtifactHash = - ArtifactHash([1; 32]); - return Some((PendingMgsUpdate{sp_type: SpType::Power, baseboard_id: baseboard_id.clone(), slot_id: 22, details: PendingMgsUpdateDetails::Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion::new("100.0.0").unwrap(), expected_inactive_version: ExpectedVersion::Version(ArtifactVersion::new("100.0.0").unwrap()) }), artifact_hash: ARTIFACT_HASH_SP_GIMLET_E, artifact_version: ArtifactVersion::new("1000.0.0").unwrap()}, PendingHostPhase2Changes::empty(), skipped_mgs_updates)); + // const ARTIFACT_HASH_SP_GIMLET_E: ArtifactHash = + // ArtifactHash([1; 32]); + return ( + None, + // Some(( + // PendingMgsUpdate { + // sp_type: SpType::Power, + // baseboard_id: baseboard_id.clone(), + // slot_id: 22, + // details: PendingMgsUpdateDetails::Sp( + // PendingMgsUpdateSpDetails { + // expected_active_version: ArtifactVersion::new( + // "100.0.0", + // ) + // .unwrap(), + // expected_inactive_version: + // ExpectedVersion::Version( + // ArtifactVersion::new("100.0.0") + // .unwrap(), + // ), + // }, + // ), + // artifact_hash: ARTIFACT_HASH_SP_GIMLET_E, + // artifact_version: ArtifactVersion::new("1000.0.0") + // .unwrap(), + // }, + // PendingHostPhase2Changes::empty(), + // )), + skipped_mgs_updates, + ); } } } @@ -648,7 +679,6 @@ mod test { use dropshot::ConfigLoggingLevel; use gateway_client::types::SpType; use gateway_types::rot::RotSlot; - use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; @@ -656,10 +686,11 @@ mod test { use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; + use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use omicron_test_utils::dev::LogContext; - use std::sync::Arc; use std::collections::BTreeSet; + use std::sync::Arc; use strum::IntoEnumIterator; // Short hand-rolled update sequence that exercises some basic behavior for @@ -1361,22 +1392,28 @@ mod test { // TODO-K: Remove fake boards? let mut fake_boards = BTreeSet::new(); - fake_boards.insert(Arc::new(BaseboardId{part_number: "bob".to_string(), serial_number: "ob".to_string()})); + fake_boards.insert(Arc::new(BaseboardId { + part_number: "bob".to_string(), + serial_number: "ob".to_string(), + })); let collection = test_boards .collection_builder() .stage0_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) .build(); - let PlannedMgsUpdates { pending_updates: updates, skipped_mgs_updates, .. } = - plan_mgs_updates( - log, - &collection, - &fake_boards, - //&BTreeSet::new(), - &PendingMgsUpdates::new(), - &TargetReleaseDescription::TufRepo(repo.clone()), - nmax_updates, - impossible_update_policy, - ); + let PlannedMgsUpdates { + pending_updates: updates, + skipped_mgs_updates, + .. + } = plan_mgs_updates( + log, + &collection, + &fake_boards, + //&BTreeSet::new(), + &PendingMgsUpdates::new(), + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); // TODO-K: Remove this assertion, or improve assert_eq!(skipped_mgs_updates, SkippedMgsUpdates::new()); assert!(updates.is_empty()); diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 8aab5dc3689..75d6a06d7a9 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -565,9 +565,10 @@ impl Display for FailedMgsUpdateReason { FailedMgsUpdateReason::TooManyMatchingArtifacts => { "too many matching artifacts were found" } - FailedMgsUpdateReason::UnableToDetermineBootDisk(err) => { - &format!("sled agent was unable to determine the boot disk: {}", err) - } + FailedMgsUpdateReason::UnableToDetermineBootDisk(err) => &format!( + "sled agent was unable to determine the boot disk: {}", + err + ), FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image(err) => { &format!( "sled agent was unable to retrieve boot disk phase 2 image: {}", @@ -604,11 +605,11 @@ impl IdOrdItem for SkippedMgsUpdate { )] // TODO-K: DO I really need this wrapper function? pub struct SkippedMgsUpdates { - pub updates: Vec + pub updates: Vec, } impl SkippedMgsUpdates { - pub fn new() -> Self { + pub fn new() -> Self { Self { updates: Vec::new() } } // TODO-K: I might need iter and into_iter @@ -617,10 +618,7 @@ impl SkippedMgsUpdates { self.updates.is_empty() } - pub fn push( - &mut self, - update: SkippedMgsUpdate, - ) { + pub fn push(&mut self, update: SkippedMgsUpdate) { self.updates.push(update) } } From ed81591e955a3b331fb33057335537e096745b75 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 13:07:44 -0600 Subject: [PATCH 11/55] Clean up --- .../planning/src/mgs_updates/mod.rs | 77 +++++++------------ .../planning/src/mgs_updates/test_helpers.rs | 2 +- nexus/types/src/deployment.rs | 27 ++++--- nexus/types/src/deployment/planning_report.rs | 4 + 4 files changed, 44 insertions(+), 66 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 392d63f23f1..375290b16ad 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -209,34 +209,36 @@ pub(crate) fn plan_mgs_updates( } match try_make_update(log, board, inventory, current_artifacts) { - (Some((update, mut host_phase_2)), skipped_updates) => { + (Some((update, mut host_phase_2)), mut skipped_updates) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); // TODO-K: Are the logs necessary? - for skipped in &skipped_updates.updates { - warn!( - log, - "update to {} {} has been skipped: {}", - skipped.baseboard_id, - skipped.component, - skipped.reason - ); - skipped_mgs_updates.push(skipped.clone()); - } + //for skipped in &skipped_updates.updates { + // warn!( + // log, + // "update to {} {} has been skipped: {}", + // skipped.baseboard_id, + // skipped.component, + // skipped.reason + // ); + // skipped_mgs_updates.push(skipped.clone()); + //} + skipped_mgs_updates.append(&mut skipped_updates); } - (None, skipped_updates) => { + (None, mut skipped_updates) => { info!(log, "skipping board for MGS-driven update"; board); - for skipped in &skipped_updates.updates { - warn!( - log, - "update to {} {} has been skipped: {}", - skipped.baseboard_id, - skipped.component, - skipped.reason - ); - skipped_mgs_updates.push(skipped.clone()); - } + //for skipped in &skipped_updates.updates { + // warn!( + // log, + // "update to {} {} has been skipped: {}", + // skipped.baseboard_id, + // skipped.component, + // skipped.reason + // ); + // skipped_mgs_updates.push(skipped.clone()); + //} + skipped_mgs_updates.append(&mut skipped_updates); } } } @@ -579,7 +581,7 @@ fn try_make_update( reason: e, }); // TODO-K: remove debugging log - warn!(log, "HERE: {:?}", skipped_mgs_updates); + // warn!(log, "HERE: {:?}", skipped_mgs_updates); } } @@ -617,36 +619,9 @@ fn try_make_update( reason: e, }); // TODO-K: remove debugging log - warn!(log, "HERE2: {:?}", skipped_mgs_updates); - // TODO-K: This is wrong there is no pending update - // const ARTIFACT_HASH_SP_GIMLET_E: ArtifactHash = - // ArtifactHash([1; 32]); + // warn!(log, "HERE2: {:?}", skipped_mgs_updates); return ( None, - // Some(( - // PendingMgsUpdate { - // sp_type: SpType::Power, - // baseboard_id: baseboard_id.clone(), - // slot_id: 22, - // details: PendingMgsUpdateDetails::Sp( - // PendingMgsUpdateSpDetails { - // expected_active_version: ArtifactVersion::new( - // "100.0.0", - // ) - // .unwrap(), - // expected_inactive_version: - // ExpectedVersion::Version( - // ArtifactVersion::new("100.0.0") - // .unwrap(), - // ), - // }, - // ), - // artifact_hash: ARTIFACT_HASH_SP_GIMLET_E, - // artifact_version: ArtifactVersion::new("1000.0.0") - // .unwrap(), - // }, - // PendingHostPhase2Changes::empty(), - // )), skipped_mgs_updates, ); } diff --git a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs index 53d903e641f..982c3a11c74 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs @@ -608,7 +608,7 @@ impl ExpectedUpdates { ) { let sp_type = update.sp_type; let sp_slot = update.slot_id; - let component = MgsUpdateComponent::from(&update.details); + let component = update.details.clone().into(); println!("found update: {} slot {}", sp_type, sp_slot); let ExpectedUpdate { expected_serial, expected_artifact, .. } = self .updates diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 44171b3a1e5..2ecc6acc672 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -1314,20 +1314,6 @@ pub enum MgsUpdateComponent { HostOs, } -// TODO-K: Implement into() instead -impl From<&'_ PendingMgsUpdateDetails> for MgsUpdateComponent { - fn from(value: &'_ PendingMgsUpdateDetails) -> Self { - match value { - PendingMgsUpdateDetails::Rot { .. } => Self::Rot, - PendingMgsUpdateDetails::RotBootloader { .. } => { - Self::RotBootloader - } - PendingMgsUpdateDetails::Sp { .. } => Self::Sp, - PendingMgsUpdateDetails::HostPhase1(_) => Self::HostOs, - } - } -} - impl Display for MgsUpdateComponent { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { @@ -1527,6 +1513,19 @@ pub enum PendingMgsUpdateDetails { HostPhase1(PendingMgsUpdateHostPhase1Details), } +impl Into for PendingMgsUpdateDetails { + fn into(self) -> MgsUpdateComponent { + match self { + PendingMgsUpdateDetails::Rot { .. } => MgsUpdateComponent::Rot, + PendingMgsUpdateDetails::RotBootloader { .. } => { + MgsUpdateComponent::RotBootloader + } + PendingMgsUpdateDetails::Sp { .. } => MgsUpdateComponent::Sp, + PendingMgsUpdateDetails::HostPhase1(_) => MgsUpdateComponent::HostOs, + } + } +} + impl slog::KV for PendingMgsUpdateDetails { fn serialize( &self, diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 75d6a06d7a9..964c3610c91 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -621,6 +621,10 @@ impl SkippedMgsUpdates { pub fn push(&mut self, update: SkippedMgsUpdate) { self.updates.push(update) } + + pub fn append(&mut self, other: &mut Self) { + self.updates.append(&mut other.updates); + } } #[derive( From 4efa7766b123e34f2b244563b549f0593f0cdd03 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 13:40:46 -0600 Subject: [PATCH 12/55] error type clean up --- .../planning/src/mgs_updates/mod.rs | 10 +-- nexus/types/src/deployment.rs | 4 +- nexus/types/src/deployment/planning_report.rs | 74 +++++-------------- 3 files changed, 23 insertions(+), 65 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 375290b16ad..86d07aa3de4 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -213,7 +213,7 @@ pub(crate) fn plan_mgs_updates( info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); pending_host_phase_2_changes.append(&mut host_phase_2); - // TODO-K: Are the logs necessary? + // TODO-K: Remove these debugging logs //for skipped in &skipped_updates.updates { // warn!( // log, @@ -222,12 +222,12 @@ pub(crate) fn plan_mgs_updates( // skipped.component, // skipped.reason // ); - // skipped_mgs_updates.push(skipped.clone()); //} skipped_mgs_updates.append(&mut skipped_updates); } (None, mut skipped_updates) => { info!(log, "skipping board for MGS-driven update"; board); + // TODO-K: Remove these debugging logs //for skipped in &skipped_updates.updates { // warn!( // log, @@ -236,7 +236,6 @@ pub(crate) fn plan_mgs_updates( // skipped.component, // skipped.reason // ); - // skipped_mgs_updates.push(skipped.clone()); //} skipped_mgs_updates.append(&mut skipped_updates); } @@ -620,10 +619,7 @@ fn try_make_update( }); // TODO-K: remove debugging log // warn!(log, "HERE2: {:?}", skipped_mgs_updates); - return ( - None, - skipped_mgs_updates, - ); + return (None, skipped_mgs_updates); } } } diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 2ecc6acc672..2787439e66d 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -1521,7 +1521,9 @@ impl Into for PendingMgsUpdateDetails { MgsUpdateComponent::RotBootloader } PendingMgsUpdateDetails::Sp { .. } => MgsUpdateComponent::Sp, - PendingMgsUpdateDetails::HostPhase1(_) => MgsUpdateComponent::HostOs, + PendingMgsUpdateDetails::HostPhase1(_) => { + MgsUpdateComponent::HostOs + } } } } diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 964c3610c91..3364df9ea83 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -30,9 +30,10 @@ use serde::Serialize; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; -use std::fmt::Display; +//use std::fmt::Display; use std::fmt::Write; use std::sync::Arc; +use thiserror::Error; /// A full blueprint planning report. Other than the blueprint ID, each /// field corresponds to a step in the update planner, i.e., a subroutine @@ -481,6 +482,7 @@ impl PlanningMupdateOverrideStepReport { /// Describes the reason why an SP component failed to update #[derive( + Error, Debug, Deserialize, Serialize, @@ -494,92 +496,51 @@ impl PlanningMupdateOverrideStepReport { )] pub enum FailedMgsUpdateReason { /// The active host phase 1 slot does not match the boot disk + #[error("active phase 1 slot does not match boot disk")] ActiveHostPhase1SlotBootDiskMismatch, /// The active host phase 1 hash was not found in inventory + #[error("active host phase 1 hash is not in inventory")] ActiveHostPhase1HashNotInInventory, /// The active host phase 1 slot was not found in inventory + #[error("active host phase 1 slot is not in inventory")] ActiveHostPhase1SlotNotInInventory, /// The component's caboose was missing a value for "sign" + #[error("caboose is missing sign")] CabooseMissingSign, /// The component's caboose was not found in the inventory + #[error("caboose is not in inventory")] CabooseNotInInventory, /// The version in the caboose or artifact was not able to be parsed + #[error("version could not be parsed")] FailedVersionParse, /// The inactive host phase 1 hash was not found in inventory + #[error("inactive host phase 1 hash is not in inventory")] InactiveHostPhase1HashNotInInventory, /// Last reconciliation details were not found in inventory + #[error("sled agent last reconciliation is not in inventory")] LastReconciliationNotInInventory, /// No artifact with the required conditions for the component was found + #[error("no matching artifact was found")] NoMatchingArtifactFound, /// Sled agent info was not found in inventory + #[error("sled agent info is not in inventory")] SledAgentInfoNotInInventory, /// The component's corresponding SP was not found in the inventory + #[error("corresponding SP is not in inventory")] SpNotInInventory, /// Too many artifacts with the required conditions for the component were /// found + #[error("too many matching artifacts were found")] TooManyMatchingArtifacts, /// The sled agent reported an error determining the boot disk + #[error("sled agent was unable to determine the boot disk: {0:?}")] UnableToDetermineBootDisk(String), /// The sled agent reported an error retrieving boot disk phase 2 image /// details + #[error("sled agent was unable to retrieve boot disk phase 2 image: {0:?}")] UnableToRetrieveBootDiskPhase2Image(String), } -// TODO-K: Keep these or the current logs or treat this more like a real error? -impl Display for FailedMgsUpdateReason { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch => { - "active phase 1 slot does not match boot disk" - } - FailedMgsUpdateReason::ActiveHostPhase1HashNotInInventory => { - "active host phase 1 hash is not in inventory" - } - FailedMgsUpdateReason::ActiveHostPhase1SlotNotInInventory => { - "active host phase 1 slot is not in inventory" - } - FailedMgsUpdateReason::CabooseMissingSign => { - "caboose is missing sign" - } - FailedMgsUpdateReason::CabooseNotInInventory => { - "caboose is not in inventory" - } - FailedMgsUpdateReason::FailedVersionParse => { - "version could not be parsed" - } - FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory => { - "inactive host phase 1 hash is not in inventory" - } - FailedMgsUpdateReason::LastReconciliationNotInInventory => { - "sled agent last reconciliation is not in inventory" - } - FailedMgsUpdateReason::NoMatchingArtifactFound => { - "no matching artifact was found" - } - FailedMgsUpdateReason::SpNotInInventory => { - "corresponding SP is not in inventory" - } - FailedMgsUpdateReason::SledAgentInfoNotInInventory => { - "sled agent info is not in inventory" - } - FailedMgsUpdateReason::TooManyMatchingArtifacts => { - "too many matching artifacts were found" - } - FailedMgsUpdateReason::UnableToDetermineBootDisk(err) => &format!( - "sled agent was unable to determine the boot disk: {}", - err - ), - FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image(err) => { - &format!( - "sled agent was unable to retrieve boot disk phase 2 image: {}", - err - ) - } - }; - write!(f, "{s}") - } -} - #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] @@ -603,7 +564,6 @@ impl IdOrdItem for SkippedMgsUpdate { #[derive( Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, )] -// TODO-K: DO I really need this wrapper function? pub struct SkippedMgsUpdates { pub updates: Vec, } From 14a43ac642920597b7bf5845f8b9e946a007e0cc Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 15:11:02 -0600 Subject: [PATCH 13/55] clean up tuple mess --- .../planning/src/mgs_updates/host_phase_1.rs | 2 +- .../planning/src/mgs_updates/mod.rs | 105 +++++++++++------- 2 files changed, 66 insertions(+), 41 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index c86017185ee..f4e832542a4 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -33,7 +33,7 @@ use tufaceous_artifact::ArtifactKind; /// /// This is generated by the planning process whenever it also generates host /// phase 1 updates. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct PendingHostPhase2Changes { by_sled: BTreeMap, } diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 86d07aa3de4..f0d60bdd275 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -208,38 +208,35 @@ pub(crate) fn plan_mgs_updates( }; } - match try_make_update(log, board, inventory, current_artifacts) { - (Some((update, mut host_phase_2)), mut skipped_updates) => { + let PendingMgsUpdateActions { + pending_update, + pending_host_os_phase2_changes: mut host_phase_2, + mut skipped_updates, + } = try_make_update(log, board, inventory, current_artifacts); + + match pending_update { + Some(update) => { info!(log, "configuring MGS-driven update"; &update); pending_updates.insert(update); - pending_host_phase_2_changes.append(&mut host_phase_2); - // TODO-K: Remove these debugging logs - //for skipped in &skipped_updates.updates { - // warn!( - // log, - // "update to {} {} has been skipped: {}", - // skipped.baseboard_id, - // skipped.component, - // skipped.reason - // ); - //} - skipped_mgs_updates.append(&mut skipped_updates); } - (None, mut skipped_updates) => { - info!(log, "skipping board for MGS-driven update"; board); - // TODO-K: Remove these debugging logs - //for skipped in &skipped_updates.updates { - // warn!( - // log, - // "update to {} {} has been skipped: {}", - // skipped.baseboard_id, - // skipped.component, - // skipped.reason - // ); - //} - skipped_mgs_updates.append(&mut skipped_updates); + None => { + info!(log, "skipping board for MGS-driven update"; board) } } + + // TODO-K: Remove debugging logs + //for skipped in &skipped_updates.updates { + // warn!( + // log, + // "update to {} {} has been skipped: {}", + // skipped.baseboard_id, + // skipped.component, + // skipped.reason + // ); + //} + + pending_host_phase_2_changes.append(&mut host_phase_2); + skipped_mgs_updates.append(&mut skipped_updates); } info!(log, "ran out of boards for MGS-driven update"); @@ -523,6 +520,13 @@ fn mgs_update_status_inactive_versions( } } +#[derive(Clone, Debug, Eq, PartialEq)] +struct PendingMgsUpdateActions { + pending_update: Option, + pending_host_os_phase2_changes: PendingHostPhase2Changes, + skipped_updates: SkippedMgsUpdates, +} + /// Determine if the given baseboard needs any MGS-driven update (e.g., update /// to its SP, RoT, etc.). If so, returns the update and a set of changes that /// need to be made to sled configs related to host phase 2 images (this set @@ -532,9 +536,8 @@ fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, - // TODO-K: This looks ugly, a struct should be better -) -> (Option<(PendingMgsUpdate, PendingHostPhase2Changes)>, SkippedMgsUpdates) { - let mut skipped_mgs_updates = SkippedMgsUpdates::new(); +) -> PendingMgsUpdateActions { + let mut skipped_updates = SkippedMgsUpdates::new(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". @@ -567,14 +570,17 @@ fn try_make_update( { // We have a non-host update; there are no pending host phase 2 changes // necessary. - return ( - Some((update, PendingHostPhase2Changes::empty())), - skipped_mgs_updates, - ); + // TODO-K: Clean up all of the returns + return PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes: + PendingHostPhase2Changes::empty(), + skipped_updates, + }; } } Err(e) => { - skipped_mgs_updates.push(SkippedMgsUpdate { + skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::RotBootloader, reason: e, @@ -608,18 +614,37 @@ fn try_make_update( current_artifacts, ) { Ok(p) => match p { - Some((u, c)) => return (Some((u, c)), skipped_mgs_updates), - None => return (None, skipped_mgs_updates), + Some((update, pending_host_os_phase2_changes)) => { + return PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes, + skipped_updates, + }; + } + // (Some((u, c)), skipped_updates), + None => { + return PendingMgsUpdateActions { + pending_update: None, + pending_host_os_phase2_changes: + PendingHostPhase2Changes::empty(), + skipped_updates, + }; + } }, Err(e) => { - skipped_mgs_updates.push(SkippedMgsUpdate { + skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::HostOs, reason: e, }); // TODO-K: remove debugging log - // warn!(log, "HERE2: {:?}", skipped_mgs_updates); - return (None, skipped_mgs_updates); + // warn!(log, "HERE2: {:?}", skipped_updates); + return PendingMgsUpdateActions { + pending_update: None, + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( + ), + skipped_updates, + }; } } } From b0409133fec39650089de95edd2157154fba8639 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 15:44:04 -0600 Subject: [PATCH 14/55] clean up --- .../planning/src/mgs_updates/mod.rs | 81 ++++++++++--------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index f0d60bdd275..c52f13cc948 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -543,51 +543,56 @@ fn try_make_update( // "Update Sequence". // // TODO-K: Will have to clean this up, the nesting will become horrible - match try_make_update_rot_bootloader( + let pending_rot_bootloader = match try_make_update_rot_bootloader( log, baseboard_id, inventory, current_artifacts, ) { - Ok(p) => { - if let Some(update) = p - .or_else(|| { - try_make_update_rot( - log, - baseboard_id, - inventory, - current_artifacts, - ) - }) - .or_else(|| { - try_make_update_sp( - log, - baseboard_id, - inventory, - current_artifacts, - ) - }) - { - // We have a non-host update; there are no pending host phase 2 changes - // necessary. - // TODO-K: Clean up all of the returns - return PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes: - PendingHostPhase2Changes::empty(), - skipped_updates, - }; - } - } + Ok(p) => match p { + Some(update) => Some(PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( + ), + skipped_updates: skipped_updates.clone(), + }), + None => None, + }, Err(e) => { skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::RotBootloader, reason: e, }); + None // TODO-K: remove debugging log // warn!(log, "HERE: {:?}", skipped_mgs_updates); } + }; + + if let Some(update_actions) = pending_rot_bootloader { + return update_actions; + } + + if let Some(update) = + try_make_update_rot(log, baseboard_id, inventory, current_artifacts) + .or_else(|| { + try_make_update_sp( + log, + baseboard_id, + inventory, + current_artifacts, + ) + }) + { + // We have a non-host update; there are no pending host phase 2 changes + // necessary. + // TODO-K: Clean up all of the returns + return PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), + skipped_updates, + }; } // if let Some(update) = try_make_update_rot_bootloader( @@ -615,20 +620,20 @@ fn try_make_update( ) { Ok(p) => match p { Some((update, pending_host_os_phase2_changes)) => { - return PendingMgsUpdateActions { + PendingMgsUpdateActions { pending_update: Some(update), pending_host_os_phase2_changes, skipped_updates, - }; + } } - // (Some((u, c)), skipped_updates), None => { - return PendingMgsUpdateActions { + // TODO-K: Add a new() method for PendingMgsUpdateActions? + PendingMgsUpdateActions { pending_update: None, pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), skipped_updates, - }; + } } }, Err(e) => { @@ -639,12 +644,12 @@ fn try_make_update( }); // TODO-K: remove debugging log // warn!(log, "HERE2: {:?}", skipped_updates); - return PendingMgsUpdateActions { + PendingMgsUpdateActions { pending_update: None, pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( ), skipped_updates, - }; + } } } } From 70f8495df4263cb1bc0cec71eed95f6626c52c85 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 20:22:40 -0600 Subject: [PATCH 15/55] fully working sample --- .../planning/src/mgs_updates/mod.rs | 96 ++++++++++++------- .../planning/src/mgs_updates/rot.rs | 23 ++--- .../planning/src/mgs_updates/sp.rs | 19 ++-- nexus/types/src/deployment/planning_report.rs | 4 + 4 files changed, 85 insertions(+), 57 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index c52f13cc948..c9e98f752d7 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -574,43 +574,67 @@ fn try_make_update( return update_actions; } - if let Some(update) = - try_make_update_rot(log, baseboard_id, inventory, current_artifacts) - .or_else(|| { - try_make_update_sp( - log, - baseboard_id, - inventory, - current_artifacts, - ) - }) - { - // We have a non-host update; there are no pending host phase 2 changes - // necessary. - // TODO-K: Clean up all of the returns - return PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), - skipped_updates, - }; + let pending_rot = match try_make_update_rot( + log, + baseboard_id, + inventory, + current_artifacts, + ) { + Ok(p) => match p { + Some(update) => Some(PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( + ), + skipped_updates: skipped_updates.clone(), + }), + None => None, + }, + Err(e) => { + skipped_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::Rot, + reason: e, + }); + None + // TODO-K: remove debugging log + // warn!(log, "HERE: {:?}", skipped_mgs_updates); + } + }; + + if let Some(update_actions) = pending_rot { + return update_actions; } - // if let Some(update) = try_make_update_rot_bootloader( - // log, - // baseboard_id, - // inventory, - // current_artifacts, - // ) - // .or_else(|| { - // try_make_update_rot(log, baseboard_id, inventory, current_artifacts) - // }) - // .or_else(|| { - // try_make_update_sp(log, baseboard_id, inventory, current_artifacts) - // }) { - // // We have a non-host update; there are no pending host phase 2 changes - // // necessary. - // return Some((update, PendingHostPhase2Changes::empty())); - // } + let pending_sp = match try_make_update_sp( + log, + baseboard_id, + inventory, + current_artifacts, + ) { + Ok(p) => match p { + Some(update) => Some(PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( + ), + skipped_updates: skipped_updates.clone(), + }), + None => None, + }, + Err(e) => { + skipped_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::Sp, + reason: e, + }); + None + // TODO-K: remove debugging log + // warn!(log, "HERE: {:?}", skipped_mgs_updates); + } + }; + + if let Some(update_actions) = pending_sp { + return update_actions; + } match host_phase_1::try_make_update( log, @@ -642,8 +666,6 @@ fn try_make_update( component: MgsUpdateComponent::HostOs, reason: e, }); - // TODO-K: remove debugging log - // warn!(log, "HERE2: {:?}", skipped_updates); PendingMgsUpdateActions { pending_update: None, pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index f7526cae2aa..8ada03216d0 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -13,6 +13,7 @@ use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateRotDetails; +use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -105,7 +106,7 @@ pub fn try_make_update_rot( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> Option { +) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( log, @@ -113,7 +114,7 @@ pub fn try_make_update_rot( (missing SP info from inventory)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::SpNotInInventory); }; let Some(rot_state) = inventory.rots.get(baseboard_id) else { @@ -123,7 +124,7 @@ pub fn try_make_update_rot( (missing RoT state from inventory)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::RotStateNotInInventory); }; let active_slot = rot_state.active_slot; @@ -137,7 +138,7 @@ pub fn try_make_update_rot( (missing active slot {active_slot} caboose from inventory)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::CabooseNotInInventory); }; let Ok(expected_active_version) = active_caboose.caboose.version.parse() @@ -149,7 +150,7 @@ pub fn try_make_update_rot( baseboard_id, "found_version" => &active_caboose.caboose.version, ); - return None; + return Err(FailedMgsUpdateReason::FailedVersionParse); }; let board = &active_caboose.caboose.board; @@ -160,7 +161,7 @@ pub fn try_make_update_rot( (missing sign in caboose from inventory)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::CabooseMissingSign); }; let matching_artifacts: Vec<_> = current_artifacts @@ -224,7 +225,7 @@ pub fn try_make_update_rot( "cannot configure RoT update for board (no matching artifact)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } if matching_artifacts.len() > 1 { @@ -241,7 +242,7 @@ pub fn try_make_update_rot( // needed. if artifact.id.version == expected_active_version { debug!(log, "no RoT update needed for board"; baseboard_id); - return None; + return Ok(None); } let expected_active_slot = ExpectedActiveRotSlot { @@ -267,11 +268,11 @@ pub fn try_make_update_rot( (found inactive slot contents but version was not valid)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::FailedVersionParse); } }; - Some(PendingMgsUpdate { + Ok(Some(PendingMgsUpdate { baseboard_id: baseboard_id.clone(), sp_type: sp_info.sp_type, slot_id: sp_info.sp_slot, @@ -287,5 +288,5 @@ pub fn try_make_update_rot( }), artifact_hash: artifact.hash, artifact_version: artifact.id.version.clone(), - }) + })) } diff --git a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs index 94de2ee1b8a..d3f1a7cefba 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs @@ -11,6 +11,7 @@ use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; +use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -59,7 +60,7 @@ pub fn try_make_update_sp( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> Option { +) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { warn!( log, @@ -67,7 +68,7 @@ pub fn try_make_update_sp( (missing SP info from inventory)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::SpNotInInventory); }; let Some(active_caboose) = @@ -79,7 +80,7 @@ pub fn try_make_update_sp( (missing active caboose from inventory)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::CabooseNotInInventory); }; let Ok(expected_active_version) = active_caboose.caboose.version.parse() @@ -91,7 +92,7 @@ pub fn try_make_update_sp( baseboard_id, "found_version" => &active_caboose.caboose.version, ); - return None; + return Err(FailedMgsUpdateReason::FailedVersionParse); }; let board = &active_caboose.caboose.board; @@ -137,7 +138,7 @@ pub fn try_make_update_sp( "cannot configure SP update for board (no matching artifact)"; baseboard_id, ); - return None; + return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } if matching_artifacts.len() > 1 { @@ -153,7 +154,7 @@ pub fn try_make_update_sp( // needed. if artifact.id.version == expected_active_version { debug!(log, "no SP update needed for board"; baseboard_id); - return None; + return Ok(None); } // Begin configuring an update. @@ -171,11 +172,11 @@ pub fn try_make_update_sp( (found inactive slot contents but version was not valid)"; baseboard_id ); - return None; + return Err(FailedMgsUpdateReason::FailedVersionParse); } }; - Some(PendingMgsUpdate { + Ok(Some(PendingMgsUpdate { baseboard_id: baseboard_id.clone(), sp_type: sp_info.sp_type, slot_id: sp_info.sp_slot, @@ -185,5 +186,5 @@ pub fn try_make_update_sp( }), artifact_hash: artifact.hash, artifact_version: artifact.id.version.clone(), - }) + })) } diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 3364df9ea83..f018ea64027 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -509,6 +509,7 @@ pub enum FailedMgsUpdateReason { CabooseMissingSign, /// The component's caboose was not found in the inventory #[error("caboose is not in inventory")] + // TODO-K: add CabooseWhich CabooseNotInInventory, /// The version in the caboose or artifact was not able to be parsed #[error("version could not be parsed")] @@ -522,6 +523,9 @@ pub enum FailedMgsUpdateReason { /// No artifact with the required conditions for the component was found #[error("no matching artifact was found")] NoMatchingArtifactFound, + /// RoT state was not found in inventory + #[error("rot state is not in inventory")] + RotStateNotInInventory, /// Sled agent info was not found in inventory #[error("sled agent info is not in inventory")] SledAgentInfoNotInInventory, From e1ea3d2a36b5684de706181f85616a7beb1efaef Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 20:52:00 -0600 Subject: [PATCH 16/55] refactor try_make_update --- .../planning/src/mgs_updates/mod.rs | 177 ++++++++---------- 1 file changed, 73 insertions(+), 104 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index c9e98f752d7..34e40f8eda2 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -28,6 +28,7 @@ use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; +use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::deployment::planning_report::SkippedMgsUpdate; use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; @@ -538,101 +539,74 @@ fn try_make_update( current_artifacts: &TufRepoDescription, ) -> PendingMgsUpdateActions { let mut skipped_updates = SkippedMgsUpdates::new(); + // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". - // - // TODO-K: Will have to clean this up, the nesting will become horrible - let pending_rot_bootloader = match try_make_update_rot_bootloader( - log, - baseboard_id, - inventory, - current_artifacts, - ) { - Ok(p) => match p { - Some(update) => Some(PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( - ), - skipped_updates: skipped_updates.clone(), + let attempts: [( + MgsUpdateComponent, + Box< + dyn Fn() -> Result, FailedMgsUpdateReason>, + >, + ); 3] = [ + ( + MgsUpdateComponent::RotBootloader, + Box::new(|| { + try_make_update_rot_bootloader( + log, + baseboard_id, + inventory, + current_artifacts, + ) }), - None => None, - }, - Err(e) => { - skipped_updates.push(SkippedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component: MgsUpdateComponent::RotBootloader, - reason: e, - }); - None - // TODO-K: remove debugging log - // warn!(log, "HERE: {:?}", skipped_mgs_updates); - } - }; - - if let Some(update_actions) = pending_rot_bootloader { - return update_actions; - } - - let pending_rot = match try_make_update_rot( - log, - baseboard_id, - inventory, - current_artifacts, - ) { - Ok(p) => match p { - Some(update) => Some(PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( - ), - skipped_updates: skipped_updates.clone(), + ), + ( + MgsUpdateComponent::Rot, + Box::new(|| { + try_make_update_rot( + log, + baseboard_id, + inventory, + current_artifacts, + ) }), - None => None, - }, - Err(e) => { - skipped_updates.push(SkippedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component: MgsUpdateComponent::Rot, - reason: e, - }); - None - // TODO-K: remove debugging log - // warn!(log, "HERE: {:?}", skipped_mgs_updates); - } - }; - - if let Some(update_actions) = pending_rot { - return update_actions; - } - - let pending_sp = match try_make_update_sp( - log, - baseboard_id, - inventory, - current_artifacts, - ) { - Ok(p) => match p { - Some(update) => Some(PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( - ), - skipped_updates: skipped_updates.clone(), + ), + ( + MgsUpdateComponent::Sp, + Box::new(|| { + try_make_update_sp( + log, + baseboard_id, + inventory, + current_artifacts, + ) }), - None => None, - }, - Err(e) => { - skipped_updates.push(SkippedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component: MgsUpdateComponent::Sp, - reason: e, - }); - None - // TODO-K: remove debugging log - // warn!(log, "HERE: {:?}", skipped_mgs_updates); - } - }; + ), + ]; - if let Some(update_actions) = pending_sp { + if let Some(update_actions) = + attempts.into_iter().find_map(|(component, attempt)| { + match attempt() { + Ok(Some(update)) => Some(PendingMgsUpdateActions { + pending_update: Some(update), + // We have a non-host update; there are no pending host + // phase 2 changes necessary. + pending_host_os_phase2_changes: + PendingHostPhase2Changes::empty(), + skipped_updates: skipped_updates.clone(), + }), + Ok(None) => None, + Err(e) => { + skipped_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component, + reason: e, + }); + None + } + } + }) + { return update_actions; } @@ -642,23 +616,18 @@ fn try_make_update( inventory, current_artifacts, ) { - Ok(p) => match p { - Some((update, pending_host_os_phase2_changes)) => { - PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes, - skipped_updates, - } - } - None => { - // TODO-K: Add a new() method for PendingMgsUpdateActions? - PendingMgsUpdateActions { - pending_update: None, - pending_host_os_phase2_changes: - PendingHostPhase2Changes::empty(), - skipped_updates, - } + Ok(Some((update, pending_host_os_phase2_changes))) => { + // TODO-K: create a new() function for this + PendingMgsUpdateActions { + pending_update: Some(update), + pending_host_os_phase2_changes, + skipped_updates, } + } + Ok(None) => PendingMgsUpdateActions { + pending_update: None, + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), + skipped_updates, }, Err(e) => { skipped_updates.push(SkippedMgsUpdate { From f2c7c2ad07261fe02202eb2c5631890914d84611 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 21:40:03 -0600 Subject: [PATCH 17/55] use builder pattern --- .../planning/src/mgs_updates/mod.rs | 95 ++++++++++++++----- 1 file changed, 72 insertions(+), 23 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 34e40f8eda2..452e5b96dc7 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -521,6 +521,7 @@ fn mgs_update_status_inactive_versions( } } +// TODO-K: I should use planned MGS updates #[derive(Clone, Debug, Eq, PartialEq)] struct PendingMgsUpdateActions { pending_update: Option, @@ -528,6 +529,50 @@ struct PendingMgsUpdateActions { skipped_updates: SkippedMgsUpdates, } +impl PendingMgsUpdateActions { + fn new() -> PendingMgsUpdateActions { + PendingMgsUpdateActions { + pending_update: None, + pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), + skipped_updates: SkippedMgsUpdates::new(), + } + } + + fn set_pending_update( + &mut self, + pending_update: PendingMgsUpdate, + ) -> &mut Self { + self.pending_update = Some(pending_update); + self + } + + fn set_pending_host_os_phase2_changes( + &mut self, + pending_host_os_phase2_changes: PendingHostPhase2Changes, + ) -> &mut Self { + self.pending_host_os_phase2_changes = pending_host_os_phase2_changes; + self + } + + fn set_skipped_updates( + &mut self, + skipped_updates: SkippedMgsUpdates, + ) -> &mut Self { + self.skipped_updates = skipped_updates; + self + } + + fn build(&self) -> Self { + PendingMgsUpdateActions { + pending_update: self.pending_update.clone(), + pending_host_os_phase2_changes: self + .pending_host_os_phase2_changes + .clone(), + skipped_updates: self.skipped_updates.clone(), + } + } +} + /// Determine if the given baseboard needs any MGS-driven update (e.g., update /// to its SP, RoT, etc.). If so, returns the update and a set of changes that /// need to be made to sled configs related to host phase 2 images (this set @@ -587,15 +632,21 @@ fn try_make_update( if let Some(update_actions) = attempts.into_iter().find_map(|(component, attempt)| { match attempt() { - Ok(Some(update)) => Some(PendingMgsUpdateActions { - pending_update: Some(update), + // There is a pending update, record it along with any previous + // failed updates + Ok(Some(update)) => { // We have a non-host update; there are no pending host // phase 2 changes necessary. - pending_host_os_phase2_changes: - PendingHostPhase2Changes::empty(), - skipped_updates: skipped_updates.clone(), - }), + let pending_actions = PendingMgsUpdateActions::new() + .set_pending_update(update) + .set_skipped_updates(skipped_updates.clone()) + .build(); + Some(pending_actions) + } + // We don't have any pending actions, the component is already + // at the expected version Ok(None) => None, + // There was a failure, skip the update and record it Err(e) => { skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), @@ -617,30 +668,28 @@ fn try_make_update( current_artifacts, ) { Ok(Some((update, pending_host_os_phase2_changes))) => { - // TODO-K: create a new() function for this - PendingMgsUpdateActions { - pending_update: Some(update), - pending_host_os_phase2_changes, - skipped_updates, - } + PendingMgsUpdateActions::new() + .set_pending_update(update) + .set_pending_host_os_phase2_changes( + pending_host_os_phase2_changes, + ) + .set_skipped_updates(skipped_updates) + .build() } - Ok(None) => PendingMgsUpdateActions { - pending_update: None, - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), - skipped_updates, - }, + // The Host OS is already at the desired version, we only need to pass + // along any previous skipped updates + Ok(None) => PendingMgsUpdateActions::new() + .set_skipped_updates(skipped_updates) + .build(), Err(e) => { skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::HostOs, reason: e, }); - PendingMgsUpdateActions { - pending_update: None, - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty( - ), - skipped_updates, - } + PendingMgsUpdateActions::new() + .set_skipped_updates(skipped_updates) + .build() } } } From 3293c4f60dcaac6f841113409cb7f673b1f72a50 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 22:08:25 -0600 Subject: [PATCH 18/55] remove unnecessary struct --- .../planning/src/mgs_updates/mod.rs | 130 ++++++++---------- 1 file changed, 61 insertions(+), 69 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 452e5b96dc7..e930bc55558 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -73,6 +73,50 @@ pub(crate) struct PlannedMgsUpdates { pub(crate) skipped_mgs_updates: SkippedMgsUpdates, } +impl PlannedMgsUpdates { + fn new() -> Self { + Self { + pending_updates: PendingMgsUpdates::new(), + pending_host_phase_2_changes: PendingHostPhase2Changes::empty(), + skipped_mgs_updates: SkippedMgsUpdates::new(), + } + } + + fn set_pending_update( + &mut self, + pending_update: PendingMgsUpdate, + ) -> &mut Self { + self.pending_updates.insert(pending_update); + self + } + + fn set_pending_host_os_phase2_changes( + &mut self, + pending_host_os_phase2_changes: PendingHostPhase2Changes, + ) -> &mut Self { + self.pending_host_phase_2_changes = pending_host_os_phase2_changes; + self + } + + fn set_skipped_updates( + &mut self, + skipped_updates: SkippedMgsUpdates, + ) -> &mut Self { + self.skipped_mgs_updates = skipped_updates; + self + } + + fn build(&self) -> Self { + Self { + pending_updates: self.pending_updates.clone(), + pending_host_phase_2_changes: self + .pending_host_phase_2_changes + .clone(), + skipped_mgs_updates: self.skipped_mgs_updates.clone(), + } + } +} + /// Generates a new set of `PendingMgsUpdates` based on: /// /// * `inventory`: the latest inventory @@ -209,20 +253,20 @@ pub(crate) fn plan_mgs_updates( }; } - let PendingMgsUpdateActions { - pending_update, - pending_host_os_phase2_changes: mut host_phase_2, - mut skipped_updates, + let PlannedMgsUpdates { + pending_updates: update, + pending_host_phase_2_changes: mut host_phase_2, + skipped_mgs_updates: mut skipped_updates, } = try_make_update(log, board, inventory, current_artifacts); - match pending_update { - Some(update) => { - info!(log, "configuring MGS-driven update"; &update); - pending_updates.insert(update); - } - None => { - info!(log, "skipping board for MGS-driven update"; board) - } + if !update.is_empty() { + // We can safely unwrap because we just created the update with the + // baseboard_id in try_make_update above + let update = update.get(&board).unwrap(); + info!(log, "configuring MGS-driven update"; update); + pending_updates.insert(update.clone()); + } else { + info!(log, "skipping board for MGS-driven update"; board) } // TODO-K: Remove debugging logs @@ -521,58 +565,6 @@ fn mgs_update_status_inactive_versions( } } -// TODO-K: I should use planned MGS updates -#[derive(Clone, Debug, Eq, PartialEq)] -struct PendingMgsUpdateActions { - pending_update: Option, - pending_host_os_phase2_changes: PendingHostPhase2Changes, - skipped_updates: SkippedMgsUpdates, -} - -impl PendingMgsUpdateActions { - fn new() -> PendingMgsUpdateActions { - PendingMgsUpdateActions { - pending_update: None, - pending_host_os_phase2_changes: PendingHostPhase2Changes::empty(), - skipped_updates: SkippedMgsUpdates::new(), - } - } - - fn set_pending_update( - &mut self, - pending_update: PendingMgsUpdate, - ) -> &mut Self { - self.pending_update = Some(pending_update); - self - } - - fn set_pending_host_os_phase2_changes( - &mut self, - pending_host_os_phase2_changes: PendingHostPhase2Changes, - ) -> &mut Self { - self.pending_host_os_phase2_changes = pending_host_os_phase2_changes; - self - } - - fn set_skipped_updates( - &mut self, - skipped_updates: SkippedMgsUpdates, - ) -> &mut Self { - self.skipped_updates = skipped_updates; - self - } - - fn build(&self) -> Self { - PendingMgsUpdateActions { - pending_update: self.pending_update.clone(), - pending_host_os_phase2_changes: self - .pending_host_os_phase2_changes - .clone(), - skipped_updates: self.skipped_updates.clone(), - } - } -} - /// Determine if the given baseboard needs any MGS-driven update (e.g., update /// to its SP, RoT, etc.). If so, returns the update and a set of changes that /// need to be made to sled configs related to host phase 2 images (this set @@ -582,7 +574,7 @@ fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -) -> PendingMgsUpdateActions { +) -> PlannedMgsUpdates { let mut skipped_updates = SkippedMgsUpdates::new(); // We try MGS-driven update components in a hardcoded priority order until @@ -637,7 +629,7 @@ fn try_make_update( Ok(Some(update)) => { // We have a non-host update; there are no pending host // phase 2 changes necessary. - let pending_actions = PendingMgsUpdateActions::new() + let pending_actions = PlannedMgsUpdates::new() .set_pending_update(update) .set_skipped_updates(skipped_updates.clone()) .build(); @@ -668,7 +660,7 @@ fn try_make_update( current_artifacts, ) { Ok(Some((update, pending_host_os_phase2_changes))) => { - PendingMgsUpdateActions::new() + PlannedMgsUpdates::new() .set_pending_update(update) .set_pending_host_os_phase2_changes( pending_host_os_phase2_changes, @@ -678,7 +670,7 @@ fn try_make_update( } // The Host OS is already at the desired version, we only need to pass // along any previous skipped updates - Ok(None) => PendingMgsUpdateActions::new() + Ok(None) => PlannedMgsUpdates::new() .set_skipped_updates(skipped_updates) .build(), Err(e) => { @@ -687,7 +679,7 @@ fn try_make_update( component: MgsUpdateComponent::HostOs, reason: e, }); - PendingMgsUpdateActions::new() + PlannedMgsUpdates::new() .set_skipped_updates(skipped_updates) .build() } From 6d866d6781730403289bc7931029949cc8d96989 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 22:25:43 -0600 Subject: [PATCH 19/55] improve error messages --- .../reconfigurator/planning/src/mgs_updates/mod.rs | 1 - .../reconfigurator/planning/src/mgs_updates/rot.rs | 13 +++++++++---- .../planning/src/mgs_updates/rot_bootloader.rs | 8 ++++++-- nexus/reconfigurator/planning/src/mgs_updates/sp.rs | 4 +++- nexus/reconfigurator/planning/src/planner.rs | 4 ---- nexus/types/src/deployment/planning_report.rs | 11 +++++------ nexus/types/src/inventory.rs | 1 + 7 files changed, 24 insertions(+), 18 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index e930bc55558..958c9180a53 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -144,7 +144,6 @@ pub(crate) fn plan_mgs_updates( let mut pending_updates = PendingMgsUpdates::new(); let mut pending_host_phase_2_changes = PendingHostPhase2Changes::empty(); let mut boards_preferred = BTreeSet::new(); - // TODO-K: Do I need this here? let mut skipped_mgs_updates = SkippedMgsUpdates::new(); // Determine the status of all currently pending updates by comparing what diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index 8ada03216d0..35ae3bef465 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -129,8 +129,9 @@ pub fn try_make_update_rot( let active_slot = rot_state.active_slot; - let Some(active_caboose) = inventory - .caboose_for(CabooseWhich::from_rot_slot(active_slot), baseboard_id) + let active_caboose_which = CabooseWhich::from_rot_slot(active_slot); + let Some(active_caboose) = + inventory.caboose_for(active_caboose_which, baseboard_id) else { warn!( log, @@ -138,7 +139,9 @@ pub fn try_make_update_rot( (missing active slot {active_slot} caboose from inventory)"; baseboard_id, ); - return Err(FailedMgsUpdateReason::CabooseNotInInventory); + return Err(FailedMgsUpdateReason::CabooseNotInInventory( + active_caboose_which, + )); }; let Ok(expected_active_version) = active_caboose.caboose.version.parse() @@ -161,7 +164,9 @@ pub fn try_make_update_rot( (missing sign in caboose from inventory)"; baseboard_id ); - return Err(FailedMgsUpdateReason::CabooseMissingSign); + return Err(FailedMgsUpdateReason::CabooseMissingSign( + active_caboose_which, + )); }; let matching_artifacts: Vec<_> = current_artifacts diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index a1fda4a12d4..7ae401e88e8 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -81,7 +81,9 @@ pub fn try_make_update_rot_bootloader( (missing stage0 caboose from inventory)"; baseboard_id, ); - return Err(FailedMgsUpdateReason::CabooseNotInInventory); + return Err(FailedMgsUpdateReason::CabooseNotInInventory( + CabooseWhich::Stage0, + )); }; let Ok(expected_stage0_version) = stage0_caboose.caboose.version.parse() @@ -104,7 +106,9 @@ pub fn try_make_update_rot_bootloader( (missing sign in stage0 caboose from inventory)"; baseboard_id ); - return Err(FailedMgsUpdateReason::CabooseMissingSign); + return Err(FailedMgsUpdateReason::CabooseMissingSign( + CabooseWhich::Stage0, + )); }; let matching_artifacts: Vec<_> = current_artifacts diff --git a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs index d3f1a7cefba..7bab3aa060a 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs @@ -80,7 +80,9 @@ pub fn try_make_update_sp( (missing active caboose from inventory)"; baseboard_id, ); - return Err(FailedMgsUpdateReason::CabooseNotInInventory); + return Err(FailedMgsUpdateReason::CabooseNotInInventory( + CabooseWhich::SpSlot0, + )); }; let Ok(expected_active_version) = active_caboose.caboose.version.parse() diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 8b7dcd9d1db..9a8882cb1be 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -192,7 +192,6 @@ impl<'a> Planner<'a> { // Only plan MGS-based updates updates if there are no outstanding // MUPdate overrides. - // TODO-K: mgs updates here let mgs_updates = if add_update_blocked_reasons.is_empty() { self.do_plan_mgs_updates()? } else { @@ -221,9 +220,6 @@ impl<'a> Planner<'a> { PlanningZoneUpdatesStepReport::waiting_on( ZoneUpdatesWaitingOn::DiscretionaryZones, ) - // TODO-K: mgs_updates can be empty if some updates failed. Make sure to - // keep track of failed updates and wait until they are on the expected - // new version befor proceeding to update zones } else if !mgs_updates.pending_mgs_updates.is_empty() { // ... or if there are still pending updates for the RoT / SP / // Host OS / etc. ... diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index f018ea64027..0c0699c5162 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -12,6 +12,7 @@ use super::PendingMgsUpdates; use super::PlannerChickenSwitches; use crate::deployment::MgsUpdateComponent; use crate::inventory::BaseboardId; +use crate::inventory::CabooseWhich; use daft::Diffable; use iddqd::IdOrdItem; @@ -505,12 +506,11 @@ pub enum FailedMgsUpdateReason { #[error("active host phase 1 slot is not in inventory")] ActiveHostPhase1SlotNotInInventory, /// The component's caboose was missing a value for "sign" - #[error("caboose is missing sign")] - CabooseMissingSign, + #[error("caboose for {0:?} is missing sign")] + CabooseMissingSign(CabooseWhich), /// The component's caboose was not found in the inventory - #[error("caboose is not in inventory")] - // TODO-K: add CabooseWhich - CabooseNotInInventory, + #[error("caboose for {0:?} is not in inventory")] + CabooseNotInInventory(CabooseWhich), /// The version in the caboose or artifact was not able to be parsed #[error("version could not be parsed")] FailedVersionParse, @@ -576,7 +576,6 @@ impl SkippedMgsUpdates { pub fn new() -> Self { Self { updates: Vec::new() } } - // TODO-K: I might need iter and into_iter pub fn is_empty(&self) -> bool { self.updates.is_empty() diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 7ca78f007ea..24be84bb552 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -469,6 +469,7 @@ pub struct HostPhase1FlashHash { Ord, Deserialize, Serialize, + JsonSchema, )] pub enum CabooseWhich { SpSlot0, From e8f027b1710cc3930e5935ceea713980600d7354 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 22:34:59 -0600 Subject: [PATCH 20/55] clippy --- nexus/reconfigurator/planning/src/mgs_updates/mod.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 958c9180a53..008ff3868ca 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -579,12 +579,9 @@ fn try_make_update( // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". - let attempts: [( - MgsUpdateComponent, - Box< - dyn Fn() -> Result, FailedMgsUpdateReason>, - >, - ); 3] = [ + type UpdateResult = Result, FailedMgsUpdateReason>; + type UpdateFn<'a> = Box UpdateResult + 'a>; + let attempts: [(MgsUpdateComponent, UpdateFn); 3] = [ ( MgsUpdateComponent::RotBootloader, Box::new(|| { From 1da94b0d75892707725f5a2ca51d675bcb758524 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 4 Sep 2025 23:02:33 -0600 Subject: [PATCH 21/55] Make the tests pass --- .../planning/src/mgs_updates/mod.rs | 37 ++++++++++++++++--- nexus/reconfigurator/planning/src/planner.rs | 1 + nexus/types/src/deployment/planning_report.rs | 2 +- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 008ff3868ca..d6e83744ce4 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -709,12 +709,15 @@ mod test { use gateway_client::types::SpType; use gateway_types::rot::RotSlot; use nexus_types::deployment::ExpectedVersion; + use nexus_types::deployment::MgsUpdateComponent; use nexus_types::deployment::PendingMgsUpdateDetails; use nexus_types::deployment::PendingMgsUpdateRotBootloaderDetails; use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; + use nexus_types::deployment::planning_report::FailedMgsUpdateReason; + use nexus_types::deployment::planning_report::SkippedMgsUpdate; use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use omicron_test_utils::dev::LogContext; @@ -1418,13 +1421,14 @@ mod test { // Test that we don't try to update boards that aren't in // `current_boards`, even if they're in inventory and outdated. - - // TODO-K: Remove fake boards? + // + // TODO-K: Remove fake boards and find a better way to test this let mut fake_boards = BTreeSet::new(); - fake_boards.insert(Arc::new(BaseboardId { + let fake_board = Arc::new(BaseboardId { part_number: "bob".to_string(), serial_number: "ob".to_string(), - })); + }); + fake_boards.insert(fake_board.clone()); let collection = test_boards .collection_builder() .stage0_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) @@ -1443,8 +1447,29 @@ mod test { nmax_updates, impossible_update_policy, ); - // TODO-K: Remove this assertion, or improve - assert_eq!(skipped_mgs_updates, SkippedMgsUpdates::new()); + // TODO-K: Find a better way to test functionality + let mut expected_skipped_updates = SkippedMgsUpdates::new(); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board.clone(), + component: MgsUpdateComponent::RotBootloader, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board.clone(), + component: MgsUpdateComponent::Rot, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board.clone(), + component: MgsUpdateComponent::Sp, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board, + component: MgsUpdateComponent::HostOs, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + assert_eq!(skipped_mgs_updates, expected_skipped_updates); assert!(updates.is_empty()); let PlannedMgsUpdates { pending_updates: updates, .. } = plan_mgs_updates( diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 9a8882cb1be..bf0163ea9ec 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -5620,6 +5620,7 @@ pub(crate) mod test { }; } + // TODO-K: Include MGS artifacts for zone update testing fn create_artifacts_at_version( version: &ArtifactVersion, ) -> Vec { diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 0c0699c5162..3bcf9bb1b4d 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -628,7 +628,7 @@ impl fmt::Display for PlanningMgsUpdatesStepReport { } } if !skipped_mgs_updates.is_empty() { - let n = pending_mgs_updates.len(); + let n = skipped_mgs_updates.updates.len(); let s = plural(n); writeln!(f, "* {n} skipped MGS update{s}:")?; for update in &skipped_mgs_updates.updates { From d0d37e62012b471686519ada02777545e6f010a6 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 5 Sep 2025 10:18:16 -0600 Subject: [PATCH 22/55] Fix openapi generation --- nexus/types/src/deployment.rs | 1 + nexus/types/src/deployment/planning_report.rs | 2 + nexus/types/src/inventory.rs | 1 + openapi/nexus-internal.json | 338 +++++++++++++++++- 4 files changed, 340 insertions(+), 2 deletions(-) diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 2787439e66d..27a29275cc7 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -1307,6 +1307,7 @@ impl fmt::Display for BlueprintHostPhase2DesiredContents { Clone, Copy, )] +#[serde(rename_all = "snake_case")] pub enum MgsUpdateComponent { Sp, Rot, diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 3bcf9bb1b4d..0f6a981aeba 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -495,6 +495,8 @@ impl PlanningMupdateOverrideStepReport { Ord, Clone, )] +#[serde(rename_all = "snake_case")] +#[serde(tag = "type", content = "value")] pub enum FailedMgsUpdateReason { /// The active host phase 1 slot does not match the boot disk #[error("active phase 1 slot does not match boot disk")] diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 24be84bb552..4ce66f8450b 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -471,6 +471,7 @@ pub struct HostPhase1FlashHash { Serialize, JsonSchema, )] +#[serde(rename_all = "snake_case")] pub enum CabooseWhich { SpSlot0, SpSlot1, diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 3d41de3fe62..2ff0da4bea8 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -3567,6 +3567,18 @@ "format": "uint64", "minimum": 0 }, + "CabooseWhich": { + "description": "Describes which caboose this is (which component, which slot)", + "type": "string", + "enum": [ + "sp_slot0", + "sp_slot1", + "rot_slot_a", + "rot_slot_b", + "stage0", + "stage0_next" + ] + }, "Certificate": { "type": "object", "properties": { @@ -4780,6 +4792,252 @@ } ] }, + "FailedMgsUpdateReason": { + "description": "Describes the reason why an SP component failed to update", + "oneOf": [ + { + "description": "The active host phase 1 slot does not match the boot disk", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "active_host_phase1_slot_boot_disk_mismatch" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The active host phase 1 hash was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "active_host_phase1_hash_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The active host phase 1 slot was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "active_host_phase1_slot_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The component's caboose was missing a value for \"sign\"", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "caboose_missing_sign" + ] + }, + "value": { + "$ref": "#/components/schemas/CabooseWhich" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The component's caboose was not found in the inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "caboose_not_in_inventory" + ] + }, + "value": { + "$ref": "#/components/schemas/CabooseWhich" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The version in the caboose or artifact was not able to be parsed", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "failed_version_parse" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The inactive host phase 1 hash was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "inactive_host_phase1_hash_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Last reconciliation details were not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "last_reconciliation_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "No artifact with the required conditions for the component was found", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "no_matching_artifact_found" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "RoT state was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "rot_state_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Sled agent info was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "sled_agent_info_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The component's corresponding SP was not found in the inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "sp_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Too many artifacts with the required conditions for the component were found", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "too_many_matching_artifacts" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The sled agent reported an error determining the boot disk", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unable_to_determine_boot_disk" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The sled agent reported an error retrieving boot disk phase 2 image details", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unable_to_retrieve_boot_disk_phase2_image" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "Generation": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "type": "integer", @@ -5532,6 +5790,15 @@ "sp" ] }, + "MgsUpdateComponent": { + "type": "string", + "enum": [ + "sp", + "rot", + "rot_bootloader", + "host_os" + ] + }, "MgsUpdateDriverStatus": { "description": "Status of ongoing update attempts, recently completed attempts, and update requests that are waiting for retry.", "type": "object", @@ -6881,10 +7148,14 @@ "properties": { "pending_mgs_updates": { "$ref": "#/components/schemas/PendingMgsUpdates" + }, + "skipped_mgs_updates": { + "$ref": "#/components/schemas/SkippedMgsUpdates" } }, "required": [ - "pending_mgs_updates" + "pending_mgs_updates", + "skipped_mgs_updates" ] }, "PlanningNoopImageSourceConverted": { @@ -8371,6 +8642,54 @@ "format": "uint64", "minimum": 0 }, + "SkippedMgsUpdate": { + "type": "object", + "properties": { + "baseboard_id": { + "description": "id of the baseboard that we attempted to update", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "component": { + "description": "type of SP component that we attempted to update", + "allOf": [ + { + "$ref": "#/components/schemas/MgsUpdateComponent" + } + ] + }, + "reason": { + "description": "reason why the update failed", + "allOf": [ + { + "$ref": "#/components/schemas/FailedMgsUpdateReason" + } + ] + } + }, + "required": [ + "baseboard_id", + "component", + "reason" + ] + }, + "SkippedMgsUpdates": { + "type": "object", + "properties": { + "updates": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SkippedMgsUpdate" + } + } + }, + "required": [ + "updates" + ] + }, "SledAgentInfo": { "description": "Sent by a sled agent to Nexus to inform about resources", "type": "object", @@ -9476,7 +9795,7 @@ ] }, { - "description": "Waiting on updates to RoT / SP / Host OS / etc.", + "description": "Waiting on updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { @@ -9490,6 +9809,21 @@ "type" ] }, + { + "description": "Waiting on skipped updates to RoT bootloader / RoT / SP / Host OS.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "skipped_mgs_updates" + ] + } + }, + "required": [ + "type" + ] + }, { "description": "Waiting on the same set of blockers zone adds are waiting on.", "type": "object", From 03b74321cb260b178182ba09c515b53a0cc61915 Mon Sep 17 00:00:00 2001 From: karencfv Date: Fri, 5 Sep 2025 11:30:40 -0600 Subject: [PATCH 23/55] Mull over tests --- nexus/reconfigurator/planning/src/planner.rs | 176 +++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index bf0163ea9ec..2e0968379ef 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -2198,6 +2198,13 @@ pub(crate) mod test { use typed_rng::TypedUuidRng; use uuid::Uuid; + const ROT_SIGN_GIMLET: &str = + "1111111111111111111111111111111111111111111111111111111111111111"; + const ROT_SIGN_PSC: &str = + "2222222222222222222222222222222222222222222222222222222222222222"; + const ROT_SIGN_SWITCH: &str = + "3333333333333333333333333333333333333333333333333333333333333333"; + // Generate a ClickhousePolicy ignoring fields we don't care about for /// planner tests fn clickhouse_policy(mode: ClickhouseMode) -> ClickhousePolicy { @@ -5620,6 +5627,56 @@ pub(crate) mod test { }; } + // TODO-K: Maybe instead of this I could simulate the SP components are + // already in the expected version + // + // Looks like I need to run the planner a few more times, unless I'm not + // simulating host OS updates properly + // + // From the logs: + // 17:19:03.675Z DEBG update_crucible_pantry: host phase 2 desired contents set to Artifact already + // hash = 0000000000000000000000000000000000000000000000000000000000000000 + // sled_id = fbaebf9a-5127-4909-bb95-0ba583f9ada9 + // slot = b + // version = version 1.0.0-freeform + // 17:19:03.675Z INFO update_crucible_pantry (host_phase_1): MGS-driven update not yet completed (will keep it) + // artifact_hash = 0000000000000000000000000000000000000000000000000000000000000000 + // artifact_version = 1.0.0-freeform + // expected_active_phase_1_hash = 0101010101010101010101010101010101010101010101010101010101010101 + // expected_active_phase_1_slot = A + // expected_active_phase_2_hash = 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a + // expected_boot_disk = A + // expected_inactive_phase_1_hash = 0202020202020202020202020202020202020202020202020202020202020202 + // expected_inactive_phase_2_hash = 0000000000000000000000000000000000000000000000000000000000000000 + // part_number = model0 + // serial_number = serial0 + // sled_agent_address = [fd00:1122:3344:101::1]:12345 + // sp_slot = 0 + // sp_type = Sled + // 17:19:03.675Z INFO update_crucible_pantry: reached maximum number of pending MGS-driven updates + // max = 1 + // + // TODO-K: Probably just merge the two function/macro into 1 + fn sp_component_artifact( + name: &str, + kind: ArtifactKind, + board: Option<&str>, + sign: Option>, + version: &ArtifactVersion, + ) -> TufArtifactMeta { + TufArtifactMeta { + id: ArtifactId { + name: name.to_string(), + version: version.clone(), + kind, + }, + hash: ArtifactHash([0; 32]), + size: 0, // unused here + board: board.map(|s| s.to_string()), + sign, + } + } + // TODO-K: Include MGS artifacts for zone update testing fn create_artifacts_at_version( version: &ArtifactVersion, @@ -5638,6 +5695,125 @@ pub(crate) mod test { fake_zone_artifact!(InternalNtp, version.clone()), fake_zone_artifact!(Nexus, version.clone()), fake_zone_artifact!(Oximeter, version.clone()), + sp_component_artifact( + "host-os-phase-1", + ArtifactKind::HOST_PHASE_1, + None, + None, + version, + ), + sp_component_artifact( + "host-os-phase-2", + ArtifactKind::HOST_PHASE_2, + None, + None, + version, + ), + sp_component_artifact( + "gimlet-d", + KnownArtifactKind::GimletSp.into(), + Some("gimlet-d"), + None, + version, + ), + sp_component_artifact( + "gimlet-e", + KnownArtifactKind::GimletSp.into(), + Some("gimlet-e"), + None, + version, + ), + sp_component_artifact( + "sidecar-b", + KnownArtifactKind::SwitchSp.into(), + Some("sidecar-b"), + None, + version, + ), + sp_component_artifact( + "sidecar-c", + KnownArtifactKind::SwitchSp.into(), + Some("sidecar-c"), + None, + version, + ), + sp_component_artifact( + "psc-b", + KnownArtifactKind::PscSp.into(), + Some("psc-b"), + None, + version, + ), + sp_component_artifact( + "psc-c", + KnownArtifactKind::PscSp.into(), + Some("psc-c"), + None, + version, + ), + sp_component_artifact( + "oxide-rot-1-fake-key", + ArtifactKind::GIMLET_ROT_IMAGE_A, + Some("oxide-rot-1"), + Some(ROT_SIGN_GIMLET.into()), + version, + ), + sp_component_artifact( + "oxide-rot-1-fake-key", + ArtifactKind::GIMLET_ROT_IMAGE_B, + Some("oxide-rot-1"), + Some(ROT_SIGN_GIMLET.into()), + version, + ), + sp_component_artifact( + "oxide-rot-1-fake-key", + ArtifactKind::PSC_ROT_IMAGE_A, + Some("oxide-rot-1"), + Some(ROT_SIGN_PSC.into()), + version, + ), + sp_component_artifact( + "oxide-rot-1-fake-key", + ArtifactKind::PSC_ROT_IMAGE_B, + Some("oxide-rot-1"), + Some(ROT_SIGN_PSC.into()), + version, + ), + sp_component_artifact( + "oxide-rot-1-fake-key", + ArtifactKind::SWITCH_ROT_IMAGE_A, + Some("oxide-rot-1"), + Some(ROT_SIGN_SWITCH.into()), + version, + ), + sp_component_artifact( + "oxide-rot-1-fake-key", + ArtifactKind::SWITCH_ROT_IMAGE_B, + Some("oxide-rot-1"), + Some(ROT_SIGN_SWITCH.into()), + version, + ), + sp_component_artifact( + "bootloader-fake-key", + ArtifactKind::GIMLET_ROT_STAGE0, + Some("oxide-rot-1"), + Some(ROT_SIGN_GIMLET.into()), + version, + ), + sp_component_artifact( + "bootloader-fake-key", + ArtifactKind::PSC_ROT_STAGE0, + Some("oxide-rot-1"), + Some(ROT_SIGN_PSC.into()), + version, + ), + sp_component_artifact( + "bootloader-fake-key", + ArtifactKind::SWITCH_ROT_STAGE0, + Some("oxide-rot-1"), + Some(ROT_SIGN_SWITCH.into()), + version, + ), ] } From 4f15d18a5b0a435200e7838690a663981eedf0b5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 15 Sep 2025 20:08:09 +1200 Subject: [PATCH 24/55] =?UTF-8?q?at=20least=20the=20error=20is=20different?= =?UTF-8?q?=20now=20=F0=9F=98=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nexus/reconfigurator/planning/src/example.rs | 11 ++ .../planning/src/mgs_updates/host_phase_1.rs | 2 + .../planning/src/mgs_updates/mod.rs | 7 +- nexus/reconfigurator/planning/src/planner.rs | 161 +++--------------- nexus/reconfigurator/planning/src/system.rs | 19 ++- nexus/types/src/deployment/blueprint_diff.rs | 2 + 6 files changed, 57 insertions(+), 145 deletions(-) diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index ce793980b2c..97a1aa51198 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -367,6 +367,7 @@ impl ExampleSystemBuilder { /// /// Return the system, and the initial blueprint that matches it. pub fn build(&self) -> (ExampleSystem, Blueprint) { + // println!("SLED SETTINGS: {:#?}", self.sled_settings); let nexus_count = self.get_nexus_zones(); slog::debug!( @@ -406,7 +407,17 @@ impl ExampleSystemBuilder { .policy(settings.policy), ) .unwrap(); + // TODO-K: Remove debugging logs + // let debug_sled = bob.sleds.get(sled_id).unwrap(); + // println!("SLED {} STAGE 0 CABOOSE: {:#?}", sled_id, debug_sled.stage0_caboose); + // println!("SLED {} STAGE 0 NEXT CABOOSE: {:#?}", sled_id, debug_sled.stage0_caboose); + // println!("SLED {} ROT A CABOOSE: {:#?}", sled_id, debug_sled.rot_slot_a_caboose); + // println!("SLED {} ROT B CABOOSE: {:#?}", sled_id, debug_sled.rot_slot_b_caboose); + // println!("SLED {} SP ACTIVE CABOOSE: {:#?}", sled_id, debug_sled.sp_active_caboose); + // println!("SLED {} SP INACTIVE CABOOSE: {:#?}", sled_id, debug_sled.sp_active_caboose); + // println!("SLED {} HOST PHASE 1 HASH FLASH: {:#?}", sled_id, debug_sled.sp_host_phase_1_hash_flash); } + // println!("SYSTEM: {:#?}", system.sleds.values()); let mut input_builder = system .to_planning_input_builder() diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index f4e832542a4..7a87feddf39 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -156,6 +156,8 @@ pub(super) fn update_status( MgsUpdateStatusError::SledAgentErrorDeterminingBootDisk(err.clone()) })?; + // TODO-K: Show boot disk to debug? + // If we find the desired artifact in the active slot _and_ we see that // sled-agent has successfully booted from that same slot, we're done. if active_phase_1_hash == desired_artifact diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index d6e83744ce4..cf00f19517c 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -253,15 +253,16 @@ pub(crate) fn plan_mgs_updates( } let PlannedMgsUpdates { - pending_updates: update, + pending_updates: updates, pending_host_phase_2_changes: mut host_phase_2, skipped_mgs_updates: mut skipped_updates, } = try_make_update(log, board, inventory, current_artifacts); - if !update.is_empty() { + // TODO-K: Is there a bug here somehow? + if !updates.is_empty() { // We can safely unwrap because we just created the update with the // baseboard_id in try_make_update above - let update = update.get(&board).unwrap(); + let update = updates.get(&board).unwrap(); info!(log, "configuring MGS-driven update"; update); pending_updates.insert(update.clone()); } else { diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 2e0968379ef..5cb315453ed 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -2198,13 +2198,6 @@ pub(crate) mod test { use typed_rng::TypedUuidRng; use uuid::Uuid; - const ROT_SIGN_GIMLET: &str = - "1111111111111111111111111111111111111111111111111111111111111111"; - const ROT_SIGN_PSC: &str = - "2222222222222222222222222222222222222222222222222222222222222222"; - const ROT_SIGN_SWITCH: &str = - "3333333333333333333333333333333333333333333333333333333333333333"; - // Generate a ClickhousePolicy ignoring fields we don't care about for /// planner tests fn clickhouse_policy(mode: ClickhouseMode) -> ClickhousePolicy { @@ -5627,57 +5620,32 @@ pub(crate) mod test { }; } - // TODO-K: Maybe instead of this I could simulate the SP components are - // already in the expected version - // - // Looks like I need to run the planner a few more times, unless I'm not - // simulating host OS updates properly - // - // From the logs: - // 17:19:03.675Z DEBG update_crucible_pantry: host phase 2 desired contents set to Artifact already - // hash = 0000000000000000000000000000000000000000000000000000000000000000 - // sled_id = fbaebf9a-5127-4909-bb95-0ba583f9ada9 - // slot = b - // version = version 1.0.0-freeform - // 17:19:03.675Z INFO update_crucible_pantry (host_phase_1): MGS-driven update not yet completed (will keep it) - // artifact_hash = 0000000000000000000000000000000000000000000000000000000000000000 - // artifact_version = 1.0.0-freeform - // expected_active_phase_1_hash = 0101010101010101010101010101010101010101010101010101010101010101 - // expected_active_phase_1_slot = A - // expected_active_phase_2_hash = 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a - // expected_boot_disk = A - // expected_inactive_phase_1_hash = 0202020202020202020202020202020202020202020202020202020202020202 - // expected_inactive_phase_2_hash = 0000000000000000000000000000000000000000000000000000000000000000 - // part_number = model0 - // serial_number = serial0 - // sled_agent_address = [fd00:1122:3344:101::1]:12345 - // sp_slot = 0 - // sp_type = Sled - // 17:19:03.675Z INFO update_crucible_pantry: reached maximum number of pending MGS-driven updates - // max = 1 - // - // TODO-K: Probably just merge the two function/macro into 1 + // TODO-K: Make this nicer fn sp_component_artifact( name: &str, kind: ArtifactKind, board: Option<&str>, sign: Option>, version: &ArtifactVersion, + hash: Option, ) -> TufArtifactMeta { + let hash = match hash { + Some(h) => h, + None => ArtifactHash([0; 32]), + }; TufArtifactMeta { id: ArtifactId { name: name.to_string(), version: version.clone(), kind, }, - hash: ArtifactHash([0; 32]), + hash, size: 0, // unused here board: board.map(|s| s.to_string()), sign, } } - // TODO-K: Include MGS artifacts for zone update testing fn create_artifacts_at_version( version: &ArtifactVersion, ) -> Vec { @@ -5695,12 +5663,15 @@ pub(crate) mod test { fake_zone_artifact!(InternalNtp, version.clone()), fake_zone_artifact!(Nexus, version.clone()), fake_zone_artifact!(Oximeter, version.clone()), + // TODO-K: This makes the updates think the MGS driven updates + // are done, but there should be a better way to represent this sp_component_artifact( "host-os-phase-1", ArtifactKind::HOST_PHASE_1, None, None, version, + Some(ArtifactHash([1; 32])), ), sp_component_artifact( "host-os-phase-2", @@ -5708,111 +5679,31 @@ pub(crate) mod test { None, None, version, + Some(ArtifactHash([0x0a; 32])), ), sp_component_artifact( - "gimlet-d", + "SimGimletSp", KnownArtifactKind::GimletSp.into(), - Some("gimlet-d"), - None, - version, - ), - sp_component_artifact( - "gimlet-e", - KnownArtifactKind::GimletSp.into(), - Some("gimlet-e"), - None, - version, - ), - sp_component_artifact( - "sidecar-b", - KnownArtifactKind::SwitchSp.into(), - Some("sidecar-b"), + Some("SimGimletSp"), None, - version, - ), - sp_component_artifact( - "sidecar-c", - KnownArtifactKind::SwitchSp.into(), - Some("sidecar-c"), + &ArtifactVersion::new("0.0.1").unwrap(), None, - version, - ), - sp_component_artifact( - "psc-b", - KnownArtifactKind::PscSp.into(), - Some("psc-b"), - None, - version, - ), - sp_component_artifact( - "psc-c", - KnownArtifactKind::PscSp.into(), - Some("psc-c"), - None, - version, ), sp_component_artifact( - "oxide-rot-1-fake-key", - ArtifactKind::GIMLET_ROT_IMAGE_A, - Some("oxide-rot-1"), - Some(ROT_SIGN_GIMLET.into()), - version, - ), - sp_component_artifact( - "oxide-rot-1-fake-key", + "SimRot", ArtifactKind::GIMLET_ROT_IMAGE_B, - Some("oxide-rot-1"), - Some(ROT_SIGN_GIMLET.into()), - version, - ), - sp_component_artifact( - "oxide-rot-1-fake-key", - ArtifactKind::PSC_ROT_IMAGE_A, - Some("oxide-rot-1"), - Some(ROT_SIGN_PSC.into()), - version, - ), - sp_component_artifact( - "oxide-rot-1-fake-key", - ArtifactKind::PSC_ROT_IMAGE_B, - Some("oxide-rot-1"), - Some(ROT_SIGN_PSC.into()), - version, - ), - sp_component_artifact( - "oxide-rot-1-fake-key", - ArtifactKind::SWITCH_ROT_IMAGE_A, - Some("oxide-rot-1"), - Some(ROT_SIGN_SWITCH.into()), - version, - ), - sp_component_artifact( - "oxide-rot-1-fake-key", - ArtifactKind::SWITCH_ROT_IMAGE_B, - Some("oxide-rot-1"), - Some(ROT_SIGN_SWITCH.into()), - version, + Some("SimRot"), + Some("sign-gimlet".into()), + &ArtifactVersion::new("0.0.2").unwrap(), + None, ), sp_component_artifact( - "bootloader-fake-key", + "SimRot", ArtifactKind::GIMLET_ROT_STAGE0, - Some("oxide-rot-1"), - Some(ROT_SIGN_GIMLET.into()), - version, - ), - sp_component_artifact( - "bootloader-fake-key", - ArtifactKind::PSC_ROT_STAGE0, - Some("oxide-rot-1"), - Some(ROT_SIGN_PSC.into()), - version, - ), - sp_component_artifact( - "bootloader-fake-key", - ArtifactKind::SWITCH_ROT_STAGE0, - Some("oxide-rot-1"), - Some(ROT_SIGN_SWITCH.into()), - version, + Some("SimRot"), + Some("sign-gimlet".into()), + &ArtifactVersion::new("0.0.1").unwrap(), + None, ), ] } @@ -5833,6 +5724,7 @@ pub(crate) mod test { ) .build(); verify_blueprint(&blueprint1); + //println!("{:#?}", example.collection.cabooses_found); // We should start with no specified TUF repo and nothing to do. assert!(example.input.tuf_repo().description().tuf_repo().is_none()); @@ -5933,6 +5825,8 @@ pub(crate) mod test { input_builder.policy_mut().target_nexus_zone_count + 1; let input = input_builder.build(); + // TODO-K: Why is there no new nexus being created? + // Check that there is a new nexus zone that does *not* use the new // artifact (since not all of its dependencies are updated yet). update_collection_from_blueprint(&mut example, &blueprint1); @@ -5950,6 +5844,7 @@ pub(crate) mod test { { let summary = blueprint2.diff_since_blueprint(&blueprint1); for sled in summary.diff.sleds.modified_values_diff() { + println!("DEBUG: {:#?}", sled.zones); assert!(sled.zones.removed.is_empty()); assert_eq!(sled.zones.added.len(), 1); let added = sled.zones.added.values().next().unwrap(); diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index a993a8de3c6..82cd47e694b 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -105,7 +105,8 @@ pub struct SystemDescription { collector: Option, // Arc to make cloning cheap. Mutating sleds is uncommon but // possible, in which case we'll clone-on-write with Arc::make_mut. - sleds: IndexMap>, + // TODO-K: remove puib + pub sleds: IndexMap>, sled_subnets: SubnetIterator, available_non_scrimlet_slots: BTreeSet, available_scrimlet_slots: BTreeSet, @@ -1188,14 +1189,14 @@ pub struct Sled { policy: SledPolicy, state: SledState, resources: SledResources, - stage0_caboose: Option>, - stage0_next_caboose: Option>, - sp_host_phase_1_active_slot: Option, - sp_host_phase_1_hash_flash: BTreeMap, - sp_active_caboose: Option>, - sp_inactive_caboose: Option>, - rot_slot_a_caboose: Option>, - rot_slot_b_caboose: Option>, + pub stage0_caboose: Option>, + pub stage0_next_caboose: Option>, + pub sp_host_phase_1_active_slot: Option, + pub sp_host_phase_1_hash_flash: BTreeMap, + pub sp_active_caboose: Option>, + pub sp_inactive_caboose: Option>, + pub rot_slot_a_caboose: Option>, + pub rot_slot_b_caboose: Option>, } impl Sled { diff --git a/nexus/types/src/deployment/blueprint_diff.rs b/nexus/types/src/deployment/blueprint_diff.rs index a29cb57317f..27c5763b96e 100644 --- a/nexus/types/src/deployment/blueprint_diff.rs +++ b/nexus/types/src/deployment/blueprint_diff.rs @@ -2133,6 +2133,8 @@ impl fmt::Display for BlueprintDiffDisplay<'_, '_> { writeln!(f, "{}", table)?; } + // TODO-K: Add skipped updates in a follow up PR? + Ok(()) } } From dd8f91191f62d2675631ddd6785713dbd56416be Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 16 Sep 2025 20:07:57 +1200 Subject: [PATCH 25/55] Fix test_update_boundary_ntp and test_update_crucible_pantry --- nexus/reconfigurator/planning/src/planner.rs | 83 ++++++++++++++++---- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 5cb315453ed..0e86799a9ca 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -5822,11 +5822,9 @@ pub(crate) mod test { // Request another Nexus zone. input_builder.policy_mut().target_nexus_zone_count = - input_builder.policy_mut().target_nexus_zone_count + 1; + input_builder.policy_mut().target_nexus_zone_count + 3; let input = input_builder.build(); - // TODO-K: Why is there no new nexus being created? - // Check that there is a new nexus zone that does *not* use the new // artifact (since not all of its dependencies are updated yet). update_collection_from_blueprint(&mut example, &blueprint1); @@ -5843,8 +5841,29 @@ pub(crate) mod test { .expect("can't re-plan for new Nexus zone"); { let summary = blueprint2.diff_since_blueprint(&blueprint1); + // TODO-K: host phase2 is showing stuff in each of the sleds, + // instead of nexus in one sled being the only change + // + // host_phase_2: BlueprintHostPhase2DesiredSlotsDiff { + // slot_a: Leaf { + // before: CurrentContents, + // after: Artifact { + // version: Available { + // version: ArtifactVersion( + // "1.0.0-freeform", + // ), + // }, + // hash: ArtifactHash( + // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + // ), + // }, + // }, + // slot_b: Leaf { + // before: CurrentContents, + // after: CurrentContents, + // }, + // }, for sled in summary.diff.sleds.modified_values_diff() { - println!("DEBUG: {:#?}", sled.zones); assert!(sled.zones.removed.is_empty()); assert_eq!(sled.zones.added.len(), 1); let added = sled.zones.added.values().next().unwrap(); @@ -5940,10 +5959,10 @@ pub(crate) mod test { .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .filter(|(_, z)| is_old_nexus(z)) .count(), - NEXUS_REDUNDANCY + 1, + NEXUS_REDUNDANCY + 3, ); let mut parent = blueprint8; - for i in 9..=16 { + for i in 9..=20 { update_collection_from_blueprint(&mut example, &parent); let blueprint_name = format!("blueprint{i}"); @@ -5982,19 +6001,19 @@ pub(crate) mod test { } // Everything's up-to-date in Kansas City! - let blueprint16 = parent; + let blueprint20 = parent; assert_eq!( - blueprint16 + blueprint20 .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .filter(|(_, z)| is_up_to_date_nexus(z)) .count(), - NEXUS_REDUNDANCY + 1, + NEXUS_REDUNDANCY + 3, ); - update_collection_from_blueprint(&mut example, &blueprint16); + update_collection_from_blueprint(&mut example, &blueprint20); assert_planning_makes_no_changes( &logctx.log, - &blueprint16, + &blueprint20, &input, &example.collection, TEST_NAME, @@ -6597,12 +6616,44 @@ pub(crate) mod test { collection.ntp_timesync = ntp_timesync; }; + // First we update the blueprints three times, as the diff will always + // report there are changes with the host phase 2 even though the artifact + // matches and no update is needed + let mut parent = blueprint; + for i in 2..=4 { + update_collection_from_blueprint(&mut example, &parent); + + let blueprint_name = format!("blueprint{i}"); + let blueprint = Planner::new_based_on( + log.clone(), + &parent, + &example.input, + &blueprint_name, + &example.collection, + PlannerRng::from_seed((TEST_NAME, &blueprint_name)), + ) + .expect("can't create planner") + .plan() + .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); + + { + let summary = blueprint.diff_since_blueprint(&parent); + for sled in summary.diff.sleds.modified_values_diff() { + assert!(sled.zones.added.is_empty()); + assert!(sled.zones.removed.is_empty()); + } + } + + parent = blueprint; + } + let blueprint4 = parent; + // If we have missing info in our inventory, the // planner will not update any boundary NTP zones. example.collection.ntp_timesync = IdOrdMap::new(); assert_planning_makes_no_changes( &log, - &blueprint, + &blueprint4, &example.input, &example.collection, TEST_NAME, @@ -6625,7 +6676,7 @@ pub(crate) mod test { example.collection.ntp_timesync.remove(&boundary_ntp_zone); assert_planning_makes_no_changes( &log, - &blueprint, + &blueprint4, &example.input, &example.collection, TEST_NAME, @@ -6653,7 +6704,7 @@ pub(crate) mod test { .synced = false; assert_planning_makes_no_changes( &log, - &blueprint, + &blueprint4, &example.input, &example.collection, TEST_NAME, @@ -6673,7 +6724,7 @@ pub(crate) mod test { // let new_blueprint = Planner::new_based_on( log.clone(), - &blueprint, + &blueprint4, &example.input, "test_blueprint_expunge_old_boundary_ntp", &example.collection, @@ -6683,7 +6734,7 @@ pub(crate) mod test { .plan() .expect("plan for trivial TUF repo"); { - let summary = new_blueprint.diff_since_blueprint(&blueprint); + let summary = new_blueprint.diff_since_blueprint(&blueprint4); eprintln!( "diff between blueprints (should be expunging boundary NTP using install dataset):\n{}", summary.display() From 23e8f54ce530a2b3177a9306a22ef93b788cc4d7 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 16 Sep 2025 20:21:17 +1200 Subject: [PATCH 26/55] fix test_update_cockroach --- nexus/reconfigurator/planning/src/planner.rs | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 0e86799a9ca..130c68a1c6e 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -6167,6 +6167,38 @@ pub(crate) mod test { result }; + // First we update the blueprints three times, as the diff will always + // report there are changes with the host phase 2 even though the artifact + // matches and no update is needed + let mut parent = blueprint; + for i in 2..=4 { + update_collection_from_blueprint(&mut example, &parent); + + let blueprint_name = format!("blueprint{i}"); + let blueprint = Planner::new_based_on( + log.clone(), + &parent, + &example.input, + &blueprint_name, + &example.collection, + PlannerRng::from_seed((TEST_NAME, &blueprint_name)), + ) + .expect("can't create planner") + .plan() + .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); + + { + let summary = blueprint.diff_since_blueprint(&parent); + for sled in summary.diff.sleds.modified_values_diff() { + assert!(sled.zones.added.is_empty()); + assert!(sled.zones.removed.is_empty()); + } + } + + parent = blueprint; + } + let mut blueprint = parent; + // If we have missing info in our inventory, the // planner will not update any Cockroach zones. example.collection.cockroach_status = BTreeMap::new(); From f47e1cd18c2d3d410755f7d37b2991bb250fab24 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 16 Sep 2025 20:43:33 +1200 Subject: [PATCH 27/55] finally all the tests pass --- nexus/reconfigurator/planning/src/planner.rs | 42 ++++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 130c68a1c6e..ec255ad251a 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -6190,8 +6190,8 @@ pub(crate) mod test { { let summary = blueprint.diff_since_blueprint(&parent); for sled in summary.diff.sleds.modified_values_diff() { - assert!(sled.zones.added.is_empty()); - assert!(sled.zones.removed.is_empty()); + assert!(sled.zones.added.is_empty()); + assert!(sled.zones.removed.is_empty()); } } @@ -6671,8 +6671,8 @@ pub(crate) mod test { { let summary = blueprint.diff_since_blueprint(&parent); for sled in summary.diff.sleds.modified_values_diff() { - assert!(sled.zones.added.is_empty()); - assert!(sled.zones.removed.is_empty()); + assert!(sled.zones.added.is_empty()); + assert!(sled.zones.removed.is_empty()); } } @@ -7084,6 +7084,40 @@ pub(crate) mod test { result }; + // First we update the blueprints three times, as the diff will always + // report there are changes with the host phase 2 even though the artifact + // matches and no update is needed + let mut parent = blueprint; + for i in 2..=4 { + update_collection_from_blueprint(&mut example, &parent); + + let blueprint_name = format!("blueprint{i}"); + // We make sure we don't update any internal DNS zones yet + example.collection.internal_dns_generation_status = IdOrdMap::new(); + let blueprint = Planner::new_based_on( + log.clone(), + &parent, + &example.input, + &blueprint_name, + &example.collection, + PlannerRng::from_seed((TEST_NAME, &blueprint_name)), + ) + .expect("can't create planner") + .plan() + .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); + + { + let summary = blueprint.diff_since_blueprint(&parent); + for sled in summary.diff.sleds.modified_values_diff() { + assert!(sled.zones.added.is_empty()); + assert!(sled.zones.removed.is_empty()); + } + } + + parent = blueprint; + } + let mut blueprint = parent; + // If we have missing info in our inventory, the // planner will not update any Internal DNS zones. example.collection.internal_dns_generation_status = IdOrdMap::new(); From 34a39061cb90eea7cae87d19e92b92a444a228ad Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 16 Sep 2025 20:55:26 +1200 Subject: [PATCH 28/55] fix after merge --- nexus/types/src/deployment/planning_report.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 5c84acf9d8f..66335c0d687 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -9,7 +9,6 @@ use super::BlueprintZoneConfig; use super::BlueprintZoneImageSource; use super::CockroachDbPreserveDowngrade; use super::PendingMgsUpdates; -use super::PlannerChickenSwitches; use super::PlannerConfig; use crate::deployment::MgsUpdateComponent; use crate::inventory::BaseboardId; From 0781a218407c2b232dc28512edf747a3d62e2ce6 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 17 Sep 2025 18:46:22 +1200 Subject: [PATCH 29/55] clean up --- nexus/reconfigurator/planning/src/example.rs | 10 - .../planning/src/mgs_updates/host_phase_1.rs | 2 - .../planning/src/mgs_updates/mod.rs | 1 - nexus/reconfigurator/planning/src/planner.rs | 184 ++++++++---------- nexus/reconfigurator/planning/src/system.rs | 3 +- 5 files changed, 78 insertions(+), 122 deletions(-) diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index 97a1aa51198..ff69a44087e 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -407,17 +407,7 @@ impl ExampleSystemBuilder { .policy(settings.policy), ) .unwrap(); - // TODO-K: Remove debugging logs - // let debug_sled = bob.sleds.get(sled_id).unwrap(); - // println!("SLED {} STAGE 0 CABOOSE: {:#?}", sled_id, debug_sled.stage0_caboose); - // println!("SLED {} STAGE 0 NEXT CABOOSE: {:#?}", sled_id, debug_sled.stage0_caboose); - // println!("SLED {} ROT A CABOOSE: {:#?}", sled_id, debug_sled.rot_slot_a_caboose); - // println!("SLED {} ROT B CABOOSE: {:#?}", sled_id, debug_sled.rot_slot_b_caboose); - // println!("SLED {} SP ACTIVE CABOOSE: {:#?}", sled_id, debug_sled.sp_active_caboose); - // println!("SLED {} SP INACTIVE CABOOSE: {:#?}", sled_id, debug_sled.sp_active_caboose); - // println!("SLED {} HOST PHASE 1 HASH FLASH: {:#?}", sled_id, debug_sled.sp_host_phase_1_hash_flash); } - // println!("SYSTEM: {:#?}", system.sleds.values()); let mut input_builder = system .to_planning_input_builder() diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index 7a87feddf39..f4e832542a4 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -156,8 +156,6 @@ pub(super) fn update_status( MgsUpdateStatusError::SledAgentErrorDeterminingBootDisk(err.clone()) })?; - // TODO-K: Show boot disk to debug? - // If we find the desired artifact in the active slot _and_ we see that // sled-agent has successfully booted from that same slot, we're done. if active_phase_1_hash == desired_artifact diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index cf00f19517c..cd216280059 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -258,7 +258,6 @@ pub(crate) fn plan_mgs_updates( skipped_mgs_updates: mut skipped_updates, } = try_make_update(log, board, inventory, current_artifacts); - // TODO-K: Is there a bug here somehow? if !updates.is_empty() { // We can safely unwrap because we just created the update with the // baseboard_id in try_make_update above diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 4a57c15c3a5..78e4b2af089 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -5621,33 +5621,7 @@ pub(crate) mod test { }; } - // TODO-K: Make this nicer - fn sp_component_artifact( - name: &str, - kind: ArtifactKind, - board: Option<&str>, - sign: Option>, - version: &ArtifactVersion, - hash: Option, - ) -> TufArtifactMeta { - let hash = match hash { - Some(h) => h, - None => ArtifactHash([0; 32]), - }; - TufArtifactMeta { - id: ArtifactId { - name: name.to_string(), - version: version.clone(), - kind, - }, - hash, - size: 0, // unused here - board: board.map(|s| s.to_string()), - sign, - } - } - - fn create_artifacts_at_version( + fn create_zone_artifacts_at_version( version: &ArtifactVersion, ) -> Vec { vec![ @@ -5664,48 +5638,64 @@ pub(crate) mod test { fake_zone_artifact!(InternalNtp, version.clone()), fake_zone_artifact!(Nexus, version.clone()), fake_zone_artifact!(Oximeter, version.clone()), - // TODO-K: This makes the updates think the MGS driven updates - // are done, but there should be a better way to represent this - sp_component_artifact( - "host-os-phase-1", - ArtifactKind::HOST_PHASE_1, - None, - None, - version, - Some(ArtifactHash([1; 32])), - ), - sp_component_artifact( - "host-os-phase-2", - ArtifactKind::HOST_PHASE_2, - None, - None, - version, - Some(ArtifactHash([0x0a; 32])), - ), - sp_component_artifact( - "SimGimletSp", - KnownArtifactKind::GimletSp.into(), - Some("SimGimletSp"), - None, - &ArtifactVersion::new("0.0.1").unwrap(), - None, - ), - sp_component_artifact( - "SimRot", - ArtifactKind::GIMLET_ROT_IMAGE_B, - Some("SimRot"), - Some("sign-gimlet".into()), - &ArtifactVersion::new("0.0.2").unwrap(), - None, - ), - sp_component_artifact( - "SimRot", - ArtifactKind::GIMLET_ROT_STAGE0, - Some("SimRot"), - Some("sign-gimlet".into()), - &ArtifactVersion::new("0.0.1").unwrap(), - None, - ), + // We create artifacts with the versions (or hash) set to those of + // the example system to simulate an environment that does not need + // SP component updates. + TufArtifactMeta { + id: ArtifactId { + name: "host-os-phase-1".to_string(), + version: version.clone(), + kind: ArtifactKind::HOST_PHASE_1, + }, + hash: ArtifactHash([1; 32]), + size: 0, + board: None, + sign: None, + }, + TufArtifactMeta { + id: ArtifactId { + name: "host-os-phase-2".to_string(), + version: version.clone(), + kind: ArtifactKind::HOST_PHASE_2, + }, + hash: ArtifactHash([0x0a; 32]), + size: 0, + board: None, + sign: None, + }, + TufArtifactMeta { + id: ArtifactId { + name: sp_sim::SIM_GIMLET_BOARD.to_string(), + version: ArtifactVersion::new("0.0.1").unwrap(), + kind: KnownArtifactKind::GimletSp.into(), + }, + hash: ArtifactHash([0; 32]), + size: 0, + board: Some(sp_sim::SIM_GIMLET_BOARD.to_string()), + sign: None, + }, + TufArtifactMeta { + id: ArtifactId { + name: sp_sim::SIM_ROT_BOARD.to_string(), + version: ArtifactVersion::new("0.0.2").unwrap(), + kind: ArtifactKind::GIMLET_ROT_IMAGE_B, + }, + hash: ArtifactHash([0; 32]), + size: 0, + board: Some(sp_sim::SIM_ROT_BOARD.to_string()), + sign: Some("sign-gimlet".into()), + }, + TufArtifactMeta { + id: ArtifactId { + name: sp_sim::SIM_ROT_BOARD.to_string(), + version: ArtifactVersion::new("0.0.1").unwrap(), + kind: ArtifactKind::GIMLET_ROT_STAGE0, + }, + hash: ArtifactHash([0; 32]), + size: 0, + board: Some(sp_sim::SIM_ROT_BOARD.to_string()), + sign: Some("sign-gimlet".into()), + }, ] } @@ -5725,7 +5715,6 @@ pub(crate) mod test { ) .build(); verify_blueprint(&blueprint1); - //println!("{:#?}", example.collection.cabooses_found); // We should start with no specified TUF repo and nothing to do. assert!(example.input.tuf_repo().description().tuf_repo().is_none()); @@ -5763,7 +5752,7 @@ pub(crate) mod test { }, hash: fake_hash, }; - let artifacts = create_artifacts_at_version(&version); + let artifacts = create_zone_artifacts_at_version(&version); let target_release_generation = target_release_generation.next(); input_builder.policy_mut().tuf_repo = TufRepoPolicy { target_release_generation, @@ -5821,7 +5810,10 @@ pub(crate) mod test { }; } - // Request another Nexus zone. + // Request 3 Nexus zones. The blueprint will show changes in each sled + // for BlueprintHostPhase2DesiredSlotsDiff even if we are not performing + // an update for the Host OS. We need each sled to look identical in the + // blueprint, so we add a nexus zone to each sled. input_builder.policy_mut().target_nexus_zone_count = input_builder.policy_mut().target_nexus_zone_count + 3; let input = input_builder.build(); @@ -5842,28 +5834,6 @@ pub(crate) mod test { .expect("can't re-plan for new Nexus zone"); { let summary = blueprint2.diff_since_blueprint(&blueprint1); - // TODO-K: host phase2 is showing stuff in each of the sleds, - // instead of nexus in one sled being the only change - // - // host_phase_2: BlueprintHostPhase2DesiredSlotsDiff { - // slot_a: Leaf { - // before: CurrentContents, - // after: Artifact { - // version: Available { - // version: ArtifactVersion( - // "1.0.0-freeform", - // ), - // }, - // hash: ArtifactHash( - // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", - // ), - // }, - // }, - // slot_b: Leaf { - // before: CurrentContents, - // after: CurrentContents, - // }, - // }, for sled in summary.diff.sleds.modified_values_diff() { assert!(sled.zones.removed.is_empty()); assert_eq!(sled.zones.added.len(), 1); @@ -6105,7 +6075,7 @@ pub(crate) mod test { }, hash: fake_hash, }; - let artifacts = create_artifacts_at_version(&version); + let artifacts = create_zone_artifacts_at_version(&version); let target_release_generation = Generation::from_u32(2); input_builder.policy_mut().tuf_repo = TufRepoPolicy { target_release_generation, @@ -6168,9 +6138,9 @@ pub(crate) mod test { result }; - // First we update the blueprints three times, as the diff will always - // report there are changes with the host phase 2 even though the artifact - // matches and no update is needed + // First we update the blueprint once for each sled, as the diff in the + // simulated system will always report there are changes with the host + // phase 2 even when no update is needed let mut parent = blueprint; for i in 2..=4 { update_collection_from_blueprint(&mut example, &parent); @@ -6558,7 +6528,7 @@ pub(crate) mod test { }, hash: fake_hash, }; - let artifacts = create_artifacts_at_version(&version); + let artifacts = create_zone_artifacts_at_version(&version); let target_release_generation = Generation::from_u32(2); input_builder.policy_mut().tuf_repo = TufRepoPolicy { target_release_generation, @@ -6649,9 +6619,9 @@ pub(crate) mod test { collection.ntp_timesync = ntp_timesync; }; - // First we update the blueprints three times, as the diff will always - // report there are changes with the host phase 2 even though the artifact - // matches and no update is needed + // First we update the blueprint once for each sled, as the diff in the + // simulated system will always report there are changes with the host + // phase 2 even when no update is needed let mut parent = blueprint; for i in 2..=4 { update_collection_from_blueprint(&mut example, &parent); @@ -7018,7 +6988,7 @@ pub(crate) mod test { }, hash: fake_hash, }; - let artifacts = create_artifacts_at_version(&version); + let artifacts = create_zone_artifacts_at_version(&version); let target_release_generation = Generation::from_u32(2); input_builder.policy_mut().tuf_repo = TufRepoPolicy { target_release_generation, @@ -7085,9 +7055,9 @@ pub(crate) mod test { result }; - // First we update the blueprints three times, as the diff will always - // report there are changes with the host phase 2 even though the artifact - // matches and no update is needed + // First we update the blueprint once for each sled, as the diff in the + // simulated system will always report there are changes with the host + // phase 2 even when no update is needed let mut parent = blueprint; for i in 2..=4 { update_collection_from_blueprint(&mut example, &parent); @@ -7330,7 +7300,7 @@ pub(crate) mod test { system_version: Version::new(1, 0, 0), file_name: String::from(""), }, - artifacts: create_artifacts_at_version(&version), + artifacts: create_zone_artifacts_at_version(&version), }, ), }; diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index ffda0555007..a6b74145f6b 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -105,8 +105,7 @@ pub struct SystemDescription { collector: Option, // Arc to make cloning cheap. Mutating sleds is uncommon but // possible, in which case we'll clone-on-write with Arc::make_mut. - // TODO-K: remove puib - pub sleds: IndexMap>, + sleds: IndexMap>, sled_subnets: SubnetIterator, available_non_scrimlet_slots: BTreeSet, available_scrimlet_slots: BTreeSet, From 5c04f223804753f02a8f9152539486a6a7c2b6b0 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 17 Sep 2025 19:58:14 +1200 Subject: [PATCH 30/55] add test --- .../planning/src/mgs_updates/mod.rs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index cd216280059..c41b0db6ed1 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -1592,6 +1592,85 @@ mod test { logctx.cleanup_successful(); } + // Confirm our behaviour for skipped updates + #[test] + fn test_skipped_updates() { + let test_name = "planning_mgs_updates_skipped_updates"; + let logctx = LogContext::new( + test_name, + &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, + ); + let log = &logctx.log; + let test_boards = TestBoards::new(test_name); + + // Initial setup: One of every possible SP component will need to be + // updated + let collection = test_boards + .collection_builder() + .stage0_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) + .rot_active_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) + .sp_active_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) + .host_active_exception( + 0, + ARTIFACT_HASH_HOST_PHASE_1_V1, + ARTIFACT_HASH_HOST_PHASE_2_V1, + ) + .build(); + let current_updates = PendingMgsUpdates::new(); + let nmax_updates = 1; + let impossible_update_policy = ImpossibleUpdatePolicy::Reevaluate; + let repo = test_boards.tuf_repo(); + + // Instead of using the baseboards from the collection, we create a new + // fake baseboard that the planner will not recognise + let mut fake_boards = BTreeSet::new(); + let fake_board = Arc::new(BaseboardId { + part_number: "fake_part".to_string(), + serial_number: "fake_serial".to_string(), + }); + fake_boards.insert(fake_board.clone()); + + let PlannedMgsUpdates { + pending_updates: updates, + skipped_mgs_updates, + .. + } = plan_mgs_updates( + log, + &collection, + &fake_boards, + ¤t_updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + + // The planner should gather each of the failed updates, and report no + // pending updates + let mut expected_skipped_updates = SkippedMgsUpdates::new(); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board.clone(), + component: MgsUpdateComponent::RotBootloader, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board.clone(), + component: MgsUpdateComponent::Rot, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board.clone(), + component: MgsUpdateComponent::Sp, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: fake_board, + component: MgsUpdateComponent::HostOs, + reason: FailedMgsUpdateReason::SpNotInInventory, + }); + assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert!(updates.is_empty()); + } + // Confirm our behavior for impossible updates #[test] fn test_impossible_update_policy() { From dc2f2322344479aa83e8d7013b3164366811eef8 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 17 Sep 2025 20:17:16 +1200 Subject: [PATCH 31/55] Clean up --- .../planning/src/mgs_updates/mod.rs | 66 +++---------------- nexus/types/src/deployment/blueprint_diff.rs | 2 +- 2 files changed, 11 insertions(+), 57 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index c41b0db6ed1..f44e6324ad5 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -268,17 +268,6 @@ pub(crate) fn plan_mgs_updates( info!(log, "skipping board for MGS-driven update"; board) } - // TODO-K: Remove debugging logs - //for skipped in &skipped_updates.updates { - // warn!( - // log, - // "update to {} {} has been skipped: {}", - // skipped.baseboard_id, - // skipped.component, - // skipped.reason - // ); - //} - pending_host_phase_2_changes.append(&mut host_phase_2); skipped_mgs_updates.append(&mut skipped_updates); } @@ -1421,55 +1410,20 @@ mod test { // Test that we don't try to update boards that aren't in // `current_boards`, even if they're in inventory and outdated. - // - // TODO-K: Remove fake boards and find a better way to test this - let mut fake_boards = BTreeSet::new(); - let fake_board = Arc::new(BaseboardId { - part_number: "bob".to_string(), - serial_number: "ob".to_string(), - }); - fake_boards.insert(fake_board.clone()); let collection = test_boards .collection_builder() .stage0_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) .build(); - let PlannedMgsUpdates { - pending_updates: updates, - skipped_mgs_updates, - .. - } = plan_mgs_updates( - log, - &collection, - &fake_boards, - //&BTreeSet::new(), - &PendingMgsUpdates::new(), - &TargetReleaseDescription::TufRepo(repo.clone()), - nmax_updates, - impossible_update_policy, - ); - // TODO-K: Find a better way to test functionality - let mut expected_skipped_updates = SkippedMgsUpdates::new(); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board.clone(), - component: MgsUpdateComponent::RotBootloader, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board.clone(), - component: MgsUpdateComponent::Rot, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board.clone(), - component: MgsUpdateComponent::Sp, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board, - component: MgsUpdateComponent::HostOs, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); - assert_eq!(skipped_mgs_updates, expected_skipped_updates); + let PlannedMgsUpdates { pending_updates: updates, .. } = + plan_mgs_updates( + log, + &collection, + &BTreeSet::new(), + &PendingMgsUpdates::new(), + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); assert!(updates.is_empty()); let PlannedMgsUpdates { pending_updates: updates, .. } = plan_mgs_updates( diff --git a/nexus/types/src/deployment/blueprint_diff.rs b/nexus/types/src/deployment/blueprint_diff.rs index de368b177fc..0d35097a1e1 100644 --- a/nexus/types/src/deployment/blueprint_diff.rs +++ b/nexus/types/src/deployment/blueprint_diff.rs @@ -2140,7 +2140,7 @@ impl fmt::Display for BlueprintDiffDisplay<'_, '_> { writeln!(f, "{}", table)?; } - // TODO-K: Add skipped updates in a follow up PR? + // TODO-K: Add skipped updates in a follow up PR Ok(()) } From fcdaa5b0d5fd4b00b5906324078294896cb7a8f5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 17 Sep 2025 20:23:37 +1200 Subject: [PATCH 32/55] Fix after merge --- dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index 2416510cf01..8f31480031d 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -600,8 +600,6 @@ T ENA ID PARENT > blueprint-plan ade5749d-bdf3-4fab-a8ae-00bea01b3a5a INFO skipping noop image source check for all sleds, reason: no target release is currently set INFO system in initial release state no update artifacts available (no update necessary) -INFO some zones not yet up-to-date, sled_id: 89d02b1b-478c-401a-8e28-7a26f74fa41b, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: b3c9c041-d2f0-4767-bdaf-0e52e9d7a013 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: InstallDataset } }] -WARN cannot issue more MGS-driven updates (no current artifacts) generated blueprint 86db3308-f817-4626-8838-4085949a6a41 based on parent blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a empty planning report for blueprint 86db3308-f817-4626-8838-4085949a6a41. From 1c6a46c3f6a41dfe45d2ce181c73400107d710f3 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 17 Sep 2025 20:32:22 +1200 Subject: [PATCH 33/55] moar cleanup --- nexus/reconfigurator/planning/src/system.rs | 16 ++++++++-------- nexus/types/src/deployment/planning_report.rs | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 1fe529208e5..8f6dcd8173d 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -1210,14 +1210,14 @@ pub struct Sled { policy: SledPolicy, state: SledState, resources: SledResources, - pub stage0_caboose: Option>, - pub stage0_next_caboose: Option>, - pub sp_host_phase_1_active_slot: Option, - pub sp_host_phase_1_hash_flash: BTreeMap, - pub sp_active_caboose: Option>, - pub sp_inactive_caboose: Option>, - pub rot_slot_a_caboose: Option>, - pub rot_slot_b_caboose: Option>, + stage0_caboose: Option>, + stage0_next_caboose: Option>, + sp_host_phase_1_active_slot: Option, + sp_host_phase_1_hash_flash: BTreeMap, + sp_active_caboose: Option>, + sp_inactive_caboose: Option>, + rot_slot_a_caboose: Option>, + rot_slot_b_caboose: Option>, } impl Sled { diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 66335c0d687..521c2c5d44d 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -31,7 +31,6 @@ use serde::Serialize; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; -//use std::fmt::Display; use std::fmt::Write; use std::sync::Arc; use thiserror::Error; From eecfe7cd44095dc54d4a50e5707aa2d4b12cb50c Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 17 Sep 2025 20:40:25 +1200 Subject: [PATCH 34/55] remove debugging println --- nexus/reconfigurator/planning/src/example.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index 6c7f3d409f1..919ab594ad1 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -366,7 +366,6 @@ impl ExampleSystemBuilder { /// /// Return the system, and the initial blueprint that matches it. pub fn build(&self) -> (ExampleSystem, Blueprint) { - // println!("SLED SETTINGS: {:#?}", self.sled_settings); let nexus_count = self.get_nexus_zones(); slog::debug!( From 8c3d0643a6c3f9a24f321fb9e916f1063e2cd48e Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 22 Sep 2025 21:19:38 +1200 Subject: [PATCH 35/55] extract target release testing into it's own test --- .../tests/output/cmds-target-release-stdout | 14 +-- nexus/reconfigurator/planning/src/planner.rs | 86 ++++++++++++++++--- 2 files changed, 80 insertions(+), 20 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index de7acfead0f..45638c1b427 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -480,7 +480,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba based on parent blueprint df06bb57-ad42-4431-9206-abff322896c7 planning report for blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba: * 1 pending MGS update: - * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) + * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -599,7 +599,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba planning report for blueprint 9034c710-3e57-45f3-99e5-4316145e87ac: * 1 pending MGS update: - * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) + * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -658,7 +658,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 based on parent blueprint 9034c710-3e57-45f3-99e5-4316145e87ac planning report for blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976: * 1 pending MGS update: - * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) + * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -716,7 +716,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 planning report for blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736: * 1 pending MGS update: - * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) + * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -1185,7 +1185,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 based on parent blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300 planning report for blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839: * 1 pending MGS update: - * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) + * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -1319,7 +1319,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c based on parent blueprint 61a93ea3-c872-48e0-aace-e86b0c52b839 planning report for blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c: * 1 pending MGS update: - * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) + * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -1795,7 +1795,7 @@ INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 based on parent blueprint 778e3f3a-58b1-4a5e-acff-d23c5d7124c2 planning report for blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6: * 1 pending MGS update: - * model2:serial2: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("f3dd0c7a1bd4500ea0d8bcf67581f576d47752b2f1998a4cb0f0c3155c483008"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) + * model2:serial2: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 023db5458cc..ee605750987 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -6009,8 +6009,6 @@ pub(crate) mod test { &logctx.log, rng.next_system_rng(), ) - .with_target_release_0_0_1() - .expect("set target release to 0.0.1") .build(); verify_blueprint(&blueprint); @@ -6054,18 +6052,12 @@ pub(crate) mod test { TEST_NAME, ); - // All zones should be sourced from the initial 0.0.1 target release by - // default. - eprintln!("{}", blueprint.display()); assert!( blueprint .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .all(|(_, z)| matches!( &z.image_source, - BlueprintZoneImageSource::Artifact { version, hash: _ } - if version == &BlueprintArtifactVersion::Available { - version: ArtifactVersion::new_const("0.0.1") - } + BlueprintZoneImageSource::InstallDataset )) ); @@ -6107,6 +6099,7 @@ pub(crate) mod test { // Manually update all zones except Cockroach // + // We just specified a new TUF repo, everything is going to shift from // the install dataset to this new repo. for mut zone in blueprint .sleds @@ -6128,10 +6121,7 @@ pub(crate) mod test { zone.zone_type.is_cockroach() && matches!( &zone.image_source, - BlueprintZoneImageSource::Artifact { version, hash: _ } - if version == &BlueprintArtifactVersion::Available { - version: ArtifactVersion::new_const("0.0.1") - } + BlueprintZoneImageSource::InstallDataset ) }; let is_up_to_date_cockroach = |zone: &BlueprintZoneConfig| -> bool { @@ -7383,4 +7373,74 @@ pub(crate) mod test { panic!("did not converge after {MAX_PLANNING_ITERATIONS} iterations"); } + + #[test] + fn test_all_zones_sourced_from_initial_target_release_by_default() { + static TEST_NAME: &str = "all_zones_sourced_from_initial_target_release_by_default"; + let logctx = test_setup_log(TEST_NAME); + let log = logctx.log.clone(); + + // Use our example system. + let mut rng = SimRngState::from_seed(TEST_NAME); + let (mut example, mut blueprint) = ExampleSystemBuilder::new_with_rng( + &logctx.log, + rng.next_system_rng(), + ) + .with_target_release_0_0_1() + .expect("set target release to 0.0.1") + .build(); + verify_blueprint(&blueprint); + + // Update the example system and blueprint, as a part of test set-up. + let mut input_builder = example.input.clone().into_builder(); + input_builder.policy_mut().target_cockroachdb_zone_count = + COCKROACHDB_REDUNDANCY; + example.input = input_builder.build(); + + let blueprint_name = "blueprint_with_cockroach"; + let new_blueprint = Planner::new_based_on( + log.clone(), + &blueprint, + &example.input, + &blueprint_name, + &example.collection, + PlannerRng::from_seed((TEST_NAME, &blueprint_name)), + ) + .expect("can't create planner") + .plan() + .unwrap_or_else(|_| panic!("can't plan to include Cockroach nodes")); + + { + let summary = new_blueprint.diff_since_blueprint(&blueprint); + assert_eq!(summary.total_zones_added(), COCKROACHDB_REDUNDANCY); + assert_eq!(summary.total_zones_removed(), 0); + assert_eq!(summary.total_zones_modified(), 0); + } + blueprint = new_blueprint; + update_collection_from_blueprint(&mut example, &blueprint); + + // We should have started with no specified TUF repo and nothing to do. + assert_planning_makes_no_changes( + &logctx.log, + &blueprint, + &example.input, + &example.collection, + TEST_NAME, + ); + + // All zones should be sourced from the initial 0.0.1 target release by + // default. + eprintln!("{}", blueprint.display()); + assert!( + blueprint + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .all(|(_, z)| matches!( + &z.image_source, + BlueprintZoneImageSource::Artifact { version, hash: _ } + if version == &BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("0.0.1") + } + )) + ); + } } From 05d2043e982232c7256de4d14742f0e1c353d6c3 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 22 Sep 2025 21:39:47 +1200 Subject: [PATCH 36/55] wasn't a typo --- nexus/reconfigurator/planning/src/planner.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index ee605750987..f48c6b03920 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -1896,7 +1896,7 @@ impl<'a> Planner<'a> { source_repo.zone_image_source(zone_kind) } - /// Return `true` if a zone of the given kind is ready to be updated; + /// Return `true` iff a zone of the given kind is ready to be updated; /// i.e., its dependencies have been updated. fn is_zone_ready_for_update( &self, @@ -7376,7 +7376,8 @@ pub(crate) mod test { #[test] fn test_all_zones_sourced_from_initial_target_release_by_default() { - static TEST_NAME: &str = "all_zones_sourced_from_initial_target_release_by_default"; + static TEST_NAME: &str = + "all_zones_sourced_from_initial_target_release_by_default"; let logctx = test_setup_log(TEST_NAME); let log = logctx.log.clone(); From 957ac0fc755e48643d1a6d70058a998bae2ead56 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 11:33:12 +1200 Subject: [PATCH 37/55] test clean up logfiles --- nexus/reconfigurator/planning/src/planner.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index f48c6b03920..880de69d98a 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -7443,5 +7443,7 @@ pub(crate) mod test { } )) ); + + logctx.cleanup_successful(); } } From e3a47fd24a46f1ff2724995711ff23feedd2fb85 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 13:51:51 +1200 Subject: [PATCH 38/55] remove unncessesary logging --- .../planning/src/mgs_updates/host_phase_1.rs | 90 ++----------------- .../planning/src/mgs_updates/rot.rs | 67 ++++---------- .../src/mgs_updates/rot_bootloader.rs | 54 +++-------- .../planning/src/mgs_updates/sp.rs | 48 +++------- nexus/reconfigurator/planning/src/planner.rs | 4 +- nexus/types/src/deployment/planning_report.rs | 17 ++-- 6 files changed, 62 insertions(+), 218 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index 875fb0aa68e..c715dc0bc28 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -276,12 +276,6 @@ pub(super) fn try_make_update( FailedMgsUpdateReason, > { let Some(sp_info) = inventory.sps.get(baseboard_id) else { - warn!( - log, - "cannot configure host OS update for board \ - (missing SP info from inventory)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::SpNotInInventory); }; @@ -298,36 +292,15 @@ pub(super) fn try_make_update( let Some(sled_agent) = inventory.sled_agents.iter().find(|sled_agent| { sled_agent.baseboard_id.as_ref() == Some(baseboard_id) }) else { - warn!( - log, - "cannot configure host OS update for board \ - (missing sled-agent info from inventory)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::SledAgentInfoNotInInventory); }; let Some(last_reconciliation) = sled_agent.last_reconciliation.as_ref() else { - warn!( - log, - "cannot configure host OS update for board \ - (missing last reconciliation details from inventory)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::LastReconciliationNotInInventory); }; let boot_disk = match &last_reconciliation.boot_partitions.boot_disk { Ok(boot_disk) => *boot_disk, Err(err) => { - // This error is a `String`; we can't use `InlineErrorChain`. - let err: &str = &err; - warn!( - log, - "cannot configure host OS update for board \ - (sled-agent reported an error determining boot disk)"; - baseboard_id, - "err" => err, - ); return Err(FailedMgsUpdateReason::UnableToDetermineBootDisk( err.to_string(), )); @@ -337,16 +310,6 @@ pub(super) fn try_make_update( match &last_reconciliation.boot_partitions.slot_details(boot_disk) { Ok(details) => details.artifact_hash, Err(err) => { - // This error is a `String`; we can't use `InlineErrorChain`. - let err: &str = &err; - warn!( - log, - "cannot configure host OS update for board \ - (sled-agent reported an error boot disk phase 2 image)"; - baseboard_id, - "boot_disk" => ?boot_disk, - "err" => err, - ); return Err( FailedMgsUpdateReason::UnableToRetrieveBootDiskPhase2Image( err.to_string(), @@ -358,12 +321,6 @@ pub(super) fn try_make_update( let Some(active_phase_1_slot) = inventory.host_phase_1_active_slot_for(baseboard_id).map(|s| s.slot) else { - warn!( - log, - "cannot configure host OS update for board \ - (inventory missing current active host phase 1 slot)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::ActiveHostPhase1SlotNotInInventory); }; @@ -379,16 +336,10 @@ pub(super) fn try_make_update( // 1 slot and the boot disk, they'll induce a support case to recover, given // this current implementation. As far as we know they shouldn't happen. if active_phase_1_slot != boot_disk { - warn!( - log, - "cannot configure host OS update for board (active phase 1 slot \ - doesn't match boot disk; is the sled already being updated?)"; - baseboard_id, - "active_phase_1_slot" => ?active_phase_1_slot, - "boot_disk" => ?boot_disk, - ); return Err( - FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch, + FailedMgsUpdateReason::ActiveHostPhase1SlotBootDiskMismatch( + active_phase_1_slot, + ), ); } @@ -396,14 +347,9 @@ pub(super) fn try_make_update( .host_phase_1_flash_hash_for(active_phase_1_slot, baseboard_id) .map(|h| h.hash) else { - warn!( - log, - "cannot configure host OS update for board \ - (missing active phase 1 hash from inventory)"; - baseboard_id, - "slot" => ?active_phase_1_slot, - ); - return Err(FailedMgsUpdateReason::ActiveHostPhase1HashNotInInventory); + return Err(FailedMgsUpdateReason::ActiveHostPhase1HashNotInInventory( + active_phase_1_slot, + )); }; let Some(inactive_phase_1_hash) = inventory @@ -421,7 +367,9 @@ pub(super) fn try_make_update( "slot" => ?active_phase_1_slot.toggled(), ); return Err( - FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory, + FailedMgsUpdateReason::InactiveHostPhase1HashNotInInventory( + active_phase_1_slot.toggled(), + ), ); }; @@ -442,21 +390,9 @@ pub(super) fn try_make_update( ([p1], [p2]) => (p1, p2), // "TUF is broken" cases: missing one or the other. ([], _) => { - warn!( - log, - "cannot configure host OS update for board \ - (no phase 1 artifact)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } (_, []) => { - warn!( - log, - "cannot configure host OS update for board \ - (no phase 2 artifact)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } // "TUF is broken" cases: have multiple of one or the other. This @@ -464,14 +400,6 @@ pub(super) fn try_make_update( // host OS images. We can't proceed, because we don't know how to // pair up which phase 1 matches which phase 2. (_, _) => { - warn!( - log, - "cannot configure host OS update for board \ - (multiple OS images in TUF repo)"; - baseboard_id, - "num-phase-1-images" => phase_1_artifacts.len(), - "num-phase-2-images" => phase_2_artifacts.len(), - ); return Err(FailedMgsUpdateReason::TooManyMatchingArtifacts); } }; diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index c8eff5ae372..068d2989232 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -108,22 +108,10 @@ pub fn try_make_update_rot( current_artifacts: &TufRepoDescription, ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { - warn!( - log, - "cannot configure RoT update for board \ - (missing SP info from inventory)"; - baseboard_id - ); return Err(FailedMgsUpdateReason::SpNotInInventory); }; let Some(rot_state) = inventory.rots.get(baseboard_id) else { - warn!( - log, - "cannot configure RoT update for board \ - (missing RoT state from inventory)"; - baseboard_id - ); return Err(FailedMgsUpdateReason::RotStateNotInInventory); }; @@ -133,37 +121,23 @@ pub fn try_make_update_rot( let Some(active_caboose) = inventory.caboose_for(active_caboose_which, baseboard_id) else { - warn!( - log, - "cannot configure RoT update for board \ - (missing active slot {active_slot} caboose from inventory)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::CabooseNotInInventory( active_caboose_which, )); }; - let Ok(expected_active_version) = active_caboose.caboose.version.parse() - else { - warn!( - log, - "cannot configure RoT update for board \ - (cannot parse current active version as an ArtifactVersion)"; - baseboard_id, - "found_version" => &active_caboose.caboose.version, - ); - return Err(FailedMgsUpdateReason::FailedVersionParse); + let expected_active_version = match active_caboose.caboose.version.parse() { + Ok(v) => v, + Err(e) => { + return Err(FailedMgsUpdateReason::FailedVersionParse { + caboose: active_caboose_which, + err: format!("{}", e), + }); + } }; let board = &active_caboose.caboose.board; let Some(rkth) = &active_caboose.caboose.sign else { - warn!( - log, - "cannot configure RoT update for board \ - (missing sign in caboose from inventory)"; - baseboard_id - ); return Err(FailedMgsUpdateReason::CabooseMissingSign( active_caboose_which, )); @@ -225,11 +199,6 @@ pub fn try_make_update_rot( }) .collect(); if matching_artifacts.is_empty() { - warn!( - log, - "cannot configure RoT update for board (no matching artifact)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } @@ -256,24 +225,20 @@ pub fn try_make_update_rot( }; // Begin configuring an update. + let inactive_caboose_which = + CabooseWhich::from_rot_slot(active_slot.toggled()); let expected_inactive_version = match inventory - .caboose_for( - CabooseWhich::from_rot_slot(active_slot.toggled()), - baseboard_id, - ) + .caboose_for(inactive_caboose_which, baseboard_id) .map(|c| c.caboose.version.parse::()) .transpose() { Ok(None) => ExpectedVersion::NoValidVersion, Ok(Some(v)) => ExpectedVersion::Version(v), - Err(_) => { - warn!( - log, - "cannot configure RoT update for board \ - (found inactive slot contents but version was not valid)"; - baseboard_id - ); - return Err(FailedMgsUpdateReason::FailedVersionParse); + Err(e) => { + return Err(FailedMgsUpdateReason::FailedVersionParse { + caboose: inactive_caboose_which, + err: format!("{}", e), + }); } }; diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index 12b41a26496..129f547da01 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -63,49 +63,29 @@ pub fn try_make_update_rot_bootloader( current_artifacts: &TufRepoDescription, ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { - warn!( - log, - "cannot configure RoT bootloader update for board \ - (missing SP info from inventory)"; - baseboard_id - ); return Err(FailedMgsUpdateReason::SpNotInInventory); }; let Some(stage0_caboose) = inventory.caboose_for(CabooseWhich::Stage0, baseboard_id) else { - warn!( - log, - "cannot configure RoT bootloader update for board \ - (missing stage0 caboose from inventory)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::CabooseNotInInventory( CabooseWhich::Stage0, )); }; - let Ok(expected_stage0_version) = stage0_caboose.caboose.version.parse() - else { - warn!( - log, - "cannot configure RoT bootloader update for board \ - (cannot parse current stage0 version as an ArtifactVersion)"; - baseboard_id, - "found_version" => &stage0_caboose.caboose.version, - ); - return Err(FailedMgsUpdateReason::FailedVersionParse); + let expected_stage0_version = match stage0_caboose.caboose.version.parse() { + Ok(v) => v, + Err(e) => { + return Err(FailedMgsUpdateReason::FailedVersionParse { + caboose: CabooseWhich::Stage0, + err: format!("{}", e), + }); + } }; let board = &stage0_caboose.caboose.board; let Some(rkth) = &stage0_caboose.caboose.sign else { - warn!( - log, - "cannot configure RoT bootloader update for board \ - (missing sign in stage0 caboose from inventory)"; - baseboard_id - ); return Err(FailedMgsUpdateReason::CabooseMissingSign( CabooseWhich::Stage0, )); @@ -160,11 +140,6 @@ pub fn try_make_update_rot_bootloader( }) .collect(); if matching_artifacts.is_empty() { - warn!( - log, - "cannot configure RoT bootloader update for board (no matching artifact)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } @@ -196,14 +171,11 @@ pub fn try_make_update_rot_bootloader( { Ok(None) => ExpectedVersion::NoValidVersion, Ok(Some(v)) => ExpectedVersion::Version(v), - Err(_) => { - warn!( - log, - "cannot configure RoT bootloader update for board \ - (found stage0 next contents but version was not valid)"; - baseboard_id - ); - return Err(FailedMgsUpdateReason::FailedVersionParse); + Err(e) => { + return Err(FailedMgsUpdateReason::FailedVersionParse { + caboose: CabooseWhich::Stage0Next, + err: format!("{}", e), + }); } }; diff --git a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs index 1f59818b884..5ee4de548f2 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs @@ -62,39 +62,25 @@ pub fn try_make_update_sp( current_artifacts: &TufRepoDescription, ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { - warn!( - log, - "cannot configure SP update for board \ - (missing SP info from inventory)"; - baseboard_id - ); return Err(FailedMgsUpdateReason::SpNotInInventory); }; let Some(active_caboose) = inventory.caboose_for(CabooseWhich::SpSlot0, baseboard_id) else { - warn!( - log, - "cannot configure SP update for board \ - (missing active caboose from inventory)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::CabooseNotInInventory( CabooseWhich::SpSlot0, )); }; - let Ok(expected_active_version) = active_caboose.caboose.version.parse() - else { - warn!( - log, - "cannot configure SP update for board \ - (cannot parse current active version as an ArtifactVersion)"; - baseboard_id, - "found_version" => &active_caboose.caboose.version, - ); - return Err(FailedMgsUpdateReason::FailedVersionParse); + let expected_active_version = match active_caboose.caboose.version.parse() { + Ok(v) => v, + Err(e) => { + return Err(FailedMgsUpdateReason::FailedVersionParse { + caboose: CabooseWhich::SpSlot0, + err: format!("{}", e), + }); + } }; let board = &active_caboose.caboose.board; @@ -135,11 +121,6 @@ pub fn try_make_update_sp( }) .collect(); if matching_artifacts.is_empty() { - warn!( - log, - "cannot configure SP update for board (no matching artifact)"; - baseboard_id, - ); return Err(FailedMgsUpdateReason::NoMatchingArtifactFound); } @@ -167,14 +148,11 @@ pub fn try_make_update_sp( { Ok(None) => ExpectedVersion::NoValidVersion, Ok(Some(v)) => ExpectedVersion::Version(v), - Err(_) => { - warn!( - log, - "cannot configure SP update for board \ - (found inactive slot contents but version was not valid)"; - baseboard_id - ); - return Err(FailedMgsUpdateReason::FailedVersionParse); + Err(e) => { + return Err(FailedMgsUpdateReason::FailedVersionParse { + caboose: CabooseWhich::SpSlot1, + err: format!("{}", e), + }); } }; diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 880de69d98a..e0cde5d6aff 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -7375,9 +7375,9 @@ pub(crate) mod test { } #[test] - fn test_all_zones_sourced_from_initial_target_release_by_default() { + fn test_zones_sourced_from_initial_target_release_by_default() { static TEST_NAME: &str = - "all_zones_sourced_from_initial_target_release_by_default"; + "zones_sourced_from_initial_target_release_by_default"; let logctx = test_setup_log(TEST_NAME); let log = logctx.log.clone(); diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 521c2c5d44d..120121f26c2 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -18,6 +18,7 @@ use daft::Diffable; use iddqd::IdOrdItem; use iddqd::id_upcast; use indent_write::fmt::IndentWriter; +use omicron_common::disk::M2Slot; use omicron_common::policy::COCKROACHDB_REDUNDANCY; use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::MupdateOverrideUuid; @@ -494,11 +495,11 @@ impl PlanningMupdateOverrideStepReport { #[serde(tag = "type", content = "value")] pub enum FailedMgsUpdateReason { /// The active host phase 1 slot does not match the boot disk - #[error("active phase 1 slot does not match boot disk")] - ActiveHostPhase1SlotBootDiskMismatch, + #[error("active phase 1 slot {0:?} does not match boot disk")] + ActiveHostPhase1SlotBootDiskMismatch(M2Slot), /// The active host phase 1 hash was not found in inventory - #[error("active host phase 1 hash is not in inventory")] - ActiveHostPhase1HashNotInInventory, + #[error("active host phase 1 hash for slot {0:?} is not in inventory")] + ActiveHostPhase1HashNotInInventory(M2Slot), /// The active host phase 1 slot was not found in inventory #[error("active host phase 1 slot is not in inventory")] ActiveHostPhase1SlotNotInInventory, @@ -509,11 +510,11 @@ pub enum FailedMgsUpdateReason { #[error("caboose for {0:?} is not in inventory")] CabooseNotInInventory(CabooseWhich), /// The version in the caboose or artifact was not able to be parsed - #[error("version could not be parsed")] - FailedVersionParse, + #[error("version from caboose {caboose:?} could not be parsed: {err}")] + FailedVersionParse { caboose: CabooseWhich, err: String }, /// The inactive host phase 1 hash was not found in inventory - #[error("inactive host phase 1 hash is not in inventory")] - InactiveHostPhase1HashNotInInventory, + #[error("inactive host phase 1 hash for slot {0:?} is not in inventory")] + InactiveHostPhase1HashNotInInventory(M2Slot), /// Last reconciliation details were not found in inventory #[error("sled agent last reconciliation is not in inventory")] LastReconciliationNotInInventory, From 9f5ca2fc292e48e363694c8c868cd840ed3cdd0b Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 15:36:56 +1200 Subject: [PATCH 39/55] Address comments --- .../output/cmds-mupdate-update-flow-stdout | 6 +- .../tests/output/cmds-target-release-stdout | 210 +++++++++--------- .../planning/src/mgs_updates/host_phase_1.rs | 12 +- .../planning/src/mgs_updates/mod.rs | 61 +++-- .../planning/src/mgs_updates/rot.rs | 1 + .../src/mgs_updates/rot_bootloader.rs | 3 +- .../planning/src/mgs_updates/sp.rs | 1 + openapi/nexus-internal.json | 36 ++- 8 files changed, 177 insertions(+), 153 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index fddccd501ef..ae823722d9c 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -1992,9 +1992,9 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 6, num_already_artifact: 6, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 2.0.0, artifact_hash: 8f89bf8bc5f3271650ad72a26fc0d116c910161ca143731473a2b20fb82653cc, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update INFO some zones not yet up-to-date, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: 0c71b3b2-6ceb-4e8f-b020-b08675e83038 (service), zone_kind: Nexus, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("0e32b4a3e5d3668bb1d6a16fb06b74dc60b973fa479dcee0aae3adbb52bf1388") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 427ec88f-f467-42fa-9bbb-66a91a36103c (service), zone_kind: InternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ffbf1373f7ee08dddd74c53ed2a94e7c4c572a982d3a9bc94000c6956b700c6a") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 5199c033-4cf9-4ab6-8ae7-566bd7606363 (service), zone_kind: Crucible, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("6f17cf65fb5a5bec5542dd07c03cd0acc01e59130f02c532c8d848ecae810047") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 6444f8a5-6465-4f0b-a549-1993c113569c (service), zone_kind: InternalNtp, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: 803bfb63-c246-41db-b0da-d3b87ddfc63d (service), zone_kind: ExternalDns, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("ccca13ed19b8731f9adaf0d6203b02ea3b9ede4fa426b9fac0a07ce95440046d") }, inv_image_source: InstallDataset } }, ZoneCurrentlyUpdating { zone_id: ba4994a8-23f9-4b1a-a84f-a08d74591389 (service), zone_kind: CruciblePantry, reason: ImageSourceMismatch { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("21f0ada306859c23917361f2e0b9235806c32607ec689c7e8cf16bb898bc5a02") }, inv_image_source: InstallDataset } }] generated blueprint 12d602a6-5ab4-487a-b94e-eb30cdf30300 based on parent blueprint 8f2d1f39-7c88-4701-aa43-56bf281b28c1 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index 45638c1b427..df22e3ef385 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -770,7 +770,7 @@ INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-495 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint 626487fa-7139-45ec-8416-902271fc730b based on parent blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 @@ -1384,8 +1384,8 @@ INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-495 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, sled_agent_address: [fd00:1122:3344:102::1]:12345, expected_inactive_phase_2_hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, expected_inactive_phase_1_hash: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 1, sp_type: Sled, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 INFO configuring MGS-driven update, artifact_version: 1.0.0, artifact_hash: 5b0f601b1fbb8674db9c751a02f8b14f8e6d4e8470f4f7b686fecb2c49ec11f9, expected_stage0_next_version: NoValidVersion, expected_stage0_version: 0.0.1, component: rot_bootloader, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 9f89efdf-a23e-4137-b7cc-79f4a91cbe1f based on parent blueprint 27e755bc-dc10-4647-853c-f89bb3a15a2c @@ -1919,9 +1919,9 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO MGS-driven update completed (will remove it and re-evaluate board), artifact_version: 1.0.0, artifact_hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, sled_agent_address: [fd00:1122:3344:103::1]:12345, expected_inactive_phase_2_hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 2, sp_type: Sled, serial_number: serial2, part_number: model2 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 INFO ran out of boards for MGS-driven update generated blueprint e54a0836-53e1-4948-a3af-0b77165289b5 based on parent blueprint 386a7ec3-7c2e-43cf-8f00-999e91e1d5e6 planning report for blueprint e54a0836-53e1-4948-a3af-0b77165289b5: @@ -2047,9 +2047,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 459a45a5-616e-421f-873b-2fb08c36205c based on parent blueprint e54a0836-53e1-4948-a3af-0b77165289b5 planning report for blueprint 459a45a5-616e-421f-873b-2fb08c36205c: @@ -2180,9 +2180,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update INFO some zones not yet up-to-date, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: f83ade6d-9ab1-4679-813b-b9457e039c0b (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") } } }] generated blueprint b2295597-5788-482e-acf9-1731ec63fbd2 based on parent blueprint 459a45a5-616e-421f-873b-2fb08c36205c @@ -2309,9 +2309,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 6fad8fd4-e825-433f-b76d-495484e068ce based on parent blueprint b2295597-5788-482e-acf9-1731ec63fbd2 planning report for blueprint 6fad8fd4-e825-433f-b76d-495484e068ce: @@ -2453,9 +2453,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 24b6e243-100c-428d-8ea6-35b504226f55 based on parent blueprint 6fad8fd4-e825-433f-b76d-495484e068ce planning report for blueprint 24b6e243-100c-428d-8ea6-35b504226f55: @@ -2598,9 +2598,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce based on parent blueprint 24b6e243-100c-428d-8ea6-35b504226f55 planning report for blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce: @@ -2721,9 +2721,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 based on parent blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce planning report for blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312: @@ -2868,9 +2868,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d based on parent blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 planning report for blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d: @@ -3016,9 +3016,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint e2125c83-b255-45c9-bc9b-802cff09a812 based on parent blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d planning report for blueprint e2125c83-b255-45c9-bc9b-802cff09a812: @@ -3151,9 +3151,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 based on parent blueprint e2125c83-b255-45c9-bc9b-802cff09a812 planning report for blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3: @@ -3287,9 +3287,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c based on parent blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 planning report for blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c: @@ -3415,9 +3415,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 based on parent blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c planning report for blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1: @@ -3546,9 +3546,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 based on parent blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 planning report for blueprint e2deb7c0-2262-49fe-855f-4250c22afb36: @@ -3662,9 +3662,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf based on parent blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 planning report for blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf: @@ -3798,9 +3798,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d based on parent blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf planning report for blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d: @@ -3935,9 +3935,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 60b55d33-5fec-4277-9864-935197eaead7 based on parent blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d planning report for blueprint 60b55d33-5fec-4277-9864-935197eaead7: @@ -4053,9 +4053,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 based on parent blueprint 60b55d33-5fec-4277-9864-935197eaead7 planning report for blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544: @@ -4180,9 +4180,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update INFO some zones not yet up-to-date, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: cc6fdaf4-0195-4cef-950d-7bacd7059787 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") } } }] generated blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 based on parent blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 @@ -4309,9 +4309,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a based on parent blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 planning report for blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a: @@ -4450,9 +4450,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 based on parent blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a planning report for blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110: @@ -4592,9 +4592,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 based on parent blueprint e7a01ffc-6b0e-408b-917b-b1efe18b3110 planning report for blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482: @@ -4724,9 +4724,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec based on parent blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 planning report for blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec: @@ -4857,9 +4857,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint a2c6496d-98fc-444d-aa36-99508aa72367 based on parent blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec planning report for blueprint a2c6496d-98fc-444d-aa36-99508aa72367: @@ -4985,9 +4985,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 based on parent blueprint a2c6496d-98fc-444d-aa36-99508aa72367 planning report for blueprint 6ed56354-5941-40d1-a06c-b0e940701d52: @@ -5101,9 +5101,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 1, num_eligible: 0, num_ineligible: 7 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 based on parent blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 planning report for blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2: @@ -5225,9 +5225,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 1, num_eligible: 0, num_ineligible: 6 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 based on parent blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 planning report for blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152: @@ -5350,9 +5350,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 2, num_eligible: 0, num_ineligible: 6 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 based on parent blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 planning report for blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88: @@ -5467,9 +5467,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 3, num_eligible: 0, num_ineligible: 5 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 based on parent blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 planning report for blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4: @@ -5602,9 +5602,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 3, num_eligible: 0, num_ineligible: 4 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 59630e63-c953-4807-9e84-9e750a79f68e based on parent blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 planning report for blueprint 59630e63-c953-4807-9e84-9e750a79f68e: @@ -5738,9 +5738,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 4, num_eligible: 0, num_ineligible: 4 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 based on parent blueprint 59630e63-c953-4807-9e84-9e750a79f68e planning report for blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194: @@ -5867,9 +5867,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 4, num_eligible: 0, num_ineligible: 3 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update INFO some zones not yet up-to-date, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, zones_currently_updating: [ZoneCurrentlyUpdating { zone_id: d5fd048a-8786-42d3-938e-820eae95d7f4 (service), zone_kind: InternalNtp, reason: MissingInInventory { bp_image_source: Artifact { version: Available { version: ArtifactVersion("1.0.0") }, hash: ArtifactHash("67593d686ed04a1709f93972b71f4ebc148a9362120f65d239943e814a9a7439") } } }] generated blueprint 90650737-8142-47a6-9a48-a10efc487e57 based on parent blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 @@ -5998,9 +5998,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 5, num_eligible: 0, num_ineligible: 3 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa based on parent blueprint 90650737-8142-47a6-9a48-a10efc487e57 planning report for blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa: @@ -6120,9 +6120,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 6, num_eligible: 0, num_ineligible: 2 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece based on parent blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa planning report for blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece: @@ -6260,9 +6260,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 7, num_already_artifact: 6, num_eligible: 0, num_ineligible: 1 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 810ea95a-4730-43dd-867e-1984aeb9d873 based on parent blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece planning report for blueprint 810ea95a-4730-43dd-867e-1984aeb9d873: diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index c715dc0bc28..ff6db6f2101 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -72,6 +72,10 @@ impl PendingHostPhase2Changes { ); } + pub(super) fn is_empty(&self) -> bool { + self.by_sled.is_empty() + } + pub(crate) fn into_iter( self, ) -> impl Iterator @@ -100,11 +104,6 @@ impl PendingHostPhase2Changes { self.by_sled.remove(sled_id) } - #[cfg(test)] - pub(super) fn is_empty(&self) -> bool { - self.by_sled.is_empty() - } - #[cfg(test)] fn len(&self) -> usize { self.by_sled.len() @@ -266,6 +265,9 @@ pub(super) fn update_status( } } +/// Determine if the given baseboard needs a Host OS update and, if so, +/// returns it. An error means an update is still necessary but cannot be +/// completed. pub(super) fn try_make_update( log: &slog::Logger, baseboard_id: &Arc, diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 1869d451112..9d5047c9d39 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -82,7 +82,7 @@ impl PlannedMgsUpdates { } } - fn set_pending_update( + fn add_pending_update( &mut self, pending_update: PendingMgsUpdate, ) -> &mut Self { @@ -252,20 +252,23 @@ pub(crate) fn plan_mgs_updates( }; } + // TODO-K: Make it clearer whether it's a single update or system-wide + // report let PlannedMgsUpdates { pending_updates: updates, pending_host_phase_2_changes: mut host_phase_2, skipped_mgs_updates: mut skipped_updates, } = try_make_update(log, board, inventory, current_artifacts); - if !updates.is_empty() { - // We can safely unwrap because we just created the update with the - // baseboard_id in try_make_update above - let update = updates.get(&board).unwrap(); + if let Some(update) = updates.into_iter().next() { info!(log, "configuring MGS-driven update"; update); pending_updates.insert(update.clone()); } else { - info!(log, "skipping board for MGS-driven update"; board) + if skipped_mgs_updates.is_empty() && host_phase_2.is_empty() { + info!(log, "skipping board for MGS-driven update (no update necessary)"; board); + } else { + info!(log, "skipping board for MGS-driven update (found issues)"; board); + } } pending_host_phase_2_changes.append(&mut host_phase_2); @@ -569,53 +572,41 @@ fn try_make_update( // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". type UpdateResult = Result, FailedMgsUpdateReason>; - type UpdateFn<'a> = Box UpdateResult + 'a>; - let attempts: [(MgsUpdateComponent, UpdateFn); 3] = [ + let attempts: [(MgsUpdateComponent, UpdateResult); 3] = [ ( MgsUpdateComponent::RotBootloader, - Box::new(|| { - try_make_update_rot_bootloader( - log, - baseboard_id, - inventory, - current_artifacts, - ) - }), + try_make_update_rot_bootloader( + log, + baseboard_id, + inventory, + current_artifacts, + ), ), ( MgsUpdateComponent::Rot, - Box::new(|| { - try_make_update_rot( - log, - baseboard_id, - inventory, - current_artifacts, - ) - }), + try_make_update_rot( + log, + baseboard_id, + inventory, + current_artifacts, + ), ), ( MgsUpdateComponent::Sp, - Box::new(|| { - try_make_update_sp( - log, - baseboard_id, - inventory, - current_artifacts, - ) - }), + try_make_update_sp(log, baseboard_id, inventory, current_artifacts), ), ]; if let Some(update_actions) = attempts.into_iter().find_map(|(component, attempt)| { - match attempt() { + match attempt { // There is a pending update, record it along with any previous // failed updates Ok(Some(update)) => { // We have a non-host update; there are no pending host // phase 2 changes necessary. let pending_actions = PlannedMgsUpdates::new() - .set_pending_update(update) + .add_pending_update(update) .set_skipped_updates(skipped_updates.clone()) .build(); Some(pending_actions) @@ -646,7 +637,7 @@ fn try_make_update( ) { Ok(Some((update, pending_host_os_phase2_changes))) => { PlannedMgsUpdates::new() - .set_pending_update(update) + .add_pending_update(update) .set_pending_host_os_phase2_changes( pending_host_os_phase2_changes, ) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index 068d2989232..6e2ac30a5ba 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -101,6 +101,7 @@ pub fn mgs_update_status_rot( } /// Determine if the given baseboard needs an RoT update and, if so, returns it. +/// An error means an update is still necessary but cannot be completed. pub fn try_make_update_rot( log: &slog::Logger, baseboard_id: &Arc, diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index 129f547da01..41367484e9e 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -55,7 +55,8 @@ pub fn mgs_update_status_rot_bootloader( } /// Determine if the given baseboard needs an RoT bootloader update and, if so, -/// returns it. +/// returns it. An error means an update is still necessary but cannot be +/// completed. pub fn try_make_update_rot_bootloader( log: &slog::Logger, baseboard_id: &Arc, diff --git a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs index 5ee4de548f2..c9ecd6c13fb 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs @@ -55,6 +55,7 @@ pub fn mgs_update_status_sp( } /// Determine if the given baseboard needs an SP update and, if so, returns it. +/// An error means an update is still necessary but cannot be completed. pub fn try_make_update_sp( log: &slog::Logger, baseboard_id: &Arc, diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 8f2b7772707..0b68dfe4ce3 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -4846,10 +4846,14 @@ "enum": [ "active_host_phase1_slot_boot_disk_mismatch" ] + }, + "value": { + "$ref": "#/components/schemas/M2Slot" } }, "required": [ - "type" + "type", + "value" ] }, { @@ -4861,10 +4865,14 @@ "enum": [ "active_host_phase1_hash_not_in_inventory" ] + }, + "value": { + "$ref": "#/components/schemas/M2Slot" } }, "required": [ - "type" + "type", + "value" ] }, { @@ -4929,10 +4937,26 @@ "enum": [ "failed_version_parse" ] + }, + "value": { + "type": "object", + "properties": { + "caboose": { + "$ref": "#/components/schemas/CabooseWhich" + }, + "err": { + "type": "string" + } + }, + "required": [ + "caboose", + "err" + ] } }, "required": [ - "type" + "type", + "value" ] }, { @@ -4944,10 +4968,14 @@ "enum": [ "inactive_host_phase1_hash_not_in_inventory" ] + }, + "value": { + "$ref": "#/components/schemas/M2Slot" } }, "required": [ - "type" + "type", + "value" ] }, { From e863155b90c876bf34fa5952149df777c7a8b815 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 15:42:36 +1200 Subject: [PATCH 40/55] Remove unnecessary test --- nexus/reconfigurator/planning/src/planner.rs | 73 -------------------- 1 file changed, 73 deletions(-) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index e0cde5d6aff..b49af95584c 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -7373,77 +7373,4 @@ pub(crate) mod test { panic!("did not converge after {MAX_PLANNING_ITERATIONS} iterations"); } - - #[test] - fn test_zones_sourced_from_initial_target_release_by_default() { - static TEST_NAME: &str = - "zones_sourced_from_initial_target_release_by_default"; - let logctx = test_setup_log(TEST_NAME); - let log = logctx.log.clone(); - - // Use our example system. - let mut rng = SimRngState::from_seed(TEST_NAME); - let (mut example, mut blueprint) = ExampleSystemBuilder::new_with_rng( - &logctx.log, - rng.next_system_rng(), - ) - .with_target_release_0_0_1() - .expect("set target release to 0.0.1") - .build(); - verify_blueprint(&blueprint); - - // Update the example system and blueprint, as a part of test set-up. - let mut input_builder = example.input.clone().into_builder(); - input_builder.policy_mut().target_cockroachdb_zone_count = - COCKROACHDB_REDUNDANCY; - example.input = input_builder.build(); - - let blueprint_name = "blueprint_with_cockroach"; - let new_blueprint = Planner::new_based_on( - log.clone(), - &blueprint, - &example.input, - &blueprint_name, - &example.collection, - PlannerRng::from_seed((TEST_NAME, &blueprint_name)), - ) - .expect("can't create planner") - .plan() - .unwrap_or_else(|_| panic!("can't plan to include Cockroach nodes")); - - { - let summary = new_blueprint.diff_since_blueprint(&blueprint); - assert_eq!(summary.total_zones_added(), COCKROACHDB_REDUNDANCY); - assert_eq!(summary.total_zones_removed(), 0); - assert_eq!(summary.total_zones_modified(), 0); - } - blueprint = new_blueprint; - update_collection_from_blueprint(&mut example, &blueprint); - - // We should have started with no specified TUF repo and nothing to do. - assert_planning_makes_no_changes( - &logctx.log, - &blueprint, - &example.input, - &example.collection, - TEST_NAME, - ); - - // All zones should be sourced from the initial 0.0.1 target release by - // default. - eprintln!("{}", blueprint.display()); - assert!( - blueprint - .all_omicron_zones(BlueprintZoneDisposition::is_in_service) - .all(|(_, z)| matches!( - &z.image_source, - BlueprintZoneImageSource::Artifact { version, hash: _ } - if version == &BlueprintArtifactVersion::Available { - version: ArtifactVersion::new_const("0.0.1") - } - )) - ); - - logctx.cleanup_successful(); - } } From cc8e07add4bdb097317641301388f3764a8384f8 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 18:08:12 +1200 Subject: [PATCH 41/55] Bail on failed update and improve testing --- .../planning/src/mgs_updates/mod.rs | 159 ++++++++++++++++-- 1 file changed, 141 insertions(+), 18 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 9d5047c9d39..461bf285f27 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -621,7 +621,10 @@ fn try_make_update( component, reason: e, }); - None + let pending_actions = PlannedMgsUpdates::new() + .set_skipped_updates(skipped_updates.clone()) + .build(); + Some(pending_actions) } } }) @@ -682,6 +685,7 @@ mod test { use dropshot::ConfigLogging; use dropshot::ConfigLoggingLevel; use gateway_client::types::SpType; + use iddqd::IdOrdMap; use nexus_types::deployment::ExpectedVersion; use nexus_types::deployment::MgsUpdateComponent; use nexus_types::deployment::PendingMgsUpdateDetails; @@ -692,6 +696,7 @@ mod test { use nexus_types::deployment::planning_report::SkippedMgsUpdate; use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; + use nexus_types::inventory::CabooseWhich; use omicron_test_utils::dev::LogContext; use std::collections::BTreeSet; use std::sync::Arc; @@ -749,29 +754,147 @@ mod test { impossible_update_policy, ); - // The planner should gather each of the failed updates, and report no - // pending updates + // The planner should only gather the first failed update (RoT + // bootloader), and report no pending updates. There will only be a + // single entry as there is only a single fake board. let mut expected_skipped_updates = SkippedMgsUpdates::new(); expected_skipped_updates.push(SkippedMgsUpdate { baseboard_id: fake_board.clone(), component: MgsUpdateComponent::RotBootloader, reason: FailedMgsUpdateReason::SpNotInInventory, }); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board.clone(), - component: MgsUpdateComponent::Rot, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board.clone(), - component: MgsUpdateComponent::Sp, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); - expected_skipped_updates.push(SkippedMgsUpdate { - baseboard_id: fake_board, - component: MgsUpdateComponent::HostOs, - reason: FailedMgsUpdateReason::SpNotInInventory, - }); + assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert!(updates.is_empty()); + + // Now we build a the collection so it only reports updates necessary + // for the RoT, SP and Host OS. + let mut collection = test_boards + .collection_builder() + .rot_active_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) + .sp_active_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) + .host_active_exception( + 0, + ARTIFACT_HASH_HOST_PHASE_1_V1, + ARTIFACT_HASH_HOST_PHASE_2_V1, + ) + .build(); + + // Let's remove all RoT information to force a failed update + for baseboard_id in &collection.baseboards { + collection.rots.remove(baseboard_id); + } + + let PlannedMgsUpdates { + pending_updates: updates, + skipped_mgs_updates, + .. + } = plan_mgs_updates( + log, + &collection, + &collection.baseboards, + ¤t_updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + + // The planner should only gather the first RoT failed update of + // each of the boards, and report no pending updates + let mut expected_skipped_updates = SkippedMgsUpdates::new(); + for baseboard_id in &collection.baseboards { + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::Rot, + reason: FailedMgsUpdateReason::RotStateNotInInventory, + }); + } + assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert!(updates.is_empty()); + + // Like before we build a collection that only reports updates necessary + // for the SP and Host OS. + let mut collection = test_boards + .collection_builder() + .sp_active_version_exception(SpType::Sled, 0, ARTIFACT_VERSION_1) + .host_active_exception( + 0, + ARTIFACT_HASH_HOST_PHASE_1_V1, + ARTIFACT_HASH_HOST_PHASE_2_V1, + ) + .build(); + + // Let's remove SP slot 0 caboose information to force a failed update + collection.cabooses_found.remove(&CabooseWhich::SpSlot0); + + let PlannedMgsUpdates { + pending_updates: updates, + skipped_mgs_updates, + .. + } = plan_mgs_updates( + log, + &collection, + &collection.baseboards, + ¤t_updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + + // The planner should only gather the first SP failed update of + // each of the boards, and report no pending updates + let mut expected_skipped_updates = SkippedMgsUpdates::new(); + for baseboard_id in &collection.baseboards { + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::Sp, + reason: FailedMgsUpdateReason::CabooseNotInInventory( + CabooseWhich::SpSlot0, + ), + }); + } + assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert!(updates.is_empty()); + + // Now we create one more collection where only the Host OS needs an + // update + let mut collection = test_boards + .collection_builder() + .host_active_exception( + 0, + ARTIFACT_HASH_HOST_PHASE_1_V1, + ARTIFACT_HASH_HOST_PHASE_2_V1, + ) + .build(); + + // Remove sled agent info to force a failed update + collection.sled_agents = IdOrdMap::new(); + + let PlannedMgsUpdates { + pending_updates: updates, + skipped_mgs_updates, + .. + } = plan_mgs_updates( + log, + &collection, + &collection.baseboards, + ¤t_updates, + &TargetReleaseDescription::TufRepo(repo.clone()), + nmax_updates, + impossible_update_policy, + ); + + // The planner should only gather the first Host OS failed update of + // each of the sled boards, and report no pending updates + let mut expected_skipped_updates = SkippedMgsUpdates::new(); + for baseboard_id in &collection.baseboards { + if baseboard_id.part_number == "dummy_sled" { + expected_skipped_updates.push(SkippedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component: MgsUpdateComponent::HostOs, + reason: FailedMgsUpdateReason::SledAgentInfoNotInInventory, + }); + } + } assert_eq!(skipped_mgs_updates, expected_skipped_updates); assert!(updates.is_empty()); } From 2444040494dc8d0022a8621bb140a21ccec79196 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 18:34:36 +1200 Subject: [PATCH 42/55] address style comments --- .../planning/src/mgs_updates/test_helpers.rs | 2 +- nexus/reconfigurator/planning/src/planner.rs | 69 +++++++++++++++---- nexus/types/src/deployment.rs | 6 +- 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs index 1601e66b456..6d2e679d1e6 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/test_helpers.rs @@ -615,7 +615,7 @@ impl ExpectedUpdates { ) { let sp_type = update.sp_type; let sp_slot = update.slot_id; - let component = update.details.clone().into(); + let component: MgsUpdateComponent = (&update.details).into(); println!("found update: {} slot {}", sp_type, sp_slot); let ExpectedUpdate { expected_serial, expected_artifact, .. } = self .updates diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index b49af95584c..530d9eedcc8 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -5815,9 +5815,21 @@ pub(crate) mod test { } // Request 3 Nexus zones. The blueprint will show changes in each sled - // for BlueprintHostPhase2DesiredSlotsDiff even if we are not performing - // an update for the Host OS. We need each sled to look identical in the - // blueprint, so we add a nexus zone to each sled. + // for BlueprintHostPhase2DesiredSlotsDiff in a simulated system even if + // we are not performing an update for the Host OS because it's going + // from `CurrentContents` to: + // `Artifact { + // version: Available { + // version: ArtifactVersion( + // "1.0.0-freeform", + // ), + // }, + // hash: ArtifactHash( + // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + // ) + // } ` + // We need each sled to look identical in the blueprint, so we add a + // nexus zone to each sled. input_builder.policy_mut().target_nexus_zone_count = input_builder.policy_mut().target_nexus_zone_count + 3; let input = input_builder.build(); @@ -6141,9 +6153,20 @@ pub(crate) mod test { result }; - // First we update the blueprint once for each sled, as the diff in the - // simulated system will always report there are changes with the host - // phase 2 even when no update is needed + // First we update the blueprint once for each sled. The blueprint will + // show changes in each sled for BlueprintHostPhase2DesiredSlotsDiff in + // a simulated system even if we are not performing an update for the + // Host OS because it's going from `CurrentContents` to: + // `Artifact { + // version: Available { + // version: ArtifactVersion( + // "1.0.0-freeform", + // ), + // }, + // hash: ArtifactHash( + // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + // ) + // } ` let mut parent = blueprint; for i in 2..=4 { update_collection_from_blueprint(&mut example, &parent); @@ -6624,9 +6647,20 @@ pub(crate) mod test { collection.ntp_timesync = ntp_timesync; }; - // First we update the blueprint once for each sled, as the diff in the - // simulated system will always report there are changes with the host - // phase 2 even when no update is needed + // First we update the blueprint once for each sled. The blueprint will + // show changes in each sled for BlueprintHostPhase2DesiredSlotsDiff in + // a simulated system even if we are not performing an update for the + // Host OS because it's going from `CurrentContents` to: + // `Artifact { + // version: Available { + // version: ArtifactVersion( + // "1.0.0-freeform", + // ), + // }, + // hash: ArtifactHash( + // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + // ) + // } ` let mut parent = blueprint; for i in 2..=4 { update_collection_from_blueprint(&mut example, &parent); @@ -7060,9 +7094,20 @@ pub(crate) mod test { result }; - // First we update the blueprint once for each sled, as the diff in the - // simulated system will always report there are changes with the host - // phase 2 even when no update is needed + // First we update the blueprint once for each sled. The blueprint will + // show changes in each sled for BlueprintHostPhase2DesiredSlotsDiff in + // a simulated system even if we are not performing an update for the + // Host OS because it's going from `CurrentContents` to: + // `Artifact { + // version: Available { + // version: ArtifactVersion( + // "1.0.0-freeform", + // ), + // }, + // hash: ArtifactHash( + // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", + // ) + // } ` let mut parent = blueprint; for i in 2..=4 { update_collection_from_blueprint(&mut example, &parent); diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index a6834526434..3b5232458b1 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -1565,9 +1565,9 @@ pub enum PendingMgsUpdateDetails { HostPhase1(PendingMgsUpdateHostPhase1Details), } -impl Into for PendingMgsUpdateDetails { - fn into(self) -> MgsUpdateComponent { - match self { +impl From<&PendingMgsUpdateDetails> for MgsUpdateComponent { + fn from(details: &PendingMgsUpdateDetails) -> Self { + match &details { PendingMgsUpdateDetails::Rot { .. } => MgsUpdateComponent::Rot, PendingMgsUpdateDetails::RotBootloader { .. } => { MgsUpdateComponent::RotBootloader From 19e8e268474c268cd4add01544bed26c06423717 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 18:51:13 +1200 Subject: [PATCH 43/55] Get rid of SkippedMgsUpdates --- .../planning/src/mgs_updates/mod.rs | 23 ++++++------ nexus/reconfigurator/planning/src/planner.rs | 3 +- nexus/types/src/deployment/planning_report.rs | 35 +++---------------- 3 files changed, 16 insertions(+), 45 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 461bf285f27..eb605593db6 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -30,7 +30,6 @@ use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::deployment::planning_report::SkippedMgsUpdate; -use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -70,7 +69,7 @@ pub(crate) struct PlannedMgsUpdates { pub(crate) pending_host_phase_2_changes: PendingHostPhase2Changes, // Updates to components that failed for some reason and have been skipped. - pub(crate) skipped_mgs_updates: SkippedMgsUpdates, + pub(crate) skipped_mgs_updates: Vec, } impl PlannedMgsUpdates { @@ -78,7 +77,7 @@ impl PlannedMgsUpdates { Self { pending_updates: PendingMgsUpdates::new(), pending_host_phase_2_changes: PendingHostPhase2Changes::empty(), - skipped_mgs_updates: SkippedMgsUpdates::new(), + skipped_mgs_updates: Vec::new(), } } @@ -100,7 +99,7 @@ impl PlannedMgsUpdates { fn set_skipped_updates( &mut self, - skipped_updates: SkippedMgsUpdates, + skipped_updates: Vec, ) -> &mut Self { self.skipped_mgs_updates = skipped_updates; self @@ -144,7 +143,7 @@ pub(crate) fn plan_mgs_updates( let mut pending_updates = PendingMgsUpdates::new(); let mut pending_host_phase_2_changes = PendingHostPhase2Changes::empty(); let mut boards_preferred = BTreeSet::new(); - let mut skipped_mgs_updates = SkippedMgsUpdates::new(); + let mut skipped_mgs_updates = Vec::new(); // Determine the status of all currently pending updates by comparing what // they were trying to do (and their preconditions) against the current @@ -566,7 +565,7 @@ fn try_make_update( inventory: &Collection, current_artifacts: &TufRepoDescription, ) -> PlannedMgsUpdates { - let mut skipped_updates = SkippedMgsUpdates::new(); + let mut skipped_updates = Vec::new(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section @@ -694,7 +693,6 @@ mod test { use nexus_types::deployment::TargetReleaseDescription; use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::deployment::planning_report::SkippedMgsUpdate; - use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use omicron_test_utils::dev::LogContext; @@ -757,12 +755,11 @@ mod test { // The planner should only gather the first failed update (RoT // bootloader), and report no pending updates. There will only be a // single entry as there is only a single fake board. - let mut expected_skipped_updates = SkippedMgsUpdates::new(); - expected_skipped_updates.push(SkippedMgsUpdate { + let expected_skipped_updates = vec![SkippedMgsUpdate { baseboard_id: fake_board.clone(), component: MgsUpdateComponent::RotBootloader, reason: FailedMgsUpdateReason::SpNotInInventory, - }); + }]; assert_eq!(skipped_mgs_updates, expected_skipped_updates); assert!(updates.is_empty()); @@ -800,7 +797,7 @@ mod test { // The planner should only gather the first RoT failed update of // each of the boards, and report no pending updates - let mut expected_skipped_updates = SkippedMgsUpdates::new(); + let mut expected_skipped_updates = Vec::new(); for baseboard_id in &collection.baseboards { expected_skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), @@ -842,7 +839,7 @@ mod test { // The planner should only gather the first SP failed update of // each of the boards, and report no pending updates - let mut expected_skipped_updates = SkippedMgsUpdates::new(); + let mut expected_skipped_updates = Vec::new(); for baseboard_id in &collection.baseboards { expected_skipped_updates.push(SkippedMgsUpdate { baseboard_id: baseboard_id.clone(), @@ -885,7 +882,7 @@ mod test { // The planner should only gather the first Host OS failed update of // each of the sled boards, and report no pending updates - let mut expected_skipped_updates = SkippedMgsUpdates::new(); + let mut expected_skipped_updates = Vec::new(); for baseboard_id in &collection.baseboards { if baseboard_id.part_number == "dummy_sled" { expected_skipped_updates.push(SkippedMgsUpdate { diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 530d9eedcc8..cb5849c85e9 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -41,7 +41,6 @@ use nexus_types::deployment::SledDetails; use nexus_types::deployment::SledFilter; use nexus_types::deployment::TufRepoContentsError; use nexus_types::deployment::ZpoolFilter; -use nexus_types::deployment::planning_report::SkippedMgsUpdates; use nexus_types::deployment::{ CockroachdbUnsafeToShutdown, PlanningAddStepReport, PlanningCockroachdbSettingsStepReport, PlanningDecommissionStepReport, @@ -197,7 +196,7 @@ impl<'a> Planner<'a> { } else { PlanningMgsUpdatesStepReport::new( PendingMgsUpdates::new(), - SkippedMgsUpdates::new(), + Vec::new(), ) }; diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index 120121f26c2..ed6b14cb2df 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -83,7 +83,7 @@ impl PlanningReport { noop_image_source: PlanningNoopImageSourceStepReport::new(), mgs_updates: PlanningMgsUpdatesStepReport::new( PendingMgsUpdates::new(), - SkippedMgsUpdates::new(), + Vec::new(), ), add: PlanningAddStepReport::new(), zone_updates: PlanningZoneUpdatesStepReport::new(), @@ -563,43 +563,18 @@ impl IdOrdItem for SkippedMgsUpdate { id_upcast!(); } -#[derive( - Clone, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, Diffable, -)] -pub struct SkippedMgsUpdates { - pub updates: Vec, -} - -impl SkippedMgsUpdates { - pub fn new() -> Self { - Self { updates: Vec::new() } - } - - pub fn is_empty(&self) -> bool { - self.updates.is_empty() - } - - pub fn push(&mut self, update: SkippedMgsUpdate) { - self.updates.push(update) - } - - pub fn append(&mut self, other: &mut Self) { - self.updates.append(&mut other.updates); - } -} - #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] pub struct PlanningMgsUpdatesStepReport { pub pending_mgs_updates: PendingMgsUpdates, - pub skipped_mgs_updates: SkippedMgsUpdates, + pub skipped_mgs_updates: Vec, } impl PlanningMgsUpdatesStepReport { pub fn new( pending_mgs_updates: PendingMgsUpdates, - skipped_mgs_updates: SkippedMgsUpdates, + skipped_mgs_updates: Vec, ) -> Self { Self { pending_mgs_updates, skipped_mgs_updates } } @@ -626,10 +601,10 @@ impl fmt::Display for PlanningMgsUpdatesStepReport { } } if !skipped_mgs_updates.is_empty() { - let n = skipped_mgs_updates.updates.len(); + let n = skipped_mgs_updates.len(); let s = plural(n); writeln!(f, "* {n} skipped MGS update{s}:")?; - for update in &skipped_mgs_updates.updates { + for update in skipped_mgs_updates { writeln!( f, " * {} {}: {}", From a78d9b19268d1f52c4f704911b884d00b204c06e Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 23 Sep 2025 19:14:54 +1200 Subject: [PATCH 44/55] use blocked instead of skipped --- .../planning/src/mgs_updates/mod.rs | 87 ++++++++-------- nexus/reconfigurator/planning/src/planner.rs | 10 +- nexus/types/src/deployment/planning_report.rs | 30 +++--- openapi/nexus-internal.json | 99 +++++++++---------- 4 files changed, 108 insertions(+), 118 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index eb605593db6..0e14d7bdd41 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -28,8 +28,8 @@ use nexus_types::deployment::PendingMgsUpdateRotDetails; use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; +use nexus_types::deployment::planning_report::BlockedMgsUpdate; use nexus_types::deployment::planning_report::FailedMgsUpdateReason; -use nexus_types::deployment::planning_report::SkippedMgsUpdate; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -68,8 +68,8 @@ pub(crate) struct PlannedMgsUpdates { /// result in a change to the respective sled's `BlueprintSledConfig`. pub(crate) pending_host_phase_2_changes: PendingHostPhase2Changes, - // Updates to components that failed for some reason and have been skipped. - pub(crate) skipped_mgs_updates: Vec, + // Updates to components that failed for some reason and have been blocked. + pub(crate) blocked_mgs_updates: Vec, } impl PlannedMgsUpdates { @@ -77,7 +77,7 @@ impl PlannedMgsUpdates { Self { pending_updates: PendingMgsUpdates::new(), pending_host_phase_2_changes: PendingHostPhase2Changes::empty(), - skipped_mgs_updates: Vec::new(), + blocked_mgs_updates: Vec::new(), } } @@ -97,11 +97,11 @@ impl PlannedMgsUpdates { self } - fn set_skipped_updates( + fn set_blocked_updates( &mut self, - skipped_updates: Vec, + blocked_updates: Vec, ) -> &mut Self { - self.skipped_mgs_updates = skipped_updates; + self.blocked_mgs_updates = blocked_updates; self } @@ -111,7 +111,7 @@ impl PlannedMgsUpdates { pending_host_phase_2_changes: self .pending_host_phase_2_changes .clone(), - skipped_mgs_updates: self.skipped_mgs_updates.clone(), + blocked_mgs_updates: self.blocked_mgs_updates.clone(), } } } @@ -143,7 +143,7 @@ pub(crate) fn plan_mgs_updates( let mut pending_updates = PendingMgsUpdates::new(); let mut pending_host_phase_2_changes = PendingHostPhase2Changes::empty(); let mut boards_preferred = BTreeSet::new(); - let mut skipped_mgs_updates = Vec::new(); + let mut blocked_mgs_updates = Vec::new(); // Determine the status of all currently pending updates by comparing what // they were trying to do (and their preconditions) against the current @@ -221,7 +221,7 @@ pub(crate) fn plan_mgs_updates( return PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, - skipped_mgs_updates, + blocked_mgs_updates, }; } TargetReleaseDescription::TufRepo(description) => description, @@ -247,23 +247,24 @@ pub(crate) fn plan_mgs_updates( return PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, - skipped_mgs_updates, + blocked_mgs_updates, }; } - // TODO-K: Make it clearer whether it's a single update or system-wide - // report + // `try_make_update` will always return at most a single update at a + // time. This means that this instance of `PlannedMgsUpdates` describes + // a single device update let PlannedMgsUpdates { pending_updates: updates, pending_host_phase_2_changes: mut host_phase_2, - skipped_mgs_updates: mut skipped_updates, + blocked_mgs_updates: mut blocked_updates, } = try_make_update(log, board, inventory, current_artifacts); if let Some(update) = updates.into_iter().next() { info!(log, "configuring MGS-driven update"; update); pending_updates.insert(update.clone()); } else { - if skipped_mgs_updates.is_empty() && host_phase_2.is_empty() { + if blocked_mgs_updates.is_empty() && host_phase_2.is_empty() { info!(log, "skipping board for MGS-driven update (no update necessary)"; board); } else { info!(log, "skipping board for MGS-driven update (found issues)"; board); @@ -271,14 +272,14 @@ pub(crate) fn plan_mgs_updates( } pending_host_phase_2_changes.append(&mut host_phase_2); - skipped_mgs_updates.append(&mut skipped_updates); + blocked_mgs_updates.append(&mut blocked_updates); } info!(log, "ran out of boards for MGS-driven update"); PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, - skipped_mgs_updates, + blocked_mgs_updates, } } @@ -565,7 +566,7 @@ fn try_make_update( inventory: &Collection, current_artifacts: &TufRepoDescription, ) -> PlannedMgsUpdates { - let mut skipped_updates = Vec::new(); + let mut blocked_updates = Vec::new(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section @@ -606,7 +607,7 @@ fn try_make_update( // phase 2 changes necessary. let pending_actions = PlannedMgsUpdates::new() .add_pending_update(update) - .set_skipped_updates(skipped_updates.clone()) + .set_blocked_updates(blocked_updates.clone()) .build(); Some(pending_actions) } @@ -615,13 +616,13 @@ fn try_make_update( Ok(None) => None, // There was a failure, skip the update and record it Err(e) => { - skipped_updates.push(SkippedMgsUpdate { + blocked_updates.push(BlockedMgsUpdate { baseboard_id: baseboard_id.clone(), component, reason: e, }); let pending_actions = PlannedMgsUpdates::new() - .set_skipped_updates(skipped_updates.clone()) + .set_blocked_updates(blocked_updates.clone()) .build(); Some(pending_actions) } @@ -643,22 +644,22 @@ fn try_make_update( .set_pending_host_os_phase2_changes( pending_host_os_phase2_changes, ) - .set_skipped_updates(skipped_updates) + .set_blocked_updates(blocked_updates) .build() } // The Host OS is already at the desired version, we only need to pass // along any previous skipped updates Ok(None) => PlannedMgsUpdates::new() - .set_skipped_updates(skipped_updates) + .set_blocked_updates(blocked_updates) .build(), Err(e) => { - skipped_updates.push(SkippedMgsUpdate { + blocked_updates.push(BlockedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::HostOs, reason: e, }); PlannedMgsUpdates::new() - .set_skipped_updates(skipped_updates) + .set_blocked_updates(blocked_updates) .build() } } @@ -691,8 +692,8 @@ mod test { use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; + use nexus_types::deployment::planning_report::BlockedMgsUpdate; use nexus_types::deployment::planning_report::FailedMgsUpdateReason; - use nexus_types::deployment::planning_report::SkippedMgsUpdate; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use omicron_test_utils::dev::LogContext; @@ -702,8 +703,8 @@ mod test { // Confirm our behaviour for skipped updates #[test] - fn test_skipped_updates() { - let test_name = "planning_mgs_updates_skipped_updates"; + fn test_blocked_updates() { + let test_name = "planning_mgs_updates_blocked_updates"; let logctx = LogContext::new( test_name, &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, @@ -740,7 +741,7 @@ mod test { let PlannedMgsUpdates { pending_updates: updates, - skipped_mgs_updates, + blocked_mgs_updates, .. } = plan_mgs_updates( log, @@ -755,12 +756,12 @@ mod test { // The planner should only gather the first failed update (RoT // bootloader), and report no pending updates. There will only be a // single entry as there is only a single fake board. - let expected_skipped_updates = vec![SkippedMgsUpdate { + let expected_blocked_updates = vec![BlockedMgsUpdate { baseboard_id: fake_board.clone(), component: MgsUpdateComponent::RotBootloader, reason: FailedMgsUpdateReason::SpNotInInventory, }]; - assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert_eq!(blocked_mgs_updates, expected_blocked_updates); assert!(updates.is_empty()); // Now we build a the collection so it only reports updates necessary @@ -783,7 +784,7 @@ mod test { let PlannedMgsUpdates { pending_updates: updates, - skipped_mgs_updates, + blocked_mgs_updates, .. } = plan_mgs_updates( log, @@ -797,15 +798,15 @@ mod test { // The planner should only gather the first RoT failed update of // each of the boards, and report no pending updates - let mut expected_skipped_updates = Vec::new(); + let mut expected_blocked_updates = Vec::new(); for baseboard_id in &collection.baseboards { - expected_skipped_updates.push(SkippedMgsUpdate { + expected_blocked_updates.push(BlockedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::Rot, reason: FailedMgsUpdateReason::RotStateNotInInventory, }); } - assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert_eq!(blocked_mgs_updates, expected_blocked_updates); assert!(updates.is_empty()); // Like before we build a collection that only reports updates necessary @@ -825,7 +826,7 @@ mod test { let PlannedMgsUpdates { pending_updates: updates, - skipped_mgs_updates, + blocked_mgs_updates, .. } = plan_mgs_updates( log, @@ -839,9 +840,9 @@ mod test { // The planner should only gather the first SP failed update of // each of the boards, and report no pending updates - let mut expected_skipped_updates = Vec::new(); + let mut expected_blocked_updates = Vec::new(); for baseboard_id in &collection.baseboards { - expected_skipped_updates.push(SkippedMgsUpdate { + expected_blocked_updates.push(BlockedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::Sp, reason: FailedMgsUpdateReason::CabooseNotInInventory( @@ -849,7 +850,7 @@ mod test { ), }); } - assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert_eq!(blocked_mgs_updates, expected_blocked_updates); assert!(updates.is_empty()); // Now we create one more collection where only the Host OS needs an @@ -868,7 +869,7 @@ mod test { let PlannedMgsUpdates { pending_updates: updates, - skipped_mgs_updates, + blocked_mgs_updates, .. } = plan_mgs_updates( log, @@ -882,17 +883,17 @@ mod test { // The planner should only gather the first Host OS failed update of // each of the sled boards, and report no pending updates - let mut expected_skipped_updates = Vec::new(); + let mut expected_blocked_updates = Vec::new(); for baseboard_id in &collection.baseboards { if baseboard_id.part_number == "dummy_sled" { - expected_skipped_updates.push(SkippedMgsUpdate { + expected_blocked_updates.push(BlockedMgsUpdate { baseboard_id: baseboard_id.clone(), component: MgsUpdateComponent::HostOs, reason: FailedMgsUpdateReason::SledAgentInfoNotInInventory, }); } } - assert_eq!(skipped_mgs_updates, expected_skipped_updates); + assert_eq!(blocked_mgs_updates, expected_blocked_updates); assert!(updates.is_empty()); } diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index cb5849c85e9..767303531c7 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -225,11 +225,11 @@ impl<'a> Planner<'a> { PlanningZoneUpdatesStepReport::waiting_on( ZoneUpdatesWaitingOn::PendingMgsUpdates, ) - } else if !mgs_updates.skipped_mgs_updates.is_empty() { - // ... or if there are skipped updates for the RoT / SP / Host OS / + } else if !mgs_updates.blocked_mgs_updates.is_empty() { + // ... or if there are blocked updates for the RoT / SP / Host OS / // RoT bootloader. PlanningZoneUpdatesStepReport::waiting_on( - ZoneUpdatesWaitingOn::SkippedMgsUpdates, + ZoneUpdatesWaitingOn::BlockedMgsUpdates, ) } else if !add.add_update_blocked_reasons.is_empty() { // ... or if there are pending zone add blockers. @@ -1258,7 +1258,7 @@ impl<'a> Planner<'a> { let PlannedMgsUpdates { pending_updates, pending_host_phase_2_changes, - skipped_mgs_updates, + blocked_mgs_updates, } = plan_mgs_updates( &self.log, &self.inventory, @@ -1283,7 +1283,7 @@ impl<'a> Planner<'a> { self.blueprint.pending_mgs_updates_replace_all(pending_updates.clone()); Ok(PlanningMgsUpdatesStepReport::new( pending_updates, - skipped_mgs_updates, + blocked_mgs_updates, )) } diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index ed6b14cb2df..d4dd8293f64 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -546,7 +546,7 @@ pub enum FailedMgsUpdateReason { #[derive( Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Diffable, JsonSchema, )] -pub struct SkippedMgsUpdate { +pub struct BlockedMgsUpdate { /// id of the baseboard that we attempted to update pub baseboard_id: Arc, /// type of SP component that we attempted to update @@ -555,7 +555,7 @@ pub struct SkippedMgsUpdate { pub reason: FailedMgsUpdateReason, } -impl IdOrdItem for SkippedMgsUpdate { +impl IdOrdItem for BlockedMgsUpdate { type Key<'a> = &'a BaseboardId; fn key(&self) -> Self::Key<'_> { &*self.baseboard_id @@ -568,26 +568,26 @@ impl IdOrdItem for SkippedMgsUpdate { )] pub struct PlanningMgsUpdatesStepReport { pub pending_mgs_updates: PendingMgsUpdates, - pub skipped_mgs_updates: Vec, + pub blocked_mgs_updates: Vec, } impl PlanningMgsUpdatesStepReport { pub fn new( pending_mgs_updates: PendingMgsUpdates, - skipped_mgs_updates: Vec, + blocked_mgs_updates: Vec, ) -> Self { - Self { pending_mgs_updates, skipped_mgs_updates } + Self { pending_mgs_updates, blocked_mgs_updates } } pub fn is_empty(&self) -> bool { self.pending_mgs_updates.is_empty() - && self.skipped_mgs_updates.is_empty() + && self.blocked_mgs_updates.is_empty() } } impl fmt::Display for PlanningMgsUpdatesStepReport { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let Self { pending_mgs_updates, skipped_mgs_updates } = self; + let Self { pending_mgs_updates, blocked_mgs_updates } = self; if !pending_mgs_updates.is_empty() { let n = pending_mgs_updates.len(); let s = plural(n); @@ -600,11 +600,11 @@ impl fmt::Display for PlanningMgsUpdatesStepReport { )?; } } - if !skipped_mgs_updates.is_empty() { - let n = skipped_mgs_updates.len(); + if !blocked_mgs_updates.is_empty() { + let n = blocked_mgs_updates.len(); let s = plural(n); - writeln!(f, "* {n} skipped MGS update{s}:")?; - for update in skipped_mgs_updates { + writeln!(f, "* {n} blocked MGS update{s}:")?; + for update in blocked_mgs_updates { writeln!( f, " * {} {}: {}", @@ -1103,8 +1103,8 @@ pub enum ZoneUpdatesWaitingOn { /// Waiting on updates to RoT bootloader / RoT / SP / Host OS. PendingMgsUpdates, - /// Waiting on skipped updates to RoT bootloader / RoT / SP / Host OS. - SkippedMgsUpdates, + /// Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS. + BlockedMgsUpdates, /// Waiting on the same set of blockers zone adds are waiting on. ZoneAddBlockers, @@ -1117,8 +1117,8 @@ impl ZoneUpdatesWaitingOn { Self::PendingMgsUpdates => { "pending MGS updates (RoT bootloader / RoT / SP / Host OS)" } - Self::SkippedMgsUpdates => { - "skipped MGS updates (RoT bootloader / RoT / SP / Host OS)" + Self::BlockedMgsUpdates => { + "blocked MGS updates (RoT bootloader / RoT / SP / Host OS)" } Self::ZoneAddBlockers => "zone add blockers", } diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 0b68dfe4ce3..1cc6c5ef9d2 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -2511,6 +2511,40 @@ "port" ] }, + "BlockedMgsUpdate": { + "type": "object", + "properties": { + "baseboard_id": { + "description": "id of the baseboard that we attempted to update", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "component": { + "description": "type of SP component that we attempted to update", + "allOf": [ + { + "$ref": "#/components/schemas/MgsUpdateComponent" + } + ] + }, + "reason": { + "description": "reason why the update failed", + "allOf": [ + { + "$ref": "#/components/schemas/FailedMgsUpdateReason" + } + ] + } + }, + "required": [ + "baseboard_id", + "component", + "reason" + ] + }, "Blueprint": { "description": "Describes a complete set of software and configuration for the system", "type": "object", @@ -7270,16 +7304,19 @@ "PlanningMgsUpdatesStepReport": { "type": "object", "properties": { + "blocked_mgs_updates": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BlockedMgsUpdate" + } + }, "pending_mgs_updates": { "$ref": "#/components/schemas/PendingMgsUpdates" - }, - "skipped_mgs_updates": { - "$ref": "#/components/schemas/SkippedMgsUpdates" } }, "required": [ - "pending_mgs_updates", - "skipped_mgs_updates" + "blocked_mgs_updates", + "pending_mgs_updates" ] }, "PlanningNoopImageSourceConverted": { @@ -8936,54 +8973,6 @@ "format": "uint64", "minimum": 0 }, - "SkippedMgsUpdate": { - "type": "object", - "properties": { - "baseboard_id": { - "description": "id of the baseboard that we attempted to update", - "allOf": [ - { - "$ref": "#/components/schemas/BaseboardId" - } - ] - }, - "component": { - "description": "type of SP component that we attempted to update", - "allOf": [ - { - "$ref": "#/components/schemas/MgsUpdateComponent" - } - ] - }, - "reason": { - "description": "reason why the update failed", - "allOf": [ - { - "$ref": "#/components/schemas/FailedMgsUpdateReason" - } - ] - } - }, - "required": [ - "baseboard_id", - "component", - "reason" - ] - }, - "SkippedMgsUpdates": { - "type": "object", - "properties": { - "updates": { - "type": "array", - "items": { - "$ref": "#/components/schemas/SkippedMgsUpdate" - } - } - }, - "required": [ - "updates" - ] - }, "SledAgentInfo": { "description": "Sent by a sled agent to Nexus to inform about resources", "type": "object", @@ -10104,13 +10093,13 @@ ] }, { - "description": "Waiting on skipped updates to RoT bootloader / RoT / SP / Host OS.", + "description": "Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "skipped_mgs_updates" + "blocked_mgs_updates" ] } }, From 1298942beb90da56813b1f12787395584f3ff391 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 15:59:22 +1200 Subject: [PATCH 45/55] address comments --- .../planning/src/mgs_updates/mod.rs | 224 +++++++++--------- nexus/reconfigurator/planning/src/planner.rs | 6 +- nexus/types/src/deployment/blueprint_diff.rs | 2 +- nexus/types/src/deployment/planning_report.rs | 12 +- 4 files changed, 127 insertions(+), 117 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 0e14d7bdd41..389a01c0222 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -29,7 +29,6 @@ use nexus_types::deployment::PendingMgsUpdateSpDetails; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::TargetReleaseDescription; use nexus_types::deployment::planning_report::BlockedMgsUpdate; -use nexus_types::deployment::planning_report::FailedMgsUpdateReason; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; @@ -68,7 +67,8 @@ pub(crate) struct PlannedMgsUpdates { /// result in a change to the respective sled's `BlueprintSledConfig`. pub(crate) pending_host_phase_2_changes: PendingHostPhase2Changes, - // Updates to components that failed for some reason and have been blocked. + /// Updates to components that cannot be planned due to a failure in a + /// previous attempt. pub(crate) blocked_mgs_updates: Vec, } @@ -89,31 +89,21 @@ impl PlannedMgsUpdates { self } - fn set_pending_host_os_phase2_changes( + fn add_blocked_update( &mut self, - pending_host_os_phase2_changes: PendingHostPhase2Changes, + blocked_update: BlockedMgsUpdate, ) -> &mut Self { - self.pending_host_phase_2_changes = pending_host_os_phase2_changes; + self.blocked_mgs_updates.push(blocked_update); self } - fn set_blocked_updates( + fn set_pending_host_os_phase2_changes( &mut self, - blocked_updates: Vec, + pending_host_os_phase2_changes: PendingHostPhase2Changes, ) -> &mut Self { - self.blocked_mgs_updates = blocked_updates; + self.pending_host_phase_2_changes = pending_host_os_phase2_changes; self } - - fn build(&self) -> Self { - Self { - pending_updates: self.pending_updates.clone(), - pending_host_phase_2_changes: self - .pending_host_phase_2_changes - .clone(), - blocked_mgs_updates: self.blocked_mgs_updates.clone(), - } - } } /// Generates a new set of `PendingMgsUpdates` based on: @@ -566,103 +556,123 @@ fn try_make_update( inventory: &Collection, current_artifacts: &TufRepoDescription, ) -> PlannedMgsUpdates { - let mut blocked_updates = Vec::new(); + let mut pending_actions = PlannedMgsUpdates::new(); // We try MGS-driven update components in a hardcoded priority order until // any of them returns `Some`. The order is described in RFD 565 section // "Update Sequence". - type UpdateResult = Result, FailedMgsUpdateReason>; - let attempts: [(MgsUpdateComponent, UpdateResult); 3] = [ - ( - MgsUpdateComponent::RotBootloader, - try_make_update_rot_bootloader( - log, - baseboard_id, - inventory, - current_artifacts, - ), - ), - ( - MgsUpdateComponent::Rot, - try_make_update_rot( - log, - baseboard_id, - inventory, - current_artifacts, - ), - ), - ( - MgsUpdateComponent::Sp, - try_make_update_sp(log, baseboard_id, inventory, current_artifacts), - ), - ]; - - if let Some(update_actions) = - attempts.into_iter().find_map(|(component, attempt)| { - match attempt { - // There is a pending update, record it along with any previous - // failed updates - Ok(Some(update)) => { - // We have a non-host update; there are no pending host - // phase 2 changes necessary. - let pending_actions = PlannedMgsUpdates::new() - .add_pending_update(update) - .set_blocked_updates(blocked_updates.clone()) - .build(); - Some(pending_actions) - } - // We don't have any pending actions, the component is already - // at the expected version - Ok(None) => None, - // There was a failure, skip the update and record it - Err(e) => { - blocked_updates.push(BlockedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component, - reason: e, - }); - let pending_actions = PlannedMgsUpdates::new() - .set_blocked_updates(blocked_updates.clone()) - .build(); - Some(pending_actions) - } + for component in [ + MgsUpdateComponent::RotBootloader, + MgsUpdateComponent::Rot, + MgsUpdateComponent::Sp, + MgsUpdateComponent::HostOs, + ] { + match component { + MgsUpdateComponent::RotBootloader => { + match try_make_update_rot_bootloader( + log, + baseboard_id, + inventory, + current_artifacts, + ) { + Ok(pending_update) => { + if let Some(update) = pending_update { + // There is a pending update, record it + pending_actions.add_pending_update(update); + } + // Otherwise, we don't have any pending actions, the + // component is already at the expected version + } + Err(e) => { + // There was a failure, skip the update and record it + pending_actions.add_blocked_update(BlockedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component, + reason: e, + }); + } + }; } - }) - { - return update_actions; - } + MgsUpdateComponent::Rot => { + match try_make_update_rot( + log, + baseboard_id, + inventory, + current_artifacts, + ) { + Ok(pending_update) => { + if let Some(update) = pending_update { + pending_actions.add_pending_update(update); + } + } + Err(e) => { + pending_actions.add_blocked_update(BlockedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component, + reason: e, + }); + } + }; + } + MgsUpdateComponent::Sp => { + match try_make_update_sp( + log, + baseboard_id, + inventory, + current_artifacts, + ) { + Ok(pending_update) => { + if let Some(update) = pending_update { + pending_actions.add_pending_update(update); + } + } + Err(e) => { + pending_actions.add_blocked_update(BlockedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component, + reason: e, + }); + } + }; + } + MgsUpdateComponent::HostOs => { + match host_phase_1::try_make_update( + log, + baseboard_id, + inventory, + current_artifacts, + ) { + Ok(pending_update) => { + if let Some((update, host_os_phase_2_changes)) = + pending_update + { + pending_actions.add_pending_update(update); + pending_actions.set_pending_host_os_phase2_changes( + host_os_phase_2_changes, + ); + } + } + Err(e) => { + pending_actions.add_blocked_update(BlockedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component, + reason: e, + }); + } + }; + } + }; - match host_phase_1::try_make_update( - log, - baseboard_id, - inventory, - current_artifacts, - ) { - Ok(Some((update, pending_host_os_phase2_changes))) => { - PlannedMgsUpdates::new() - .add_pending_update(update) - .set_pending_host_os_phase2_changes( - pending_host_os_phase2_changes, - ) - .set_blocked_updates(blocked_updates) - .build() - } - // The Host OS is already at the desired version, we only need to pass - // along any previous skipped updates - Ok(None) => PlannedMgsUpdates::new() - .set_blocked_updates(blocked_updates) - .build(), - Err(e) => { - blocked_updates.push(BlockedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component: MgsUpdateComponent::HostOs, - reason: e, - }); - PlannedMgsUpdates::new() - .set_blocked_updates(blocked_updates) - .build() + // If there is a pending or blocked MGS driven update we return it + // immediately. + if !pending_actions.blocked_mgs_updates.is_empty() + || !pending_actions.pending_updates.is_empty() + { + return pending_actions; } } + + pending_actions } #[cfg(test)] diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 767303531c7..d222475a22f 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -35,7 +35,6 @@ use nexus_types::deployment::CockroachDbClusterVersion; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::CockroachDbSettings; use nexus_types::deployment::DiskFilter; -use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::PlanningInput; use nexus_types::deployment::SledDetails; use nexus_types::deployment::SledFilter; @@ -194,10 +193,7 @@ impl<'a> Planner<'a> { let mgs_updates = if add_update_blocked_reasons.is_empty() { self.do_plan_mgs_updates()? } else { - PlanningMgsUpdatesStepReport::new( - PendingMgsUpdates::new(), - Vec::new(), - ) + PlanningMgsUpdatesStepReport::empty() }; // Likewise for zone additions, unless overridden by the config. diff --git a/nexus/types/src/deployment/blueprint_diff.rs b/nexus/types/src/deployment/blueprint_diff.rs index 0d35097a1e1..b4be6835c65 100644 --- a/nexus/types/src/deployment/blueprint_diff.rs +++ b/nexus/types/src/deployment/blueprint_diff.rs @@ -2140,7 +2140,7 @@ impl fmt::Display for BlueprintDiffDisplay<'_, '_> { writeln!(f, "{}", table)?; } - // TODO-K: Add skipped updates in a follow up PR + // TODO-K: Add blocked updates in a follow up PR Ok(()) } diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index d4dd8293f64..42156f2eed0 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -81,10 +81,7 @@ impl PlanningReport { expunge: PlanningExpungeStepReport::new(), decommission: PlanningDecommissionStepReport::new(), noop_image_source: PlanningNoopImageSourceStepReport::new(), - mgs_updates: PlanningMgsUpdatesStepReport::new( - PendingMgsUpdates::new(), - Vec::new(), - ), + mgs_updates: PlanningMgsUpdatesStepReport::empty(), add: PlanningAddStepReport::new(), zone_updates: PlanningZoneUpdatesStepReport::new(), cockroachdb_settings: PlanningCockroachdbSettingsStepReport::new(), @@ -579,6 +576,13 @@ impl PlanningMgsUpdatesStepReport { Self { pending_mgs_updates, blocked_mgs_updates } } + pub fn empty() -> Self { + Self { + pending_mgs_updates: PendingMgsUpdates::new(), + blocked_mgs_updates: Vec::new(), + } + } + pub fn is_empty(&self) -> bool { self.pending_mgs_updates.is_empty() && self.blocked_mgs_updates.is_empty() From 45f9dbd77985ec1558fbc3d08f62e4ac87fe9ee1 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 17:28:30 +1200 Subject: [PATCH 46/55] expectorate --- .../output/cmds-mupdate-update-flow-stdout | 5 +- .../tests/output/cmds-target-release-stdout | 1330 ++--------------- 2 files changed, 168 insertions(+), 1167 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout index b8eb312f53b..c48a4381f89 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-mupdate-update-flow-stdout @@ -1813,7 +1813,7 @@ planning report: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * only placed 0/1 desired nexus zones -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated @@ -1997,9 +1997,8 @@ planning report: * noop converting 7/7 install-dataset zones to artifact store on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) -* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) * only placed 0/1 desired nexus zones -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index ef4d2b46fc4..d9095695454 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -148,7 +148,7 @@ planning report: * noop converting 8/8 install-dataset zones to artifact store on sled d81c6a84-79b8-4958-ae41-ea46c9b19763 * 1 pending MGS update: * model0:serial0: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) @@ -1121,6 +1121,8 @@ planning report: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1183,6 +1185,8 @@ planning report: * 1 pending MGS update: * model0:serial0: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1247,6 +1251,8 @@ planning report: * 1 pending MGS update: * model0:serial0: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1320,6 +1326,8 @@ planning report: * 1 pending MGS update: * model0:serial0: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1394,6 +1402,8 @@ planning report: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1508,123 +1518,6 @@ INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noo INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 INFO reached maximum number of pending MGS-driven updates, max: 1 -generated blueprint 9034c710-3e57-45f3-99e5-4316145e87ac based on parent blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -planning report for blueprint 9034c710-3e57-45f3-99e5-4316145e87ac: -* 1 pending MGS update: - * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) - - -> blueprint-diff latest -from: blueprint 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba -to: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> # Update only the simulated host phase 2; this is a prerequisite for updating -> # the phase 1, and must be done first. -> sled-update-host-phase2 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --slot-b d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47 -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 host phase 2 details: B -> d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47 - -> inventory-generate -generated inventory collection a71f7a73-35a6-45e8-acbe-f1c5925eed69 from configured sleds - - -> # Planning after only phase 2 has changed should make no changes. We're still -> # waiting on phase 1 to change. -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO MGS-driven update not yet completed (will keep it), artifact_version: 1.0.0, artifact_hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO reached maximum number of pending MGS-driven updates, max: 1 -generated blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 based on parent blueprint 9034c710-3e57-45f3-99e5-4316145e87ac -planning report for blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976: -* 1 pending MGS update: - * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) - - -> blueprint-diff latest -from: blueprint 9034c710-3e57-45f3-99e5-4316145e87ac -to: blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> # Now update the simulated SP to reflect that the phase 1 update is done. -> sled-update-host-phase1 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 --active B --slot-b b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 host phase 1 details: active -> B, B -> b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b - -> inventory-generate -generated inventory collection 0b5efbb3-0b1b-4bbf-b7d8-a2d6fca074c6 from configured sleds - - -> # Planning _still_ shouldn't make any new changes; the OS update as a whole -> # isn't done until sled-agent reports it has booted from the new image. -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b -INFO keeping apparently-impossible MGS-driven update (waiting for recent update to be applied), artifact_version: 1.0.0, artifact_hash: b99d5273ba1418bebb19d74b701d716896409566d41de76ada71bded4c9b166b, sled_agent_address: [fd00:1122:3344:101::1]:12345, expected_inactive_phase_2_hash: d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47, expected_inactive_phase_1_hash: 0202020202020202020202020202020202020202020202020202020202020202, expected_active_phase_2_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a, expected_active_phase_1_hash: 0101010101010101010101010101010101010101010101010101010101010101, expected_boot_disk: A, expected_active_phase_1_slot: A, component: host_phase_1, sp_slot: 0, sp_type: Sled, serial_number: serial0, part_number: model0 -INFO reached maximum number of pending MGS-driven updates, max: 1 generated blueprint a5a8f242-ffa5-473c-8efd-2acf2dc0b736 based on parent blueprint d60afc57-f15d-476c-bd0f-b1071e2bb976 blueprint source: planner with report: planning report: @@ -1634,6 +1527,8 @@ planning report: * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1696,7 +1591,7 @@ planning report: * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 8 zones are already from artifacts * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated @@ -1760,7 +1655,7 @@ planning report: * skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 8 zones are already from artifacts * 1 pending MGS update: * model0:serial0: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:101::1]:12345 }) -* zone updates waiting on pending MGS updates (RoT / SP / Host OS / etc.) +* zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated @@ -1827,6 +1722,8 @@ planning report: * 1 pending MGS update: * model1:serial1: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1902,6 +1799,8 @@ planning report: * 1 pending MGS update: * model1:serial1: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: Version(ArtifactVersion("0.5.0")) }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -1972,6 +1871,8 @@ planning report: * 1 pending MGS update: * model1:serial1: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2046,6 +1947,8 @@ planning report: * 1 pending MGS update: * model1:serial1: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("0.5.0")), expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2116,6 +2019,8 @@ planning report: * 1 pending MGS update: * model1:serial1: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2189,6 +2094,8 @@ planning report: * 1 pending MGS update: * model1:serial1: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: Version(ArtifactVersion("0.5.0")) }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2260,6 +2167,8 @@ planning report: * 1 pending MGS update: * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2398,6 +2307,8 @@ planning report: * 1 pending MGS update: * model1:serial1: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:102::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2474,6 +2385,8 @@ planning report: * 1 pending MGS update: * model2:serial2: RotBootloader(PendingMgsUpdateRotBootloaderDetails { expected_stage0_version: ArtifactVersion("0.0.1"), expected_stage0_next_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2543,6 +2456,8 @@ planning report: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: NoValidVersion, expected_persistent_boot_preference: A, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2616,6 +2531,8 @@ planning report: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: A, version: ArtifactVersion("0.0.2") }, expected_inactive_version: Version(ArtifactVersion("1.0.0")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2691,6 +2608,8 @@ planning report: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: Some(B), expected_transient_boot_preference: None }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2763,6 +2682,8 @@ planning report: * 1 pending MGS update: * model2:serial2: Rot(PendingMgsUpdateRotDetails { expected_active_slot: ExpectedActiveRotSlot { slot: B, version: ArtifactVersion("1.1.0") }, expected_inactive_version: Version(ArtifactVersion("0.0.2")), expected_persistent_boot_preference: B, expected_pending_persistent_boot_preference: None, expected_transient_boot_preference: Some(B) }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2833,6 +2754,8 @@ planning report: * 1 pending MGS update: * model2:serial2: Sp(PendingMgsUpdateSpDetails { expected_active_version: ArtifactVersion("0.0.1"), expected_inactive_version: NoValidVersion }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -2902,6 +2825,8 @@ planning report: * 1 pending MGS update: * model2:serial2: HostPhase1(PendingMgsUpdateHostPhase1Details { expected_active_phase_1_slot: A, expected_boot_disk: A, expected_active_phase_1_hash: ArtifactHash("0101010101010101010101010101010101010101010101010101010101010101"), expected_active_phase_2_hash: ArtifactHash("0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"), expected_inactive_phase_1_hash: ArtifactHash("0202020202020202020202020202020202020202020202020202020202020202"), expected_inactive_phase_2_hash: ArtifactHash("d944ae205b61ccf4322448f7d0311a819c53d9844769de066c5307c1682abb47"), sled_agent_address: [fd00:1122:3344:103::1]:12345 }) * zone updates waiting on pending MGS updates (RoT bootloader / RoT / SP / Host OS) +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest @@ -3600,12 +3525,12 @@ planning report: > blueprint-diff latest -from: blueprint 6fad8fd4-e825-433f-b76d-495484e068ce -to: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 +from: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce +to: blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 MODIFIED SLEDS: - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 7 -> 8): + sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 8 -> 9): host phase 2 contents: ------------------------------ @@ -3646,12 +3571,12 @@ to: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off ++ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 22a4acd6-9d38-43e2-a3bf-c85f5c2f3246 in service none none off ++ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_4ab0ec67-b27e-42b5-af22-9117ad11113b 75ffc8e6-b071-4f51-966d-4a6e6b01b432 in service none none off omicron zones: @@ -3659,17 +3584,17 @@ to: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 zone type zone id image source disposition underlay IP -------------------------------------------------------------------------------------------------------------------------- clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 install dataset in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset in service fd00:1122:3344:102::25 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset - expunged ⏳ fd00:1122:3344:102::24 + crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 0.0.1 in service fd00:1122:3344:102::28 + crucible bd354eef-d8a6-4165-9124-283fb5e46d77 artifact: version 0.0.1 in service fd00:1122:3344:102::26 + crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c artifact: version 0.0.1 in service fd00:1122:3344:102::27 + crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 artifact: version 0.0.1 in service fd00:1122:3344:102::25 + internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca artifact: version 0.0.1 in service fd00:1122:3344:1::1 + internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 artifact: version 0.0.1 expunged ✓ fd00:1122:3344:102::21 + internal_ntp ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::29 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec artifact: version 0.0.1 in service fd00:1122:3344:102::22 +* external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a artifact: version 0.0.1 - expunged ⏳ fd00:1122:3344:102::24 └─ + expunged ✓ -+ external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a ++ external_dns 4ab0ec67-b27e-42b5-af22-9117ad11113b artifact: version 1.0.0 in service fd00:1122:3344:102::2a COCKROACHDB SETTINGS: @@ -3689,15 +3614,15 @@ to: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 internal DNS: * DNS zone: "control-plane.oxide.internal": ++ name: 4ab0ec67-b27e-42b5-af22-9117ad11113b.host (records: 1) ++ AAAA fd00:1122:3344:102::2a * name: _external-dns._tcp (records: 2 -> 3) - SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal - SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal ++ SRV port 5353 4ab0ec67-b27e-42b5-af22-9117ad11113b.host.control-plane.oxide.internal + SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -+ SRV port 5353 ba87399e-e9b7-4ee4-8cb7-0032822630e9.host.control-plane.oxide.internal + SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ name: ba87399e-e9b7-4ee4-8cb7-0032822630e9.host (records: 1) -+ AAAA fd00:1122:3344:102::2a - unchanged names: 49 (records: 61) + unchanged names: 50 (records: 64) external DNS: * DNS zone: "oxide.example": @@ -3721,36 +3646,42 @@ external DNS: > sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest blueprint (24b6e243-100c-428d-8ea6-35b504226f55) +set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest blueprint (3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312) > inventory-generate -generated inventory collection 756aecb6-8353-46ad-a6c4-10ad0f2bbb7f from configured sleds +generated inventory collection bcfc7436-77de-47e4-8158-daad15a54da2 from configured sleds > blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 3, num_eligible: 0, num_ineligible: 6 +INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 +INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update -generated blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce based on parent blueprint 24b6e243-100c-428d-8ea6-35b504226f55 -planning report for blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce: +generated blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d based on parent blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 +blueprint source: planner with report: +planning report: +* skipping noop zone image source check on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: all 9 zones are already from artifacts +* skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 8 zones are already from artifacts +* skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 8 zones are already from artifacts * 1 out-of-date zone updated in-place: * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 86a22a56-0168-453d-9df1-cb2a7c64b5d3 (crucible) * 22 remaining out-of-date zones +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest -from: blueprint 24b6e243-100c-428d-8ea6-35b504226f55 -to: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce +from: blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 +to: blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d MODIFIED SLEDS: - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 8 -> 9): + sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 9 -> 10): host phase 2 contents: ------------------------------ @@ -3778,6 +3709,7 @@ to: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 22a4acd6-9d38-43e2-a3bf-c85f5c2f3246 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off @@ -3787,6 +3719,7 @@ to: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off + oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_4ab0ec67-b27e-42b5-af22-9117ad11113b 75ffc8e6-b071-4f51-966d-4a6e6b01b432 in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off @@ -3795,26 +3728,24 @@ to: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 22a4acd6-9d38-43e2-a3bf-c85f5c2f3246 in service none none off -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_4ab0ec67-b27e-42b5-af22-9117ad11113b 75ffc8e6-b071-4f51-966d-4a6e6b01b432 in service none none off omicron zones: -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP + zone type zone id image source disposition underlay IP -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 0.0.1 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 artifact: version 0.0.1 in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c artifact: version 0.0.1 in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 artifact: version 0.0.1 in service fd00:1122:3344:102::25 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca artifact: version 0.0.1 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 artifact: version 0.0.1 expunged ✓ fd00:1122:3344:102::21 - internal_ntp ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec artifact: version 0.0.1 in service fd00:1122:3344:102::22 -* external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a artifact: version 0.0.1 - expunged ⏳ fd00:1122:3344:102::24 - └─ + expunged ✓ -+ external_dns 4ab0ec67-b27e-42b5-af22-9117ad11113b artifact: version 1.0.0 in service fd00:1122:3344:102::2a + clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 + crucible bd354eef-d8a6-4165-9124-283fb5e46d77 artifact: version 0.0.1 in service fd00:1122:3344:102::26 + crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c artifact: version 0.0.1 in service fd00:1122:3344:102::27 + crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 artifact: version 0.0.1 in service fd00:1122:3344:102::25 + external_dns 4ab0ec67-b27e-42b5-af22-9117ad11113b artifact: version 1.0.0 in service fd00:1122:3344:102::2a + external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a artifact: version 0.0.1 expunged ✓ fd00:1122:3344:102::24 + internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca artifact: version 0.0.1 in service fd00:1122:3344:1::1 + internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 artifact: version 0.0.1 expunged ✓ fd00:1122:3344:102::21 + internal_ntp ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::29 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec artifact: version 0.0.1 in service fd00:1122:3344:102::22 +* crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 - artifact: version 0.0.1 in service fd00:1122:3344:102::28 + └─ + artifact: version 1.0.0 COCKROACHDB SETTINGS: @@ -3833,159 +3764,8 @@ to: blueprint 79fff7a2-2495-4c75-8465-4dc01bab48ce internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 4ab0ec67-b27e-42b5-af22-9117ad11113b.host (records: 1) -+ AAAA fd00:1122:3344:102::2a -* name: _external-dns._tcp (records: 2 -> 3) -- SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -- SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal -+ SRV port 5353 4ab0ec67-b27e-42b5-af22-9117ad11113b.host.control-plane.oxide.internal -+ SRV port 5353 803bfb63-c246-41db-b0da-d3b87ddfc63d.host.control-plane.oxide.internal -+ SRV port 5353 f6ec9c67-946a-4da3-98d5-581f72ce8bf0.host.control-plane.oxide.internal - unchanged names: 50 (records: 64) - -external DNS: -* DNS zone: "oxide.example": -* name: @ (records: 2 -> 3) -- NS ns1.oxide.example -- NS ns2.oxide.example -+ NS ns1.oxide.example -+ NS ns2.oxide.example -+ NS ns3.oxide.example -* name: ns1 (records: 1 -> 1) -- A 198.51.100.2 -+ A 198.51.100.1 -* name: ns2 (records: 1 -> 1) -- A 198.51.100.3 -+ A 198.51.100.2 -+ name: ns3 (records: 1) -+ A 198.51.100.3 - unchanged names: 1 (records: 3) - - - - -> sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest blueprint (3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312) - -> inventory-generate -generated inventory collection bcfc7436-77de-47e4-8158-daad15a54da2 from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d based on parent blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 -blueprint source: planner with report: -planning report: -* skipping noop zone image source check on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: all 9 zones are already from artifacts -* skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 8 zones are already from artifacts -* skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 8 zones are already from artifacts -* 1 out-of-date zone updated in-place: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone 86a22a56-0168-453d-9df1-cb2a7c64b5d3 (crucible) -* 22 remaining out-of-date zones -* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated - - - -> blueprint-diff latest -from: blueprint 3bcc37b2-0c0b-44d0-b4ed-3bcb605e4312 -to: blueprint 4d2eb6f3-7eb1-443a-8e76-7ecf05da2f6d - - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 9 -> 10): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 22a4acd6-9d38-43e2-a3bf-c85f5c2f3246 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_4ab0ec67-b27e-42b5-af22-9117ad11113b 75ffc8e6-b071-4f51-966d-4a6e6b01b432 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 artifact: version 0.0.1 in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c artifact: version 0.0.1 in service fd00:1122:3344:102::27 - crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 artifact: version 0.0.1 in service fd00:1122:3344:102::25 - external_dns 4ab0ec67-b27e-42b5-af22-9117ad11113b artifact: version 1.0.0 in service fd00:1122:3344:102::2a - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a artifact: version 0.0.1 expunged ✓ fd00:1122:3344:102::24 - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca artifact: version 0.0.1 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 artifact: version 0.0.1 expunged ✓ fd00:1122:3344:102::21 - internal_ntp ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec artifact: version 0.0.1 in service fd00:1122:3344:102::22 -* crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 - artifact: version 0.0.1 in service fd00:1122:3344:102::28 - └─ + artifact: version 1.0.0 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 52 (records: 68) + DNS zone: "control-plane.oxide.internal" (unchanged) + unchanged names: 52 (records: 68) external DNS: DNS zone: "oxide.example" (unchanged) @@ -4473,144 +4253,8 @@ planning report: > blueprint-diff latest -from: blueprint e2125c83-b255-45c9-bc9b-802cff09a812 -to: blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 - - MODIFIED SLEDS: - - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 12 -> 13): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-727522a7-934f-494d-b5b3-160968e74463 in service - fake-vendor fake-model serial-72c59873-31ff-4e36-8d76-ff834009349a in service - fake-vendor fake-model serial-b5fd5bc1-099e-4e77-8028-a9793c11f43b in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_727522a7-934f-494d-b5b3-160968e74463/crucible 2f204c50-a327-479c-8852-f53ec7a19c1f in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crucible 78f34ce7-42f1-41da-995f-318f32054ad2 in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crucible 1640adb6-70bf-44cf-b05c-bff6dd300cf3 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/clickhouse 841d5648-05f0-47b0-b446-92f6b60fe9a6 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 8e0bd2bd-23b7-4bc6-9e73-c4d4ebc0bc8c expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/external_dns 9f185688-315b-445a-824a-84ff526417f2 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 5c01fdbd-ff37-44b4-a17b-6d625e6fa48d in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/internal_dns 2ad1875a-92ac-472f-8c26-593309f0e4da expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone 4829f422-aa31-41a8-ab73-95684ff1ef48 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone 775f9207-c42d-4af2-9186-27ffef67735e in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone 3b66453b-7148-4c1b-84a9-499e43290ab4 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_clickhouse_353b3b65-20f7-48c3-88f7-495bd5d31545 b46de15d-33e7-4cd0-aa7c-e7be2a61e71b in service none none off - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/zone/oxz_crucible_86a22a56-0168-453d-9df1-cb2a7c64b5d3 3e0d6188-c503-49cf-a441-fa7df40ceb43 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_bd354eef-d8a6-4165-9124-283fb5e46d77 5ae11c7e-08fa-4d78-a4ea-14b4a9a10241 in service none none off - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/zone/oxz_crucible_e2fdefe7-95b2-4fd2-ae37-56929a06d58c b8f2a09f-8bd2-4418-872b-a4457a3f958c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_ad6a3a03-8d0f-4504-99a4-cbf73d69b973 49f8fbb6-5bac-4609-907f-6e3dfc206059 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_6c3ae381-04f7-41ea-b0ac-74db387dbc3a 8c4fa711-1d5d-4e93-85f0-d17bff47b063 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_external_dns_ba87399e-e9b7-4ee4-8cb7-0032822630e9 484f151e-c290-48bd-99b2-c97ef85c9844 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_99e2f30b-3174-40bf-a78a-90da8abba8ca c31623de-c19b-4615-9f1d-5e1daa5d3bda expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_internal_dns_c821c39d-2b2c-4c55-8874-ac12315ba1e4 a21812e1-1b80-4faa-9f2b-51189e0f6999 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_nexus_466a9f29-62bf-4e63-924a-b9efdb86afec 3560dd69-3b23-4c69-807d-d673104cfc68 in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_62620961-fc4a-481e-968b-f5acbac0dc63 09b9cc9b-3426-470b-a7bc-538f82dede03 expunged none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_ntp_f83ade6d-9ab1-4679-813b-b9457e039c0b 7642e649-44d6-49b0-ba7c-cc3f6b6f2f9c in service none none off - oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/debug 93957ca0-9ed1-4e7b-8c34-2ce07a69541c in service 100 GiB none gzip-9 - oxp_72c59873-31ff-4e36-8d76-ff834009349a/crypt/debug 2db6b7c1-0f46-4ced-a3ad-48872793360e in service 100 GiB none gzip-9 - oxp_b5fd5bc1-099e-4e77-8028-a9793c11f43b/crypt/debug 318fae85-abcb-4259-b1b6-ac96d193f7b7 in service 100 GiB none gzip-9 -+ oxp_727522a7-934f-494d-b5b3-160968e74463/crypt/zone/oxz_crucible_pantry_698d1d82-0620-4978-93ac-0ba5d40f3da9 dfe5586b-e4a8-4b98-ad72-eabc34988177 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - clickhouse 353b3b65-20f7-48c3-88f7-495bd5d31545 artifact: version 1.0.0 in service fd00:1122:3344:102::23 - crucible 86a22a56-0168-453d-9df1-cb2a7c64b5d3 artifact: version 1.0.0 in service fd00:1122:3344:102::28 - crucible bd354eef-d8a6-4165-9124-283fb5e46d77 install dataset in service fd00:1122:3344:102::26 - crucible e2fdefe7-95b2-4fd2-ae37-56929a06d58c install dataset in service fd00:1122:3344:102::27 - external_dns 6c3ae381-04f7-41ea-b0ac-74db387dbc3a install dataset expunged ✓ fd00:1122:3344:102::24 - external_dns ba87399e-e9b7-4ee4-8cb7-0032822630e9 artifact: version 1.0.0 in service fd00:1122:3344:102::2a - internal_dns 99e2f30b-3174-40bf-a78a-90da8abba8ca install dataset expunged ✓ fd00:1122:3344:1::1 - internal_dns c821c39d-2b2c-4c55-8874-ac12315ba1e4 artifact: version 1.0.0 in service fd00:1122:3344:1::1 - internal_ntp 62620961-fc4a-481e-968b-f5acbac0dc63 install dataset expunged ✓ fd00:1122:3344:102::21 - internal_ntp f83ade6d-9ab1-4679-813b-b9457e039c0b artifact: version 1.0.0 in service fd00:1122:3344:102::29 - nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 -* crucible_pantry ad6a3a03-8d0f-4504-99a4-cbf73d69b973 install dataset - expunged ⏳ fd00:1122:3344:102::25 - └─ + expunged ✓ -+ crucible_pantry 698d1d82-0620-4978-93ac-0ba5d40f3da9 artifact: version 1.0.0 in service fd00:1122:3344:102::2b - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 698d1d82-0620-4978-93ac-0ba5d40f3da9.host (records: 1) -+ AAAA fd00:1122:3344:102::2b -* name: _crucible-pantry._tcp (records: 2 -> 3) -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -- SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -+ SRV port 17000 ba4994a8-23f9-4b1a-a84f-a08d74591389.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron-config latest -set sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c omicron config from latest blueprint (f4a6848e-d13c-46e1-8c6a-944f886d7ba3) - -> inventory-generate -generated inventory collection 4492baf6-5638-4c1f-bba2-608163519022 from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 6, num_eligible: 0, num_ineligible: 3 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c based on parent blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 -planning report for blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c: -* 1 out-of-date zone updated in-place: - * sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zone bd354eef-d8a6-4165-9124-283fb5e46d77 (crucible) -* 19 remaining out-of-date zones - - -> blueprint-diff latest -from: blueprint f4a6848e-d13c-46e1-8c6a-944f886d7ba3 -to: blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c +from: blueprint 834e4dbe-3b71-443d-bd4c-20e8253abc0c +to: blueprint d9c5c5e3-c532-4c45-9ef5-22cb00f6a2e1 MODIFIED SLEDS: @@ -4867,9 +4511,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf based on parent blueprint e2deb7c0-2262-49fe-855f-4250c22afb36 blueprint source: planner with report: @@ -5286,12 +4930,12 @@ planning report: > blueprint-diff latest -from: blueprint 23ce505c-8991-44a5-8863-f2b906fba9cf -to: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d +from: blueprint 60b55d33-5fec-4277-9864-935197eaead7 +to: blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 MODIFIED SLEDS: - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 5 -> 6): + sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 6 -> 7): host phase 2 contents: ------------------------------ @@ -5333,8 +4977,8 @@ to: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off ++ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns 68e4149e-114c-460e-8f33-54dd5f9a274e in service none none off ++ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_abd27551-4027-4084-8b52-13a575b035b4 6119babc-cdc9-4b6b-afa4-9037eee05728 in service none none off omicron zones: @@ -5342,269 +4986,15 @@ to: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d zone type zone id image source disposition underlay IP -------------------------------------------------------------------------------------------------------------------------- crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 install dataset in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset - expunged ⏳ fd00:1122:3344:2::1 + crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 0.0.1 in service fd00:1122:3344:101::25 + crucible dfac80b4-a887-430a-ae87-a4e065dba787 artifact: version 0.0.1 in service fd00:1122:3344:101::26 + crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 artifact: version 0.0.1 in service fd00:1122:3344:101::24 + external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d artifact: version 0.0.1 in service fd00:1122:3344:101::23 + internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c artifact: version 0.0.1 in service fd00:1122:3344:101::21 + nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 artifact: version 0.0.1 in service fd00:1122:3344:101::22 +* internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c artifact: version 0.0.1 - expunged ⏳ fd00:1122:3344:2::1 └─ + expunged ✓ -+ internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 71f71743-8c73-43c6-b080-427ec28ef4c9.host (records: 1) -+ AAAA fd00:1122:3344:2::1 -* name: @ (records: 2 -> 3) -- NS ns1.control-plane.oxide.internal -- NS ns2.control-plane.oxide.internal -+ NS ns1.control-plane.oxide.internal -+ NS ns2.control-plane.oxide.internal -+ NS ns3.control-plane.oxide.internal -* name: _nameservice._tcp (records: 2 -> 3) -- SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -- SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -+ SRV port 5353 71f71743-8c73-43c6-b080-427ec28ef4c9.host.control-plane.oxide.internal -+ SRV port 5353 c821c39d-2b2c-4c55-8874-ac12315ba1e4.host.control-plane.oxide.internal -+ SRV port 5353 ea5b4030-b52f-44b2-8d70-45f15f987d01.host.control-plane.oxide.internal -* name: ns2 (records: 1 -> 1) -- AAAA fd00:1122:3344:3::1 -+ AAAA fd00:1122:3344:2::1 -+ name: ns3 (records: 1) -+ AAAA fd00:1122:3344:3::1 - unchanged names: 46 (records: 56) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest blueprint (c0d81ea6-909c-4efb-964e-beff67f6da0d) - -> inventory-generate -generated inventory collection b460bcc7-664d-4dff-92fb-f250def5537c from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 2, num_eligible: 0, num_ineligible: 6 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint 60b55d33-5fec-4277-9864-935197eaead7 based on parent blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d -planning report for blueprint 60b55d33-5fec-4277-9864-935197eaead7: -* 1 out-of-date zone updated in-place: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 5199c033-4cf9-4ab6-8ae7-566bd7606363 (crucible) -* 15 remaining out-of-date zones - - -> blueprint-diff latest -from: blueprint c0d81ea6-909c-4efb-964e-beff67f6da0d -to: blueprint 60b55d33-5fec-4277-9864-935197eaead7 - - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 6 -> 7): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset in service fd00:1122:3344:101::23 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 - install dataset in service fd00:1122:3344:101::25 - └─ + artifact: version 1.0.0 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: - DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest blueprint (60b55d33-5fec-4277-9864-935197eaead7) - -> inventory-generate -generated inventory collection f8212fb6-115e-4568-a05c-b241e2e8ffb9 from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 3, num_eligible: 0, num_ineligible: 5 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 based on parent blueprint 60b55d33-5fec-4277-9864-935197eaead7 -planning report for blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544: -* 1 out-of-date zone expunged: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone 6444f8a5-6465-4f0b-a549-1993c113569c (internal_ntp) -* 14 remaining out-of-date zones - - -> blueprint-diff latest -from: blueprint 60b55d33-5fec-4277-9864-935197eaead7 -to: blueprint aa13f40f-41ff-4b68-bee1-df2e1f805544 - - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 7 -> 8): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -* oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 - in service none none off - └─ + expunged - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 0.0.1 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 artifact: version 0.0.1 in service fd00:1122:3344:101::26 - crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 artifact: version 0.0.1 in service fd00:1122:3344:101::24 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d artifact: version 0.0.1 in service fd00:1122:3344:101::23 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c artifact: version 0.0.1 in service fd00:1122:3344:101::21 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 artifact: version 0.0.1 in service fd00:1122:3344:101::22 -* internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c artifact: version 0.0.1 - expunged ⏳ fd00:1122:3344:2::1 - └─ + expunged ✓ -+ internal_dns abd27551-4027-4084-8b52-13a575b035b4 artifact: version 1.0.0 in service fd00:1122:3344:2::1 ++ internal_dns abd27551-4027-4084-8b52-13a575b035b4 artifact: version 1.0.0 in service fd00:1122:3344:2::1 COCKROACHDB SETTINGS: @@ -5789,9 +5179,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 02078c95-3d58-4b7b-a03f-9b160361c50a based on parent blueprint 316ccd9e-5c53-46c3-a2e9-20c3867b7111 blueprint source: planner with report: @@ -6204,9 +5594,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec based on parent blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 blueprint source: planner with report: @@ -6494,155 +5884,22 @@ INFO skipping board for MGS-driven update (no update necessary), serial_number: INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update -generated blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 based on parent blueprint a2c6496d-98fc-444d-aa36-99508aa72367 -blueprint source: planner with report: -planning report: -* skipping noop zone image source check on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: all 9 zones are already from artifacts -* skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 7 zones are already from artifacts -* skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 8 zones are already from artifacts -* discretionary zones placed: - * crucible_pantry zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 from source artifact: version 1.0.0 -* zone updates waiting on discretionary zones -* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated - - - -> blueprint-diff latest -from: blueprint 880e2ffc-8187-4275-a2f3-1b36aa2f4482 -to: blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec - - MODIFIED SLEDS: - - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 12 -> 13): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-073979dd-3248-44a5-9fa1-cc72a140d682 in service - fake-vendor fake-model serial-c6d33b64-fb96-4129-bab1-7878a06a5f9b in service - fake-vendor fake-model serial-e4d937e1-6ddc-4eca-bb08-c1f73791e608 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crucible 7b4ce6bf-95bb-42fe-a4a0-dff31211ab88 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crucible ea8a11bf-a884-4c4f-8df0-3ef9b7aacf43 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crucible 50b029e3-96aa-41e5-bf39-023193a4355e in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 4847a96e-a267-4ae7-aa3d-805c1e77f81e expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/external_dns 44811c39-a4a2-4be3-85a6-954cf148e4b2 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns dedff151-ee94-4ead-a742-f973b39e21db in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/internal_dns ad41be71-6c15-4428-b510-20ceacde4fa6 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone 4617d206-4330-4dfa-b9f3-f63a3db834f9 in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone 793ac181-1b01-403c-850d-7f5c54bda6c9 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone 4f60b534-eaa3-40a1-b60f-bfdf147af478 in service none none off - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/zone/oxz_crucible_058fd5f9-60a8-4e11-9302-15172782e17d 02c56a30-7d97-406d-bd34-1eb437fd517d in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_5199c033-4cf9-4ab6-8ae7-566bd7606363 832fd140-d467-4bad-b5e9-63171634087c in service none none off - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/zone/oxz_crucible_dfac80b4-a887-430a-ae87-a4e065dba787 4d7e3e8e-06bd-414c-a468-779e056a9b75 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_ba4994a8-23f9-4b1a-a84f-a08d74591389 42430c80-7836-4191-a4f6-bcee749010fe expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_803bfb63-c246-41db-b0da-d3b87ddfc63d 43931274-7fe8-4077-825d-dff2bc8efa58 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_external_dns_e14f91b0-0c41-48a0-919d-e5078d2b89b0 312286f1-e378-464d-97cb-6fa06ba2dab7 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_427ec88f-f467-42fa-9bbb-66a91a36103c 1bca7f71-5e42-4749-91ec-fa40793a3a9a expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_internal_dns_71f71743-8c73-43c6-b080-427ec28ef4c9 d8368353-9ae6-4ed0-99ad-21783f514ba6 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_nexus_0c71b3b2-6ceb-4e8f-b020-b08675e83038 a4c3032e-21fa-4d4a-b040-a7e3c572cf3c in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_6444f8a5-6465-4f0b-a549-1993c113569c 3ac089c9-9dec-465b-863a-188e80d71fb4 expunged none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_ntp_cc6fdaf4-0195-4cef-950d-7bacd7059787 5d6116c2-ade3-4246-8d02-5591d2622c10 in service none none off - oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/debug 248c6c10-1ac6-45de-bb55-ede36ca56bbd in service 100 GiB none gzip-9 - oxp_c6d33b64-fb96-4129-bab1-7878a06a5f9b/crypt/debug cdf3684f-a6cf-4449-b9ec-e696b2c663e2 in service 100 GiB none gzip-9 - oxp_e4d937e1-6ddc-4eca-bb08-c1f73791e608/crypt/debug 686c19cf-a0d7-45f6-866f-c564612b2664 in service 100 GiB none gzip-9 -+ oxp_073979dd-3248-44a5-9fa1-cc72a140d682/crypt/zone/oxz_crucible_pantry_26bdd109-c842-43a9-95cb-15aba9b0832b 16a8c618-d062-4bde-8ca4-301b5f14ccf2 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 058fd5f9-60a8-4e11-9302-15172782e17d artifact: version 1.0.0 in service fd00:1122:3344:101::27 - crucible 5199c033-4cf9-4ab6-8ae7-566bd7606363 artifact: version 1.0.0 in service fd00:1122:3344:101::25 - crucible dfac80b4-a887-430a-ae87-a4e065dba787 install dataset in service fd00:1122:3344:101::26 - external_dns 803bfb63-c246-41db-b0da-d3b87ddfc63d install dataset expunged ✓ fd00:1122:3344:101::23 - external_dns e14f91b0-0c41-48a0-919d-e5078d2b89b0 artifact: version 1.0.0 in service fd00:1122:3344:101::29 - internal_dns 427ec88f-f467-42fa-9bbb-66a91a36103c install dataset expunged ✓ fd00:1122:3344:2::1 - internal_dns 71f71743-8c73-43c6-b080-427ec28ef4c9 artifact: version 1.0.0 in service fd00:1122:3344:2::1 - internal_ntp 6444f8a5-6465-4f0b-a549-1993c113569c install dataset expunged ✓ fd00:1122:3344:101::21 - internal_ntp cc6fdaf4-0195-4cef-950d-7bacd7059787 artifact: version 1.0.0 in service fd00:1122:3344:101::28 - nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 -* crucible_pantry ba4994a8-23f9-4b1a-a84f-a08d74591389 install dataset - expunged ⏳ fd00:1122:3344:101::24 - └─ + expunged ✓ -+ crucible_pantry 26bdd109-c842-43a9-95cb-15aba9b0832b artifact: version 1.0.0 in service fd00:1122:3344:101::2a - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: -* DNS zone: "control-plane.oxide.internal": -+ name: 26bdd109-c842-43a9-95cb-15aba9b0832b.host (records: 1) -+ AAAA fd00:1122:3344:101::2a -* name: _crucible-pantry._tcp (records: 2 -> 3) -- SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -- SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal -+ SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 75b220ba-a0f4-4872-8202-dc7c87f062d0.host.control-plane.oxide.internal - unchanged names: 49 (records: 61) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron-config latest -set sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 omicron config from latest blueprint (c4a20bcb-1a71-4e88-97b4-36d16f55daec) - -> inventory-generate -generated inventory collection 2d608b8f-bf88-4707-ac27-6be62f3d5146 from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 6, num_eligible: 0, num_ineligible: 2 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint a2c6496d-98fc-444d-aa36-99508aa72367 based on parent blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec -planning report for blueprint a2c6496d-98fc-444d-aa36-99508aa72367: -* 1 out-of-date zone updated in-place: - * sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, zone dfac80b4-a887-430a-ae87-a4e065dba787 (crucible) -* 11 remaining out-of-date zones +generated blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 based on parent blueprint a2c6496d-98fc-444d-aa36-99508aa72367 +blueprint source: planner with report: +planning report: +* skipping noop zone image source check on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c: all 9 zones are already from artifacts +* skipping noop zone image source check on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6: all 7 zones are already from artifacts +* skipping noop zone image source check on sled d81c6a84-79b8-4958-ae41-ea46c9b19763: all 8 zones are already from artifacts +* discretionary zones placed: + * crucible_pantry zone on sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 from source artifact: version 1.0.0 +* zone updates waiting on discretionary zones +* waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated + > blueprint-diff latest -from: blueprint c4a20bcb-1a71-4e88-97b4-36d16f55daec -to: blueprint a2c6496d-98fc-444d-aa36-99508aa72367 +from: blueprint a2c6496d-98fc-444d-aa36-99508aa72367 +to: blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 MODIFIED SLEDS: @@ -6762,9 +6019,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 based on parent blueprint 6ed56354-5941-40d1-a06c-b0e940701d52 blueprint source: planner with report: @@ -7166,133 +6423,8 @@ planning report: > blueprint-diff latest -from: blueprint 9078c4ba-3a73-4b3f-ac2c-acb501f89cb2 -to: blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 - - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 5 -> 6): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 install dataset in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset in service fd00:1122:3344:3::1 - internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset in service fd00:1122:3344:103::21 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset - expunged ⏳ fd00:1122:3344:103::24 - └─ + expunged ✓ -+ crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _crucible-pantry._tcp (records: 2 -> 3) -- SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -- SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 26bdd109-c842-43a9-95cb-15aba9b0832b.host.control-plane.oxide.internal -+ SRV port 17000 698d1d82-0620-4978-93ac-0ba5d40f3da9.host.control-plane.oxide.internal -+ SRV port 17000 c88fcd7d-9509-470e-8c4f-3e6f09104cdc.host.control-plane.oxide.internal -+ name: c88fcd7d-9509-470e-8c4f-3e6f09104cdc.host (records: 1) -+ AAAA fd00:1122:3344:103::28 - unchanged names: 49 (records: 61) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest blueprint (8763abc1-8a42-4932-b5a7-33109e0e0152) - -> inventory-generate -generated inventory collection 897721fc-b087-41be-a566-809d59c8aeea from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 2, num_eligible: 0, num_ineligible: 6 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 based on parent blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 -planning report for blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88: -* 1 out-of-date zone updated in-place: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone 7c252b64-c5af-4ec1-989e-9a03f3b0f111 (crucible) -* 8 remaining out-of-date zones - - -> blueprint-diff latest -from: blueprint 8763abc1-8a42-4932-b5a7-33109e0e0152 -to: blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 +from: blueprint 2b89e0d7-f15b-4474-8ac4-85959ed1bc88 +to: blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 MODIFIED SLEDS: @@ -7404,9 +6536,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 59630e63-c953-4807-9e84-9e750a79f68e based on parent blueprint 7f6b7297-c2bc-4f67-b3c0-c8e555ebbdc4 blueprint source: planner with report: @@ -7964,138 +7096,8 @@ planning report: > blueprint-diff latest -from: blueprint e93650dc-b5ba-4ec7-8550-9171c1ada194 -to: blueprint 90650737-8142-47a6-9a48-a10efc487e57 - - MODIFIED SLEDS: - - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 10 -> 11): - - host phase 2 contents: - ------------------------------ - slot boot image source - ------------------------------ - A current contents - B artifact: version 1.0.0 - - - physical disks: - ------------------------------------------------------------------------------------ - vendor model serial disposition - ------------------------------------------------------------------------------------ - fake-vendor fake-model serial-18b20749-0748-4105-bb10-7b13cfc776e2 in service - fake-vendor fake-model serial-30c16fe4-4229-49d0-ab01-3138f2c7dff2 in service - fake-vendor fake-model serial-4930954e-9ac7-4453-b63f-5ab97c389a99 in service - - - datasets: - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - dataset name dataset id disposition quota reservation compression - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crucible 7ea73f80-c4e0-450a-92dc-8397ce2af14f in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crucible 6f04dd20-5e2c-4fa8-8430-a886470ed140 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crucible a50cd13a-5749-4e79-bb8b-19229500a8b3 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/external_dns 96ae8389-3027-4260-9374-e0f6ce851de2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns 1cb0a47a-59ac-4892-8e92-cf87b4290f96 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/internal_dns bdc35fc8-8541-4dfc-b9fa-db05eceb5c55 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone 45cd9687-20be-4247-b62a-dfdacf324929 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone e009d8b8-4695-4322-b53f-f03f2744aef7 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone 252ac39f-b9e2-4697-8c07-3a833115d704 in service none none off - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/zone/oxz_crucible_694bd14f-cb24-4be4-bb19-876e79cda2c8 3443a368-199e-4d26-b59f-3f2bbd507761 in service none none off - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/zone/oxz_crucible_7c252b64-c5af-4ec1-989e-9a03f3b0f111 429da94b-19f7-48bd-98e9-47842863ba7b in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_f55647d4-5500-4ad3-893a-df45bd50d622 50ea8c15-c4c0-4403-a490-d14b3405dfc2 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_75b220ba-a0f4-4872-8202-dc7c87f062d0 54bbadaf-ec04-41a2-a62f-f5ac5bf321be expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_crucible_pantry_c88fcd7d-9509-470e-8c4f-3e6f09104cdc fed6b06e-ad3f-41c2-b7cd-e783462ec58c in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_external_dns_f6ec9c67-946a-4da3-98d5-581f72ce8bf0 090bd88d-0a43-4040-a832-b13ae721f74f in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_7fbd0103-d7f8-48a5-b95e-29bf812cac1f 49336223-f6df-4fe7-bd9f-95123c5622a9 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_internal_dns_ea5b4030-b52f-44b2-8d70-45f15f987d01 b1deff4b-51df-4a37-9043-afbd7c70a1cb expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_nexus_3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 4da74a5b-6911-4cca-b624-b90c65530117 in service none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_f10a4fb9-759f-4a65-b25e-5794ad2d07d8 c65a9c1c-36dc-4ddb-8aac-ec3be8dbb209 expunged none none off - oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/debug 7a6a2058-ea78-49de-9730-cce5e28b4cfb in service 100 GiB none gzip-9 - oxp_30c16fe4-4229-49d0-ab01-3138f2c7dff2/crypt/debug 41071985-1dfd-4ce5-8bc2-897161a8bce4 in service 100 GiB none gzip-9 - oxp_4930954e-9ac7-4453-b63f-5ab97c389a99/crypt/debug 21fd4f3a-ec31-469b-87b1-087c343a2422 in service 100 GiB none gzip-9 -+ oxp_18b20749-0748-4105-bb10-7b13cfc776e2/crypt/zone/oxz_ntp_d5fd048a-8786-42d3-938e-820eae95d7f4 c22b409e-662b-48a9-ac70-29f7487aa6b1 in service none none off - - - omicron zones: - -------------------------------------------------------------------------------------------------------------------------- - zone type zone id image source disposition underlay IP - -------------------------------------------------------------------------------------------------------------------------- - crucible 694bd14f-cb24-4be4-bb19-876e79cda2c8 artifact: version 1.0.0 in service fd00:1122:3344:103::26 - crucible 7c252b64-c5af-4ec1-989e-9a03f3b0f111 artifact: version 1.0.0 in service fd00:1122:3344:103::27 - crucible f55647d4-5500-4ad3-893a-df45bd50d622 install dataset in service fd00:1122:3344:103::25 - crucible_pantry 75b220ba-a0f4-4872-8202-dc7c87f062d0 install dataset expunged ✓ fd00:1122:3344:103::24 - crucible_pantry c88fcd7d-9509-470e-8c4f-3e6f09104cdc artifact: version 1.0.0 in service fd00:1122:3344:103::28 - external_dns f6ec9c67-946a-4da3-98d5-581f72ce8bf0 install dataset in service fd00:1122:3344:103::23 - internal_dns 7fbd0103-d7f8-48a5-b95e-29bf812cac1f artifact: version 1.0.0 in service fd00:1122:3344:3::1 - internal_dns ea5b4030-b52f-44b2-8d70-45f15f987d01 install dataset expunged ✓ fd00:1122:3344:3::1 - nexus 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6 install dataset in service fd00:1122:3344:103::22 -* internal_ntp f10a4fb9-759f-4a65-b25e-5794ad2d07d8 install dataset - expunged ⏳ fd00:1122:3344:103::21 - └─ + expunged ✓ -+ internal_ntp d5fd048a-8786-42d3-938e-820eae95d7f4 artifact: version 1.0.0 in service fd00:1122:3344:103::29 - - - COCKROACHDB SETTINGS: - state fingerprint::::::::::::::::: (none) (unchanged) - cluster.preserve_downgrade_option: (do not modify) (unchanged) - - METADATA: - internal DNS version::: 1 (unchanged) - external DNS version::: 1 (unchanged) - target release min gen: 1 (unchanged) - nexus gen:::::::::::::: 1 (unchanged) - - OXIMETER SETTINGS: - generation: 1 (unchanged) - read from:: SingleNode (unchanged) - - -internal DNS: -* DNS zone: "control-plane.oxide.internal": -* name: _internal-ntp._tcp (records: 2 -> 3) -- SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -- SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ SRV port 123 cc6fdaf4-0195-4cef-950d-7bacd7059787.host.control-plane.oxide.internal -+ SRV port 123 d5fd048a-8786-42d3-938e-820eae95d7f4.host.control-plane.oxide.internal -+ SRV port 123 f83ade6d-9ab1-4679-813b-b9457e039c0b.host.control-plane.oxide.internal -+ name: d5fd048a-8786-42d3-938e-820eae95d7f4.host (records: 1) -+ AAAA fd00:1122:3344:103::29 - unchanged names: 49 (records: 61) - -external DNS: - DNS zone: "oxide.example" (unchanged) - unchanged names: 5 (records: 9) - - - - -> sled-set d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron-config latest -set sled d81c6a84-79b8-4958-ae41-ea46c9b19763 omicron config from latest blueprint (90650737-8142-47a6-9a48-a10efc487e57) - -> inventory-generate -generated inventory collection a4dab274-0fff-47fa-bc22-b98d11ec54d2 from configured sleds - -> blueprint-plan latest latest -INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 8, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, num_total: 8, num_already_artifact: 7, num_eligible: 0, num_ineligible: 1 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 5, num_eligible: 0, num_ineligible: 3 -INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 -INFO ran out of boards for MGS-driven update -generated blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa based on parent blueprint 90650737-8142-47a6-9a48-a10efc487e57 -planning report for blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa: -* 1 out-of-date zone updated in-place: - * sled d81c6a84-79b8-4958-ae41-ea46c9b19763, zone f55647d4-5500-4ad3-893a-df45bd50d622 (crucible) -* 5 remaining out-of-date zones - - -> blueprint-diff latest -from: blueprint 90650737-8142-47a6-9a48-a10efc487e57 -to: blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa +from: blueprint 2182613d-dc9f-41eb-9c6a-d33801849caa +to: blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece MODIFIED SLEDS: @@ -8212,9 +7214,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 810ea95a-4730-43dd-867e-1984aeb9d873 based on parent blueprint e8b088a8-7da0-480b-a2dc-75ffef068ece blueprint source: planner with report: @@ -8635,9 +7637,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint d69e1109-06be-4469-8876-4292dc7885d7 based on parent blueprint 0a8d2f87-1d3e-4296-ba07-108940a7a57e blueprint source: planner with report: @@ -8958,9 +7960,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 12f19448-6a10-4b4b-ae19-a8c94a566097 based on parent blueprint d69e1109-06be-4469-8876-4292dc7885d7 blueprint source: planner with report: @@ -9015,9 +8017,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 4713f6c4-e8ba-4a28-87a0-df75ebf7b8b6 based on parent blueprint 12f19448-6a10-4b4b-ae19-a8c94a566097 blueprint source: planner with report: @@ -9038,9 +8040,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 6d830d26-547e-492b-adfe-c5c4ad9c3751 based on parent blueprint 4713f6c4-e8ba-4a28-87a0-df75ebf7b8b6 blueprint source: planner with report: @@ -9061,9 +8063,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 8e0cc787-e068-4a45-97ed-21029cbe4ddf based on parent blueprint 6d830d26-547e-492b-adfe-c5c4ad9c3751 blueprint source: planner with report: @@ -9087,9 +8089,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 8, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint e31c9054-8549-4c68-acf9-a01f68d1fc9b based on parent blueprint 8e0cc787-e068-4a45-97ed-21029cbe4ddf blueprint source: planner with report: From 7d9d1a02e3dd4e88c791376ecb73a85448fa1c7c Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 18:31:16 +1200 Subject: [PATCH 47/55] =?UTF-8?q?tests=20are=20passing=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + nexus/reconfigurator/planning/Cargo.toml | 1 + .../planning/src/mgs_updates/mod.rs | 2 +- nexus/reconfigurator/planning/src/planner.rs | 60 ++++++++++--------- 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ffba7cd297..9d894c8b7e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6912,6 +6912,7 @@ dependencies = [ "expectorate", "gateway-client", "gateway-types", + "hex-literal", "id-map", "iddqd", "illumos-utils", diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml index 0fd3625db71..ed54003c58f 100644 --- a/nexus/reconfigurator/planning/Cargo.toml +++ b/nexus/reconfigurator/planning/Cargo.toml @@ -57,6 +57,7 @@ omicron-workspace-hack.workspace = true [dev-dependencies] dropshot.workspace = true expectorate.workspace = true +hex-literal.workspace = true maplit.workspace = true omicron-common = { workspace = true, features = ["testing"] } omicron-test-utils.workspace = true diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index 389a01c0222..fc52552bb10 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -584,7 +584,7 @@ fn try_make_update( // component is already at the expected version } Err(e) => { - // There was a failure, skip the update and record it + // There was a failure, skip the update and record it pending_actions.add_blocked_update(BlockedMgsUpdate { baseboard_id: baseboard_id.clone(), component, diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index e54a6c85544..d698e5a52f5 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -6378,7 +6378,9 @@ pub(crate) mod test { version: version.clone(), kind: ArtifactKind::HOST_PHASE_2, }, - hash: ArtifactHash([0x0a; 32]), + hash: ArtifactHash(hex_literal::hex!( + "7cd830e1682d50620de0f5c24b8cca15937eb10d2a415ade6ad28c0d314408eb" + )), size: 0, board: None, sign: None, @@ -6397,7 +6399,7 @@ pub(crate) mod test { TufArtifactMeta { id: ArtifactId { name: sp_sim::SIM_ROT_BOARD.to_string(), - version: ArtifactVersion::new("0.0.2").unwrap(), + version: ArtifactVersion::new("0.0.1").unwrap(), kind: ArtifactKind::GIMLET_ROT_IMAGE_B, }, hash: ArtifactHash([0; 32]), @@ -6536,24 +6538,8 @@ pub(crate) mod test { }; } - // Request 3 Nexus zones. The blueprint will show changes in each sled - // for BlueprintHostPhase2DesiredSlotsDiff in a simulated system even if - // we are not performing an update for the Host OS because it's going - // from `CurrentContents` to: - // `Artifact { - // version: Available { - // version: ArtifactVersion( - // "1.0.0-freeform", - // ), - // }, - // hash: ArtifactHash( - // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", - // ) - // } ` - // We need each sled to look identical in the blueprint, so we add a - // nexus zone to each sled. let expected_new_nexus_zones = - input_builder.policy_mut().target_nexus_zone_count + 3; + input_builder.policy_mut().target_nexus_zone_count; let expected_pantries = input_builder.policy_mut().target_crucible_pantry_zone_count; example.input = input_builder.build(); @@ -6827,26 +6813,26 @@ pub(crate) mod test { } // Everything's up-to-date in Kansas City! - let blueprint20 = parent; + let blueprint12 = parent; assert_eq!( - blueprint20 + blueprint12 .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .filter(|(_, z)| is_up_to_date_nexus(z)) .count(), - NEXUS_REDUNDANCY + 3, + NEXUS_REDUNDANCY, ); assert_eq!( - blueprint20 + blueprint12 .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .filter(|(_, z)| is_old_nexus(z)) .count(), 0, ); - update_collection_from_blueprint(&mut example, &blueprint20); + update_collection_from_blueprint(&mut example, &blueprint12); assert_planning_makes_no_changes( &logctx.log, - &blueprint20, + &blueprint12, &example.input, &example.collection, TEST_NAME, @@ -6867,6 +6853,8 @@ pub(crate) mod test { &logctx.log, rng.next_system_rng(), ) + .with_target_release_0_0_1() + .expect("set target release to 0.0.1") .build(); verify_blueprint(&blueprint); @@ -6910,12 +6898,18 @@ pub(crate) mod test { TEST_NAME, ); + // All zones should be sourced from the initial 0.0.1 target release by + // default. + eprintln!("{}", blueprint.display()); assert!( blueprint .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .all(|(_, z)| matches!( &z.image_source, - BlueprintZoneImageSource::InstallDataset + BlueprintZoneImageSource::Artifact { version, hash: _ } + if version == &BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("0.0.1") + } )) ); @@ -6979,7 +6973,10 @@ pub(crate) mod test { zone.zone_type.is_cockroach() && matches!( &zone.image_source, - BlueprintZoneImageSource::InstallDataset + BlueprintZoneImageSource::Artifact { version, hash: _ } + if version == &BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("0.0.1") + } ) }; let is_up_to_date_cockroach = |zone: &BlueprintZoneConfig| -> bool { @@ -8669,6 +8666,8 @@ pub(crate) mod test { rng.next_system_rng(), ) .nexus_count(3) + .with_target_release_0_0_1() + .expect("set target release to 0.0.1") .build(); verify_blueprint(&blueprint); @@ -8691,8 +8690,11 @@ pub(crate) mod test { .blueprint .all_omicron_zones(BlueprintZoneDisposition::is_in_service) .all(|(_, z)| matches!( - z.image_source, - BlueprintZoneImageSource::InstallDataset + &z.image_source, + BlueprintZoneImageSource::Artifact { version, hash: _ } + if version == &BlueprintArtifactVersion::Available { + version: ArtifactVersion::new_const("0.0.1") + } )) ); From 0e367a966df192384632dd33c28473ab1d4a493b Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 18:37:48 +1200 Subject: [PATCH 48/55] remove unnecessary blueprint updates --- nexus/reconfigurator/planning/src/planner.rs | 140 +------------------ 1 file changed, 5 insertions(+), 135 deletions(-) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index d698e5a52f5..946af2409a1 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -6996,48 +6996,6 @@ pub(crate) mod test { result }; - // First we update the blueprint once for each sled. The blueprint will - // show changes in each sled for BlueprintHostPhase2DesiredSlotsDiff in - // a simulated system even if we are not performing an update for the - // Host OS because it's going from `CurrentContents` to: - // `Artifact { - // version: Available { - // version: ArtifactVersion( - // "1.0.0-freeform", - // ), - // }, - // hash: ArtifactHash( - // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", - // ) - // } ` - let mut parent = blueprint; - for i in 2..=4 { - update_collection_from_blueprint(&mut example, &parent); - - let blueprint_name = format!("blueprint{i}"); - let blueprint = Planner::new_based_on( - log.clone(), - &parent, - &example.input, - &blueprint_name, - &example.collection, - PlannerRng::from_seed((TEST_NAME, &blueprint_name)), - ) - .expect("can't create planner") - .plan() - .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); - - { - let summary = blueprint.diff_since_blueprint(&parent); - for sled in summary.diff.sleds.modified_values_diff() { - assert!(sled.zones.added.is_empty()); - assert!(sled.zones.removed.is_empty()); - } - } - - parent = blueprint; - } - let mut blueprint = parent; // If we have missing info in our inventory, the // planner will not update any Cockroach zones. @@ -7498,55 +7456,12 @@ pub(crate) mod test { collection.ntp_timesync = ntp_timesync; }; - // First we update the blueprint once for each sled. The blueprint will - // show changes in each sled for BlueprintHostPhase2DesiredSlotsDiff in - // a simulated system even if we are not performing an update for the - // Host OS because it's going from `CurrentContents` to: - // `Artifact { - // version: Available { - // version: ArtifactVersion( - // "1.0.0-freeform", - // ), - // }, - // hash: ArtifactHash( - // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", - // ) - // } ` - let mut parent = blueprint; - for i in 2..=4 { - update_collection_from_blueprint(&mut example, &parent); - - let blueprint_name = format!("blueprint{i}"); - let blueprint = Planner::new_based_on( - log.clone(), - &parent, - &example.input, - &blueprint_name, - &example.collection, - PlannerRng::from_seed((TEST_NAME, &blueprint_name)), - ) - .expect("can't create planner") - .plan() - .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); - - { - let summary = blueprint.diff_since_blueprint(&parent); - for sled in summary.diff.sleds.modified_values_diff() { - assert!(sled.zones.added.is_empty()); - assert!(sled.zones.removed.is_empty()); - } - } - - parent = blueprint; - } - let blueprint4 = parent; - // If we have missing info in our inventory, the // planner will not update any boundary NTP zones. example.collection.ntp_timesync = IdOrdMap::new(); assert_planning_makes_no_changes( &log, - &blueprint4, + &blueprint, &example.input, &example.collection, TEST_NAME, @@ -7569,7 +7484,7 @@ pub(crate) mod test { example.collection.ntp_timesync.remove(&boundary_ntp_zone); assert_planning_makes_no_changes( &log, - &blueprint4, + &blueprint, &example.input, &example.collection, TEST_NAME, @@ -7597,7 +7512,7 @@ pub(crate) mod test { .synced = false; assert_planning_makes_no_changes( &log, - &blueprint4, + &blueprint, &example.input, &example.collection, TEST_NAME, @@ -7617,7 +7532,7 @@ pub(crate) mod test { // let new_blueprint = Planner::new_based_on( log.clone(), - &blueprint4, + &blueprint, &example.input, "test_blueprint_expunge_old_boundary_ntp", &example.collection, @@ -7627,7 +7542,7 @@ pub(crate) mod test { .plan() .expect("plan for trivial TUF repo"); { - let summary = new_blueprint.diff_since_blueprint(&blueprint4); + let summary = new_blueprint.diff_since_blueprint(&blueprint); eprintln!( "diff between blueprints (should be expunging \ boundary NTP using 0.0.1 artifact):\n{}", @@ -7955,51 +7870,6 @@ pub(crate) mod test { result }; - // First we update the blueprint once for each sled. The blueprint will - // show changes in each sled for BlueprintHostPhase2DesiredSlotsDiff in - // a simulated system even if we are not performing an update for the - // Host OS because it's going from `CurrentContents` to: - // `Artifact { - // version: Available { - // version: ArtifactVersion( - // "1.0.0-freeform", - // ), - // }, - // hash: ArtifactHash( - // "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a", - // ) - // } ` - let mut parent = blueprint; - for i in 2..=4 { - update_collection_from_blueprint(&mut example, &parent); - - let blueprint_name = format!("blueprint{i}"); - // We make sure we don't update any internal DNS zones yet - example.collection.internal_dns_generation_status = IdOrdMap::new(); - let blueprint = Planner::new_based_on( - log.clone(), - &parent, - &example.input, - &blueprint_name, - &example.collection, - PlannerRng::from_seed((TEST_NAME, &blueprint_name)), - ) - .expect("can't create planner") - .plan() - .unwrap_or_else(|_| panic!("can't re-plan after {i} iterations")); - - { - let summary = blueprint.diff_since_blueprint(&parent); - for sled in summary.diff.sleds.modified_values_diff() { - assert!(sled.zones.added.is_empty()); - assert!(sled.zones.removed.is_empty()); - } - } - - parent = blueprint; - } - let mut blueprint = parent; - // If we have missing info in our inventory, the // planner will not update any Internal DNS zones. example.collection.internal_dns_generation_status = IdOrdMap::new(); From 7e4e36ea0feb895cf2c11ef4ba809c23f3e42eec Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 18:44:51 +1200 Subject: [PATCH 49/55] generate openapi spec --- openapi/nexus-internal.json | 80 -------- openapi/nexus-lockstep.json | 353 +++++++++++++++++++++++++++++++++++- 2 files changed, 352 insertions(+), 81 deletions(-) diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index f8c1c8c7720..7d1b86efd8e 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -3756,33 +3756,6 @@ "minLength": 5, "maxLength": 17 }, - "MgsDrivenUpdateStatus": { - "type": "object", - "properties": { - "baseboard_description": { - "type": "string" - }, - "host_os_phase_1": { - "$ref": "#/components/schemas/HostPhase1Status" - }, - "rot": { - "$ref": "#/components/schemas/RotStatus" - }, - "rot_bootloader": { - "$ref": "#/components/schemas/RotBootloaderStatus" - }, - "sp": { - "$ref": "#/components/schemas/SpStatus" - } - }, - "required": [ - "baseboard_description", - "host_os_phase_1", - "rot", - "rot_bootloader", - "sp" - ] - }, "MgsUpdateComponent": { "type": "string", "enum": [ @@ -3792,59 +3765,6 @@ "host_os" ] }, - "MgsUpdateDriverStatus": { - "description": "Status of ongoing update attempts, recently completed attempts, and update requests that are waiting for retry.", - "type": "object", - "properties": { - "in_progress": { - "title": "IdOrdMap", - "x-rust-type": { - "crate": "iddqd", - "parameters": [ - { - "$ref": "#/components/schemas/InProgressUpdateStatus" - } - ], - "path": "iddqd::IdOrdMap", - "version": "*" - }, - "type": "array", - "items": { - "$ref": "#/components/schemas/InProgressUpdateStatus" - }, - "uniqueItems": true - }, - "recent": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CompletedAttempt" - } - }, - "waiting": { - "title": "IdOrdMap", - "x-rust-type": { - "crate": "iddqd", - "parameters": [ - { - "$ref": "#/components/schemas/WaitingStatus" - } - ], - "path": "iddqd::IdOrdMap", - "version": "*" - }, - "type": "array", - "items": { - "$ref": "#/components/schemas/WaitingStatus" - }, - "uniqueItems": true - } - }, - "required": [ - "in_progress", - "recent", - "waiting" - ] - }, "MigrationRuntimeState": { "description": "An update from a sled regarding the state of a migration, indicating the role of the VMM whose migration state was updated.", "type": "object", diff --git a/openapi/nexus-lockstep.json b/openapi/nexus-lockstep.json index 8dbc8b3c484..3cee722a425 100644 --- a/openapi/nexus-lockstep.json +++ b/openapi/nexus-lockstep.json @@ -1476,6 +1476,40 @@ "serial_number" ] }, + "BlockedMgsUpdate": { + "type": "object", + "properties": { + "baseboard_id": { + "description": "id of the baseboard that we attempted to update", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "component": { + "description": "type of SP component that we attempted to update", + "allOf": [ + { + "$ref": "#/components/schemas/MgsUpdateComponent" + } + ] + }, + "reason": { + "description": "reason why the update failed", + "allOf": [ + { + "$ref": "#/components/schemas/FailedMgsUpdateReason" + } + ] + } + }, + "required": [ + "baseboard_id", + "component", + "reason" + ] + }, "Blueprint": { "description": "Describes a complete set of software and configuration for the system", "type": "object", @@ -2696,6 +2730,18 @@ "format": "uint64", "minimum": 0 }, + "CabooseWhich": { + "description": "Describes which caboose this is (which component, which slot)", + "type": "string", + "enum": [ + "sp_slot0", + "sp_slot1", + "rot_slot_a", + "rot_slot_b", + "stage0", + "stage0_next" + ] + }, "ClickhouseClusterConfig": { "description": "Global configuration for all clickhouse servers (replicas) and keepers", "type": "object", @@ -3415,6 +3461,280 @@ } ] }, + "FailedMgsUpdateReason": { + "description": "Describes the reason why an SP component failed to update", + "oneOf": [ + { + "description": "The active host phase 1 slot does not match the boot disk", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "active_host_phase1_slot_boot_disk_mismatch" + ] + }, + "value": { + "$ref": "#/components/schemas/M2Slot" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The active host phase 1 hash was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "active_host_phase1_hash_not_in_inventory" + ] + }, + "value": { + "$ref": "#/components/schemas/M2Slot" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The active host phase 1 slot was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "active_host_phase1_slot_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The component's caboose was missing a value for \"sign\"", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "caboose_missing_sign" + ] + }, + "value": { + "$ref": "#/components/schemas/CabooseWhich" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The component's caboose was not found in the inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "caboose_not_in_inventory" + ] + }, + "value": { + "$ref": "#/components/schemas/CabooseWhich" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The version in the caboose or artifact was not able to be parsed", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "failed_version_parse" + ] + }, + "value": { + "type": "object", + "properties": { + "caboose": { + "$ref": "#/components/schemas/CabooseWhich" + }, + "err": { + "type": "string" + } + }, + "required": [ + "caboose", + "err" + ] + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The inactive host phase 1 hash was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "inactive_host_phase1_hash_not_in_inventory" + ] + }, + "value": { + "$ref": "#/components/schemas/M2Slot" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Last reconciliation details were not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "last_reconciliation_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "No artifact with the required conditions for the component was found", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "no_matching_artifact_found" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "RoT state was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "rot_state_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Sled agent info was not found in inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "sled_agent_info_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The component's corresponding SP was not found in the inventory", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "sp_not_in_inventory" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Too many artifacts with the required conditions for the component were found", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "too_many_matching_artifacts" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "The sled agent reported an error determining the boot disk", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unable_to_determine_boot_disk" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "The sled agent reported an error retrieving boot disk phase 2 image details", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unable_to_retrieve_boot_disk_phase2_image" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "Generation": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "type": "integer", @@ -4037,6 +4357,15 @@ "sp" ] }, + "MgsUpdateComponent": { + "type": "string", + "enum": [ + "sp", + "rot", + "rot_bootloader", + "host_os" + ] + }, "MgsUpdateDriverStatus": { "description": "Status of ongoing update attempts, recently completed attempts, and update requests that are waiting for retry.", "type": "object", @@ -5310,11 +5639,18 @@ "PlanningMgsUpdatesStepReport": { "type": "object", "properties": { + "blocked_mgs_updates": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BlockedMgsUpdate" + } + }, "pending_mgs_updates": { "$ref": "#/components/schemas/PendingMgsUpdates" } }, "required": [ + "blocked_mgs_updates", "pending_mgs_updates" ] }, @@ -7088,7 +7424,7 @@ ] }, { - "description": "Waiting on updates to RoT / SP / Host OS / etc.", + "description": "Waiting on updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { @@ -7102,6 +7438,21 @@ "type" ] }, + { + "description": "Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "blocked_mgs_updates" + ] + } + }, + "required": [ + "type" + ] + }, { "description": "Waiting on the same set of blockers zone adds are waiting on.", "type": "object", From f921c50a0110faa53e80863020172925f5bb8fcd Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 18:50:20 +1200 Subject: [PATCH 50/55] fmt --- nexus/reconfigurator/planning/src/planner.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 946af2409a1..a71f90ceb24 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -6996,7 +6996,6 @@ pub(crate) mod test { result }; - // If we have missing info in our inventory, the // planner will not update any Cockroach zones. example.collection.cockroach_status = BTreeMap::new(); From cb197db98eed6cec392d8265e4e18f4408b7f0f6 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 19:06:14 +1200 Subject: [PATCH 51/55] add the todos --- nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs | 2 ++ nexus/reconfigurator/planning/src/mgs_updates/rot.rs | 2 ++ .../reconfigurator/planning/src/mgs_updates/rot_bootloader.rs | 2 ++ nexus/reconfigurator/planning/src/mgs_updates/sp.rs | 2 ++ nexus/types/src/deployment/planning_report.rs | 4 ++++ 5 files changed, 12 insertions(+) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index ff6db6f2101..2a9cb415386 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -273,6 +273,8 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, +// TODO-K: Instead of this convoluted return type use an enum as suggested in +// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result< Option<(PendingMgsUpdate, PendingHostPhase2Changes)>, FailedMgsUpdateReason, diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index 6e2ac30a5ba..d4c45ba82b1 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -107,6 +107,8 @@ pub fn try_make_update_rot( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, +// TODO-K: Like the Host OS, use an enum here as the return type as suggested in +// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { return Err(FailedMgsUpdateReason::SpNotInInventory); diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index 41367484e9e..08bcb01d360 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -62,6 +62,8 @@ pub fn try_make_update_rot_bootloader( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, +// TODO-K: Like the Host OS, use an enum here as the return type as suggested in +// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { return Err(FailedMgsUpdateReason::SpNotInInventory); diff --git a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs index c9ecd6c13fb..ae94e04c7f8 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs @@ -61,6 +61,8 @@ pub fn try_make_update_sp( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, +// TODO-K: Like the Host OS, use an enum here as the return type as suggested in +// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { return Err(FailedMgsUpdateReason::SpNotInInventory); diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index e536a3bd530..defd3e827bf 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -486,6 +486,10 @@ impl PlanningMupdateOverrideStepReport { )] #[serde(rename_all = "snake_case")] #[serde(tag = "type", content = "value")] +// TODO-K: Separate into enums for each component as suggested in +// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372863166 +// and including more detailed information as suggested in +// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372842378 pub enum FailedMgsUpdateReason { /// The active host phase 1 slot does not match the boot disk #[error("active phase 1 slot {0:?} does not match boot disk")] From 01391ede26af0a3b716ea5f183ae475c7e3b1f3c Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 24 Sep 2025 19:10:35 +1200 Subject: [PATCH 52/55] fmt --- nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs | 4 ++-- nexus/reconfigurator/planning/src/mgs_updates/rot.rs | 4 ++-- .../reconfigurator/planning/src/mgs_updates/rot_bootloader.rs | 4 ++-- nexus/reconfigurator/planning/src/mgs_updates/sp.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs index 2a9cb415386..330ee478e6a 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/host_phase_1.rs @@ -273,8 +273,8 @@ pub(super) fn try_make_update( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -// TODO-K: Instead of this convoluted return type use an enum as suggested in -// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 + // TODO-K: Instead of this convoluted return type use an enum as suggested in + // https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result< Option<(PendingMgsUpdate, PendingHostPhase2Changes)>, FailedMgsUpdateReason, diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs index d4c45ba82b1..450a312059b 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot.rs @@ -107,8 +107,8 @@ pub fn try_make_update_rot( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -// TODO-K: Like the Host OS, use an enum here as the return type as suggested in -// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 + // TODO-K: Like the Host OS, use an enum here as the return type as suggested in + // https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { return Err(FailedMgsUpdateReason::SpNotInInventory); diff --git a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs index 08bcb01d360..db332d93cc3 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/rot_bootloader.rs @@ -62,8 +62,8 @@ pub fn try_make_update_rot_bootloader( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -// TODO-K: Like the Host OS, use an enum here as the return type as suggested in -// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 + // TODO-K: Like the Host OS, use an enum here as the return type as suggested in + // https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { return Err(FailedMgsUpdateReason::SpNotInInventory); diff --git a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs index ae94e04c7f8..4694186f096 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/sp.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/sp.rs @@ -61,8 +61,8 @@ pub fn try_make_update_sp( baseboard_id: &Arc, inventory: &Collection, current_artifacts: &TufRepoDescription, -// TODO-K: Like the Host OS, use an enum here as the return type as suggested in -// https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 + // TODO-K: Like the Host OS, use an enum here as the return type as suggested in + // https://github.com/oxidecomputer/omicron/pull/9001#discussion_r2372837627 ) -> Result, FailedMgsUpdateReason> { let Some(sp_info) = inventory.sps.get(baseboard_id) else { return Err(FailedMgsUpdateReason::SpNotInInventory); From 52d399f5ab6976f20e5a925a6f7ad64c443e5d43 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 25 Sep 2025 09:56:06 +1200 Subject: [PATCH 53/55] Address comments --- .../planning/src/mgs_updates/mod.rs | 121 +++++++----------- 1 file changed, 44 insertions(+), 77 deletions(-) diff --git a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs index fc52552bb10..fe60c43a074 100644 --- a/nexus/reconfigurator/planning/src/mgs_updates/mod.rs +++ b/nexus/reconfigurator/planning/src/mgs_updates/mod.rs @@ -567,74 +567,27 @@ fn try_make_update( MgsUpdateComponent::Sp, MgsUpdateComponent::HostOs, ] { - match component { + let update_attempt = match component { MgsUpdateComponent::RotBootloader => { - match try_make_update_rot_bootloader( + try_make_update_rot_bootloader( log, baseboard_id, inventory, current_artifacts, - ) { - Ok(pending_update) => { - if let Some(update) = pending_update { - // There is a pending update, record it - pending_actions.add_pending_update(update); - } - // Otherwise, we don't have any pending actions, the - // component is already at the expected version - } - Err(e) => { - // There was a failure, skip the update and record it - pending_actions.add_blocked_update(BlockedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component, - reason: e, - }); - } - }; - } - MgsUpdateComponent::Rot => { - match try_make_update_rot( - log, - baseboard_id, - inventory, - current_artifacts, - ) { - Ok(pending_update) => { - if let Some(update) = pending_update { - pending_actions.add_pending_update(update); - } - } - Err(e) => { - pending_actions.add_blocked_update(BlockedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component, - reason: e, - }); - } - }; - } - MgsUpdateComponent::Sp => { - match try_make_update_sp( - log, - baseboard_id, - inventory, - current_artifacts, - ) { - Ok(pending_update) => { - if let Some(update) = pending_update { - pending_actions.add_pending_update(update); - } - } - Err(e) => { - pending_actions.add_blocked_update(BlockedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component, - reason: e, - }); - } - }; + ) } + MgsUpdateComponent::Rot => try_make_update_rot( + log, + baseboard_id, + inventory, + current_artifacts, + ), + MgsUpdateComponent::Sp => try_make_update_sp( + log, + baseboard_id, + inventory, + current_artifacts, + ), MgsUpdateComponent::HostOs => { match host_phase_1::try_make_update( log, @@ -643,32 +596,46 @@ fn try_make_update( current_artifacts, ) { Ok(pending_update) => { + // Host updates also return host OS phase 2 changes; + // pull those out here and insert them into + // `pending_actions`, then return the typical pending + // update (which will itself be inserted into + // `pending_actions` below). if let Some((update, host_os_phase_2_changes)) = pending_update { - pending_actions.add_pending_update(update); pending_actions.set_pending_host_os_phase2_changes( host_os_phase_2_changes, ); + Ok(Some(update)) + } else { + Ok(None) } } - Err(e) => { - pending_actions.add_blocked_update(BlockedMgsUpdate { - baseboard_id: baseboard_id.clone(), - component, - reason: e, - }); - } - }; + Err(e) => Err(e), + } } }; - // If there is a pending or blocked MGS driven update we return it - // immediately. - if !pending_actions.blocked_mgs_updates.is_empty() - || !pending_actions.pending_updates.is_empty() - { - return pending_actions; + match update_attempt { + Ok(None) => { + // No update needed; try the next component. + continue; + } + // If there is a pending or blocked MGS-driven update, we break so + // we can return it immediately. + Ok(Some(update)) => { + pending_actions.add_pending_update(update); + break; + } + Err(e) => { + pending_actions.add_blocked_update(BlockedMgsUpdate { + baseboard_id: baseboard_id.clone(), + component, + reason: e, + }); + break; + } } } From 4ca45ba5c77a83ea2457fce262f1e0e8066fdd2c Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 25 Sep 2025 10:12:52 +1200 Subject: [PATCH 54/55] fixes, expectorations, and openapi doc gen after merge with main --- ...mds-add-zones-with-mupdate-override-stdout | 12 ++++++++-- .../tests/output/cmds-target-release-stdout | 6 ++--- nexus/types/src/deployment/planning_report.rs | 11 +++++---- openapi/nexus-internal.json | 16 ++++++------- openapi/nexus-lockstep.json | 23 +++++++++++-------- 5 files changed, 41 insertions(+), 27 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-add-zones-with-mupdate-override-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-add-zones-with-mupdate-override-stdout index 1ee59db0ce8..040173296fc 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-add-zones-with-mupdate-override-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-add-zones-with-mupdate-override-stdout @@ -194,7 +194,7 @@ to: blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 52 (records: 68) external DNS: DNS zone: "oxide.example" (unchanged) @@ -255,7 +255,7 @@ to: blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 internal DNS: DNS zone: "control-plane.oxide.internal" (unchanged) - unchanged names: 51 (records: 65) + unchanged names: 52 (records: 68) external DNS: DNS zone: "oxide.example" (unchanged) @@ -386,6 +386,14 @@ internal DNS: * DNS zone: "control-plane.oxide.internal": + name: 571a8adf-f0b8-458c-8e6c-5a71e82af7ae.host (records: 1) + AAAA fd00:1122:3344:103::28 +* name: _nexus-lockstep._tcp (records: 3 -> 4) +- SRV port 12232 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal +- SRV port 12232 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal +- SRV port 12232 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal ++ SRV port 12232 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal ++ SRV port 12232 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal ++ SRV port 12232 466a9f29-62bf-4e63-924a-b9efdb86afec.host.control-plane.oxide.internal ++ SRV port 12232 571a8adf-f0b8-458c-8e6c-5a71e82af7ae.host.control-plane.oxide.internal * name: _nexus._tcp (records: 3 -> 4) - SRV port 12221 0c71b3b2-6ceb-4e8f-b020-b08675e83038.host.control-plane.oxide.internal - SRV port 12221 3eeb8d49-eb1a-43f8-bb64-c2338421c2c6.host.control-plane.oxide.internal diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout index ceebd2c0c28..f8f9dc5cf6f 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-target-release-stdout @@ -8216,9 +8216,9 @@ INFO performed noop zone image source checks on sled, sled_id: 98e6b7c2-2efa-41c INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 9, num_already_artifact: 9, num_eligible: 0, num_ineligible: 0 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a -INFO skipping board for MGS-driven update, serial_number: serial0, part_number: model0 -INFO skipping board for MGS-driven update, serial_number: serial1, part_number: model1 -INFO skipping board for MGS-driven update, serial_number: serial2, part_number: model2 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial0, part_number: model0 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial1, part_number: model1 +INFO skipping board for MGS-driven update (no update necessary), serial_number: serial2, part_number: model2 INFO ran out of boards for MGS-driven update generated blueprint 8e0cc787-e068-4a45-97ed-21029cbe4ddf based on parent blueprint 6d830d26-547e-492b-adfe-c5c4ad9c3751 blueprint source: planner with report: diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index cd2c2b664b0..af579663d78 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -1154,6 +1154,9 @@ impl fmt::Display for PlanningZoneUpdatesStepReport { )] #[serde(rename_all = "snake_case", tag = "type")] pub enum ZoneUpdatesWaitingOn { + /// Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS. + BlockedMgsUpdates, + /// Waiting on discretionary zone placement. DiscretionaryZones, @@ -1163,8 +1166,6 @@ pub enum ZoneUpdatesWaitingOn { /// Waiting on updates to RoT bootloader / RoT / SP / Host OS. PendingMgsUpdates, - /// Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS. - BlockedMgsUpdates, /// Waiting on the same set of blockers zone adds are waiting on. ZoneAddBlockers, @@ -1173,14 +1174,14 @@ pub enum ZoneUpdatesWaitingOn { impl ZoneUpdatesWaitingOn { pub fn as_str(&self) -> &'static str { match self { + Self::BlockedMgsUpdates => { + "blocked MGS updates (RoT bootloader / RoT / SP / Host OS)" + } Self::DiscretionaryZones => "discretionary zones", Self::InventoryPropagation => "zone propagation to inventory", Self::PendingMgsUpdates => { "pending MGS updates (RoT bootloader / RoT / SP / Host OS)" } - Self::BlockedMgsUpdates => { - "blocked MGS updates (RoT bootloader / RoT / SP / Host OS)" - } Self::ZoneAddBlockers => "zone add blockers", } } diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 4b281e97190..d1dfb00fa43 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -6340,13 +6340,13 @@ "ZoneUpdatesWaitingOn": { "oneOf": [ { - "description": "Waiting on discretionary zone placement.", + "description": "Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "discretionary_zones" + "blocked_mgs_updates" ] } }, @@ -6355,13 +6355,13 @@ ] }, { - "description": "Waiting on zones to propagate to inventory.", + "description": "Waiting on discretionary zone placement.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "inventory_propagation" + "discretionary_zones" ] } }, @@ -6370,13 +6370,13 @@ ] }, { - "description": "Waiting on updates to RoT / SP / Host OS / etc.", + "description": "Waiting on zones to propagate to inventory.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "pending_mgs_updates" + "inventory_propagation" ] } }, @@ -6385,13 +6385,13 @@ ] }, { - "description": "Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS.", + "description": "Waiting on updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "blocked_mgs_updates" + "pending_mgs_updates" ] } }, diff --git a/openapi/nexus-lockstep.json b/openapi/nexus-lockstep.json index 2dd1aed0fac..07d590a5efc 100644 --- a/openapi/nexus-lockstep.json +++ b/openapi/nexus-lockstep.json @@ -5551,6 +5551,10 @@ "$ref": "#/components/schemas/PlanningAddSufficientZonesExist" } }, + "target_release_generation_is_one": { + "description": "Set to true if the target release generation is 1, which would allow zones to be added.", + "type": "boolean" + }, "waiting_on": { "nullable": true, "description": "What are we waiting on to start zone additions?", @@ -5572,7 +5576,8 @@ "sleds_waiting_for_ntp_zone", "sleds_without_ntp_zones_in_inventory", "sleds_without_zpools_for_ntp_zones", - "sufficient_zones_exist" + "sufficient_zones_exist", + "target_release_generation_is_one" ] }, "PlanningAddSufficientZonesExist": { @@ -7409,13 +7414,13 @@ "ZoneUpdatesWaitingOn": { "oneOf": [ { - "description": "Waiting on discretionary zone placement.", + "description": "Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "discretionary_zones" + "blocked_mgs_updates" ] } }, @@ -7424,13 +7429,13 @@ ] }, { - "description": "Waiting on zones to propagate to inventory.", + "description": "Waiting on discretionary zone placement.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "inventory_propagation" + "discretionary_zones" ] } }, @@ -7439,13 +7444,13 @@ ] }, { - "description": "Waiting on updates to RoT / SP / Host OS / etc.", + "description": "Waiting on zones to propagate to inventory.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "pending_mgs_updates" + "inventory_propagation" ] } }, @@ -7454,13 +7459,13 @@ ] }, { - "description": "Waiting on blocked updates to RoT bootloader / RoT / SP / Host OS.", + "description": "Waiting on updates to RoT bootloader / RoT / SP / Host OS.", "type": "object", "properties": { "type": { "type": "string", "enum": [ - "blocked_mgs_updates" + "pending_mgs_updates" ] } }, From 536996dec47dd36a88e6753a3b89b9ea449fb419 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 25 Sep 2025 10:16:52 +1200 Subject: [PATCH 55/55] =?UTF-8?q?fmt=20=F0=9F=98=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nexus/types/src/deployment/planning_report.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/types/src/deployment/planning_report.rs b/nexus/types/src/deployment/planning_report.rs index af579663d78..92b1fafeabd 100644 --- a/nexus/types/src/deployment/planning_report.rs +++ b/nexus/types/src/deployment/planning_report.rs @@ -1166,7 +1166,6 @@ pub enum ZoneUpdatesWaitingOn { /// Waiting on updates to RoT bootloader / RoT / SP / Host OS. PendingMgsUpdates, - /// Waiting on the same set of blockers zone adds are waiting on. ZoneAddBlockers, }