diff --git a/Cargo.lock b/Cargo.lock index 3f9d01f1566..79dc56a3a11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8255,6 +8255,7 @@ dependencies = [ "newtype_derive", "nexus-types-versions", "omicron-common", + "omicron-git-version", "omicron-passwords", "omicron-test-utils", "omicron-uuid-kinds", diff --git a/Cargo.toml b/Cargo.toml index fc16e0bc2d9..87463ad25fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,6 +103,7 @@ members = [ "nexus/defaults", "nexus/external-api", "nexus/fm", + "nexus/fm/state-report" "nexus/internal-api", "nexus/inventory", "nexus/lockstep-api", @@ -298,6 +299,7 @@ default-members = [ "nexus/defaults", "nexus/external-api", "nexus/fm", + "nexus/fm/state-report", "nexus/internal-api", "nexus/inventory", "nexus/lockstep-api", diff --git a/git-version/src/lib.rs b/git-version/src/lib.rs index 190e7fad41e..214c9df9a54 100644 --- a/git-version/src/lib.rs +++ b/git-version/src/lib.rs @@ -80,7 +80,7 @@ use std::str::FromStr; /// See [here][1] for discussion of this limitation. /// /// [1]: https://github.com/oxidecomputer/omicron/pull/10578#discussion_r3384362440 -#[derive(Debug, serde_with::DeserializeFromStr)] +#[derive(Debug, serde_with::DeserializeFromStr, Clone)] pub struct GitVersion { // We use a `Cow` here so that we need not allocate when constructing a // `GitVersion` to represent the current state of the repository, as it can diff --git a/nexus/db-model/src/fm/sitrep_analysis_report.rs b/nexus/db-model/src/fm/sitrep_analysis_report.rs index dd97871ed60..9197127fd99 100644 --- a/nexus/db-model/src/fm/sitrep_analysis_report.rs +++ b/nexus/db-model/src/fm/sitrep_analysis_report.rs @@ -8,7 +8,9 @@ use crate::DbTypedUuid; use anyhow::Context; use nexus_db_schema::schema::fm_sitrep_analysis_report; -use nexus_types::fm::analysis_reports::{AnalysisReport, InputReport}; +use nexus_types::fm::analysis_reports::{ + AnalysisReport, InputReport, UnparsedSitrepReport, +}; use omicron_uuid_kinds::SitrepKind; #[derive(Queryable, Insertable, Clone, Debug, Selectable)] @@ -42,3 +44,21 @@ impl SitrepAnalysisReport { Ok(Self { sitrep_id, git_commit, input_report, analysis_report }) } } + +impl From for UnparsedSitrepReport { + fn from(report: SitrepAnalysisReport) -> Self { + let SitrepAnalysisReport { + sitrep_id: _, + git_commit, + input_report, + analysis_report, + } = report; + Self { + git_commit: git_commit + .parse() + .expect("GitVersion::from_str is infallible"), + input_report, + analysis_report, + } + } +} diff --git a/nexus/db-queries/src/db/datastore/fm.rs b/nexus/db-queries/src/db/datastore/fm.rs index f86e3362100..3d6d23271b2 100644 --- a/nexus/db-queries/src/db/datastore/fm.rs +++ b/nexus/db-queries/src/db/datastore/fm.rs @@ -47,6 +47,7 @@ use nexus_db_schema::schema::fm_sitrep_history::dsl as history_dsl; use nexus_db_schema::schema::fm_support_bundle_request::dsl as support_bundle_req_dsl; use nexus_types::fm; use nexus_types::fm::Sitrep; +use nexus_types::fm::analysis_reports::SitrepSummary; use nexus_types::support_bundle::{BundleData, BundleDataSelection}; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -1640,6 +1641,94 @@ impl DataStore { .select(model::SitrepVersion::as_select()) } + /// Lists summaries of the sitreps in the sitrep history, paginated by + /// version number. + /// + /// Unlike [`DataStore::fm_sitrep_version_list`], which returns only the + /// [`fm::SitrepVersion`] records from the history table, this method + /// returns a [`SitrepSummary`] for each version in the history, which + /// includes the sitrep's [`fm::SitrepMetadata`] record, along with + /// [analysis report](fm::analysis_reports::UnparsedSitrepReport) reports + /// describing the analysis that produced it, if one exists. + pub async fn fm_sitrep_history_summary_list( + &self, + opctx: &OpContext, + pagparams: &DataPageParams<'_, SqlU32>, + ) -> ListResultVec { + // TODO(eliza): there should probably be an authz object for the fm + // sitrep? + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let summaries = Self::sitrep_history_summary_list_query(pagparams) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .filter_map(|(version, metadata, report)| { + let version: fm::SitrepVersion = version.into(); + // This *should* never be null, as discussed in the comment in + // `sitrep_history_summary_list_query`. Throwing the whole thing + // out if it is is probably fine, since it should never happen. + let Some(metadata) = metadata else { + slog::warn!( + opctx.log, + "sitrep v{} has ID {}, but no corresponding fm_sitrep \ + metadata record exists with that ID! this is a bug!", + version.version, + version.id; + "sitrep_id" => ?version.id, + "sitrep_version" => version.version, + ); + return None; + }; + Some(SitrepSummary::new(version, metadata.into(), report)) + }) + .collect::>(); + + Ok(summaries) + } + + fn sitrep_history_summary_list_query( + pagparams: &DataPageParams<'_, SqlU32>, + ) -> impl RunnableQuery<( + model::SitrepVersion, + Option, + Option, + )> + use<> { + paginated( + history_dsl::fm_sitrep_history, + history_dsl::version, + &pagparams, + ) + // Here we come to a somewhat sad state of affairs: each row in + // `fm_sitrep_history` should always have a corresponding row in + // `fm_sitrep` for the history record's sitrep ID, so logically, this is + // an INNER JOIN. However! An INNER JOIN prevents CockroachDB's query + // planner from enforcing the LIMIT until after the JOIN is evaluated, + // since an INNER JOIN may discard rows. This means we perform a "full + // scan" of the `fm_sitrep_history` table, which runs afoul of the "no + // full table scans" setting. Using a LEFT JOIN here allows the query + // planner to apply the LIMIT to the scan over the history table, and + // avoids the full scan. Unfortunately, this means that the caller has + // to handle the fact that the "shouldn't happen" case where a history + // row lacks a sitrep with the same UUID. + .left_join( + sitrep_dsl::fm_sitrep.on(sitrep_dsl::id.eq(history_dsl::sitrep_id)), + ) + // The analysis report may or may not exist, so this one actually + // *should* be a LEFT JOIN. + .left_join( + analysis_report_dsl::fm_sitrep_analysis_report + .on(analysis_report_dsl::sitrep_id.eq(history_dsl::sitrep_id)), + ) + .select(( + model::SitrepVersion::as_select(), + Option::::as_select(), + Option::::as_select(), + )) + } + /// Check whether the given sitrep limit has been reached. /// /// This (necessarily) does a full table scan on the sitrep table up to @@ -2201,6 +2290,35 @@ mod tests { logctx.cleanup_successful(); } + #[tokio::test] + async fn explain_sitrep_history_summary_list_query() { + let logctx = + dev::test_setup_log("explain_sitrep_history_summary_list_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let pagparams = DataPageParams { + marker: None, + limit: std::num::NonZeroU32::new(420).unwrap(), + direction: dropshot::PaginationOrder::Descending, + }; + let query = DataStore::sitrep_history_summary_list_query(&pagparams); + let explanation = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + eprintln!("{explanation}"); + assert!( + !explanation.contains("FULL SCAN"), + "Found an unexpected FULL SCAN: {}", + explanation + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn explain_sitrep_read_ereports_query() { let logctx = dev::test_setup_log("explain_sitrep_read_ereports_query"); diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index 234a3e77883..5b93bf15e71 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -3286,6 +3286,10 @@ table! { } allow_tables_to_appear_in_same_query!(fm_sitrep_analysis_report, fm_sitrep); +allow_tables_to_appear_in_same_query!( + fm_sitrep_analysis_report, + fm_sitrep_history +); table! { disk_type_local_storage (disk_id) { diff --git a/nexus/fm/state-report/Cargo.toml b/nexus/fm/state-report/Cargo.toml new file mode 100644 index 00000000000..ea3cab65ef2 --- /dev/null +++ b/nexus/fm/state-report/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "nexus-fm-state-report" +edition.workspace = true + +[lints] +workspace = true + +[build-dependencies] +omicron-rpaths.workspace = true + +[dependencies] +anyhow.workspace = true +futures.workspace = true +nexus-db-model.workspace = true +nexus-db-queries.workspace = true +nexus-types.workspace = true +# See omicron-rpaths for more about the "pq-sys" dependency. +pq-sys = "*" + +omicron-workspace-hack.workspace = true diff --git a/nexus/fm/state-report/build.rs b/nexus/fm/state-report/build.rs new file mode 100644 index 00000000000..dbdb6a3c8b0 --- /dev/null +++ b/nexus/fm/state-report/build.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +fn main() { + // See omicron-rpaths for documentation. NOTE: This file MUST be kept in + // sync with the other build.rs files in this repository. + omicron_rpaths::configure_default_omicron_rpaths(); +} diff --git a/nexus/fm/state-report/src/lib.rs b/nexus/fm/state-report/src/lib.rs new file mode 100644 index 00000000000..5648d1594d6 --- /dev/null +++ b/nexus/fm/state-report/src/lib.rs @@ -0,0 +1,36 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A li'l dingus for collecting a snapshot of fault management state. + +use anyhow::Context; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_db_queries::db::datastore::SQL_BATCH_SIZE; +use nexus_db_queries::db::pagination::Paginator; + +pub use nexus_types::fm::analysis_reports::*; + +pub struct SnapshotParams { + pub requested_sitrep_id: Option, + pub max_historical_sitreps: usize, +} + +pub fn snapshot( + opctx: &OpContext, + datastore: &DataStore, + params: &SnapshotParams, +) -> anyhow::Result { + // We are about to read A Whole Bunch of Stuff. Make sure that's oaky + opctx.check_complex_operations_allowed()?; + + let (current_version, current_sitrep) = + datastore.fm_sitrep_read_current(opctx).await?; + let current_config = datastore + .fm_config_get_latest(opctx) + .await? + .map_or_else(PlannerConfig::default, |c| c.config.planner_config); + + todo!("eliza: draw the rest of the owl") +} diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index 331e64835c8..ad2e22ad95a 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -73,6 +73,7 @@ gateway-client.workspace = true gateway-types.workspace = true internal-dns-types.workspace = true omicron-common.workspace = true +omicron-git-version.workspace = true omicron-passwords.workspace = true omicron-workspace-hack.workspace = true semver.workspace = true diff --git a/nexus/types/src/fm/analysis_reports.rs b/nexus/types/src/fm/analysis_reports.rs index 1f37016622e..6d0f2f77b84 100644 --- a/nexus/types/src/fm/analysis_reports.rs +++ b/nexus/types/src/fm/analysis_reports.rs @@ -5,10 +5,18 @@ //! Human-readable reports summarizing what occurred during fault management //! analysis. +use super::FmConfigView; +use super::Sitrep; +use super::SitrepMetadata; +use super::SitrepVersion; use super::case; use super::display; use super::ereport::EreportId; +use anyhow::Context; +use chrono::DateTime; +use chrono::Utc; use iddqd::IdOrdMap; +use omicron_git_version::GitVersion; use omicron_uuid_kinds::{ AlertUuid, CaseUuid, CollectionUuid, PhysicalDiskUuid, SitrepUuid, SupportBundleUuid, @@ -17,6 +25,109 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; +use std::sync::Arc; + +/// A snapshot of the state of the fault management subsystem, for diagnostic +/// purposes. +/// +/// **This format is not stable. It may change at any time without +/// backwards-compatibility guarantees.** +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FmStateReport { + /// The current configuration for the fault management subsystem. + pub current_config: FmConfigView, + /// A complete snapshot of the current sitrep, if one exists. + pub current_sitrep: Option>, + /// A complete snapshot of the sitrep in which + /// Summaries of historical sitreps, including the reports for the current + /// sitrep at the first index. + pub history: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommittedSitrep { + pub version: u32, + pub time_made_current: DateTime, + pub sitrep: Sitrep, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum RequestedSitrep { + NoLongerExists(SitrepUuid), + Found(Arc), +} + +/// An entry in the sitrep history in a [`FmStateReport`] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SitrepSummary { + #[serde(flatten)] + pub metadata: SitrepMetadata, + pub version: u32, + pub time_made_current: DateTime, + pub reports: Option, +} + +impl SitrepSummary { + pub fn new( + version: SitrepVersion, + metadata: SitrepMetadata, + reports: Option>, + ) -> Self { + let SitrepVersion { + version, + time_made_current, + // drop the ID here as it's already in the metadaa + id: _, + } = version; + Self { + version, + time_made_current, + metadata, + reports: reports.map(Into::into), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UnparsedSitrepReport { + pub git_commit: GitVersion, + pub input_report: serde_json::Value, + pub analysis_report: serde_json::Value, +} + +impl UnparsedSitrepReport { + pub fn parse_analysis_report( + &self, + current_git_commit: &GitVersion, + ) -> anyhow::Result { + AnalysisReport::deserialize(&self.analysis_report) + .with_context(self.parse_err("analysis_report", current_git_commit)) + } + + pub fn parse_inputreport( + &self, + current_git_commit: &GitVersion, + ) -> anyhow::Result { + InputReport::deserialize(&self.input_report) + .with_context(self.parse_err("input_report", current_git_commit)) + } + + fn parse_err( + &self, + which: &str, + current_git_commit: &GitVersion, + ) -> impl FnOnce() -> String { + move || { + format!( + "could not interpret {which}. note: it was produced by Nexus \ + on Git commit {}, while the current Git commit is \ + {current_git_commit}. perhaps these versions are \ + incompatible?", + self.git_commit, + ) + } + } +} #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct AnalysisReport { diff --git a/support-bundle-collection/src/steps/fm.rs b/support-bundle-collection/src/steps/fm.rs new file mode 100644 index 00000000000..7be716e2709 --- /dev/null +++ b/support-bundle-collection/src/steps/fm.rs @@ -0,0 +1,3 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. diff --git a/support-bundle-collection/src/steps/mod.rs b/support-bundle-collection/src/steps/mod.rs index 57ddda584ed..a2f0bd00eee 100644 --- a/support-bundle-collection/src/steps/mod.rs +++ b/support-bundle-collection/src/steps/mod.rs @@ -10,6 +10,7 @@ use futures::FutureExt; use nexus_types::internal_api::background::SupportBundleCollectionStep; mod ereports; +mod fm; mod host_info; mod metadata; mod reconfigurator;