[WIP][nexus] initial diagnosis engine prototype (PSU insert/remove)

Cargo.toml

@@ -86,6 +86,7 @@ members = [
     "nexus/db-schema",
     "nexus/defaults",
     "nexus/external-api",
+    "nexus/fm",
     "nexus/internal-api",
     "nexus/inventory",
     "nexus/lockstep-api",

dev-tools/omdb/src/bin/omdb/db/sitrep.rs

@@ -238,7 +238,7 @@ async fn cmd_db_sitrep_show(
         }
     };
 
-    let fm::Sitrep { metadata } = sitrep;
+    let fm::Sitrep { metadata, cases } = sitrep;
     let fm::SitrepMetadata {
         id,
         creator_id,
@@ -345,5 +345,12 @@ async fn cmd_db_sitrep_show(
         }
     }
 
+    if !cases.is_empty() {
+        println!("\n{:-<80}\n", "== CASES");
+        for case in cases {
+            println!("{}", case.display_indented(4, Some(id)));
+        }
+    }
+
     Ok(())
 }

ereport/types/src/lib.rs

@@ -32,6 +32,7 @@ pub struct Ereport {
     Serialize,
     Deserialize,
     JsonSchema,
+    Hash,
 )]
 #[repr(transparent)]
 #[serde(from = "u64", into = "u64")]
@@ -102,7 +103,18 @@ impl TryFrom<i64> for Ena {
 }
 
 /// Unique identifier for an ereport.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    PartialOrd,
+    Ord,
+    Hash,
+)]
 pub struct EreportId {
     pub restart_id: EreporterRestartUuid,
     pub ena: Ena,

nexus-config/src/nexus_config.rs

@@ -930,6 +930,11 @@ pub struct FmTasksConfig {
     /// garbage collects unneeded fault management sitreps in the database.
     #[serde_as(as = "DurationSeconds<u64>")]
     pub sitrep_gc_period_secs: Duration,
+    /// period (in seconds) for periodic activations of the background task that
+    /// updates externally-visible database tables to match the current situation
+    /// report.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub rendezvouz_period_secs: Duration,
 }
 
 impl Default for FmTasksConfig {
@@ -940,6 +945,9 @@ impl Default for FmTasksConfig {
             // time the current sitrep changes, and activating it more
             // frequently won't make things more responsive.
             sitrep_gc_period_secs: Duration::from_secs(600),
+            // This, too, is activated whenever a new sitrep is loaded, so we
+            // need not set the periodic activation interval too high.
+            rendezvouz_period_secs: Duration::from_secs(300),
         }
     }
 }
@@ -1281,6 +1289,7 @@ mod test {
             fm.sitrep_gc_period_secs = 49
             probe_distributor.period_secs = 50
             multicast_reconciler.period_secs = 60
+            fm.rendezvous_period_secs = 51
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -1529,6 +1538,7 @@ mod test {
                         fm: FmTasksConfig {
                             sitrep_load_period_secs: Duration::from_secs(48),
                             sitrep_gc_period_secs: Duration::from_secs(49),
+                            rendezvouz_period_secs: Duration::from_secs(51),
                         },
                         probe_distributor: ProbeDistributorConfig {
                             period_secs: Duration::from_secs(50),
@@ -1640,6 +1650,7 @@ mod test {
             fm.sitrep_load_period_secs = 45
             fm.sitrep_gc_period_secs = 46
             probe_distributor.period_secs = 47
+            fm.rendezvous_period_secs = 48
             multicast_reconciler.period_secs = 60
 
             [default_region_allocation_strategy]

nexus/background-task-interface/src/init.rs

@@ -51,6 +51,7 @@ pub struct BackgroundTasks {
     pub task_webhook_deliverator: Activator,
     pub task_sp_ereport_ingester: Activator,
     pub task_reconfigurator_config_loader: Activator,
+    pub task_fm_rendezvous: Activator,
     pub task_fm_sitrep_loader: Activator,
     pub task_fm_sitrep_gc: Activator,
     pub task_probe_distributor: Activator,

nexus/db-model/src/alert_class.rs

@@ -4,6 +4,7 @@
 
 use super::impl_enum_type;
 use nexus_types::external_api::views;
+use omicron_common::api::external::Error;
 use serde::de::{self, Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use std::fmt;
@@ -30,6 +31,8 @@ impl_enum_type!(
     TestFooBaz => b"test.foo.baz"
     TestQuuxBar => b"test.quux.bar"
     TestQuuxBarBaz => b"test.quux.bar.baz"
+    PsuInserted => b"hw.insert.power.power_shelf.psu"
+    PsuRemoved => b"hw.remove.power.power_shelf.psu"
 );
 
 impl AlertClass {
@@ -44,6 +47,8 @@ impl AlertClass {
             Self::TestFooBaz => "test.foo.baz",
             Self::TestQuuxBar => "test.quux.bar",
             Self::TestQuuxBarBaz => "test.quux.bar.baz",
+            Self::PsuInserted => "hw.insert.power.power_shelf.psu",
+            Self::PsuRemoved => "hw.remove.power.power_shelf.psu",
         }
     }
 
@@ -76,6 +81,12 @@ impl AlertClass {
             | Self::TestQuuxBarBaz => {
                 "This is a test of the emergency alert system"
             }
+            Self::PsuInserted => {
+                "A power supply unit (PSU) has been inserted into the power shelf"
+            }
+            Self::PsuRemoved => {
+                "A power supply unit (PSU) has been removed from the power shelf"
+            }
         }
     }
 
@@ -84,6 +95,32 @@ impl AlertClass {
         <Self as strum::VariantArray>::VARIANTS;
 }
 
+impl From<nexus_types::fm::AlertClass> for AlertClass {
+    fn from(input: nexus_types::fm::AlertClass) -> Self {
+        use nexus_types::fm::AlertClass as In;
+        match input {
+            In::PsuRemoved => Self::PsuRemoved,
+            In::PsuInserted => Self::PsuInserted,
+        }
+    }
+}
+
+impl TryFrom<AlertClass> for nexus_types::fm::AlertClass {
+    type Error = Error;
+
+    fn try_from(input: AlertClass) -> Result<Self, Self::Error> {
+        use nexus_types::fm::AlertClass as Out;
+        match input {
+            AlertClass::PsuRemoved => Ok(Out::PsuRemoved),
+            AlertClass::PsuInserted => Ok(Out::PsuInserted),
+            class => Err(Error::invalid_value(
+                "alert_class",
+                format!("'{class}' is not a FM alert class"),
+            )),
+        }
+    }
+}
+
 impl fmt::Display for AlertClass {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", self.as_str())

nexus/db-model/src/fm.rs

@@ -19,6 +19,13 @@ use chrono::{DateTime, Utc};
 use nexus_db_schema::schema::{fm_sitrep, fm_sitrep_history};
 use omicron_uuid_kinds::{CollectionKind, OmicronZoneKind, SitrepKind};
 
+mod alert_request;
+pub use alert_request::*;
+mod case;
+pub use case::*;
+mod diagnosis_engine;
+pub use diagnosis_engine::*;
+
 #[derive(Queryable, Insertable, Clone, Debug, Selectable)]
 #[diesel(table_name = fm_sitrep)]
 pub struct SitrepMetadata {

nexus/db-model/src/fm/alert_request.rs

@@ -0,0 +1,55 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Fault management alert requests.
+
+use crate::AlertClass;
+use crate::DbTypedUuid;
+use nexus_db_schema::schema::fm_alert_request;
+use nexus_types::fm;
+use omicron_uuid_kinds::{
+    AlertKind, CaseKind, CaseUuid, SitrepKind, SitrepUuid,
+};
+
+#[derive(Queryable, Insertable, Clone, Debug, Selectable)]
+#[diesel(table_name = fm_alert_request)]
+pub struct AlertRequest {
+    pub id: DbTypedUuid<AlertKind>,
+    pub sitrep_id: DbTypedUuid<SitrepKind>,
+    pub requested_sitrep_id: DbTypedUuid<SitrepKind>,
+    pub case_id: DbTypedUuid<CaseKind>,
+    #[diesel(column_name = "class")]
+    pub class: AlertClass,
+    pub payload: serde_json::Value,
+}
+
+impl AlertRequest {
+    pub fn new(
+        current_sitrep_id: SitrepUuid,
+        case_id: CaseUuid,
+        req: fm::AlertRequest,
+    ) -> Self {
+        let fm::AlertRequest { id, requested_sitrep_id, payload, class } = req;
+        AlertRequest {
+            id: id.into(),
+            sitrep_id: current_sitrep_id.into(),
+            requested_sitrep_id: requested_sitrep_id.into(),
+            case_id: case_id.into(),
+            class: class.into(),
+            payload,
+        }
+    }
+}
+
+impl TryFrom<AlertRequest> for fm::AlertRequest {
+    type Error = <fm::AlertClass as TryFrom<AlertClass>>::Error;
+    fn try_from(req: AlertRequest) -> Result<Self, Self::Error> {
+        Ok(fm::AlertRequest {
+            id: req.id.into(),
+            requested_sitrep_id: req.requested_sitrep_id.into(),
+            payload: req.payload,
+            class: req.class.try_into()?,
+        })
+    }
+}