[nexus] config flag to disable SP ereport ingestion (#8709)

PR #8296 added the `sp_ereport_ingester` background task to Nexus for
periodically collecting ereports from SPs via MGS. However, the Hubris
PR adding the Hubris task that actually responds to these requests from
the control plane, oxidecomputer/hubris#2126, won't make it in until
after R16. This means that if we release R16 with a control plane that
tries to collect ereports, and a SP firmware that doesn't know how to
respond to such requests, the Nexus logs will be littered with 36 log
lines like this every 30 seconds:

```
20:58:04.603Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): client response
    background_task = sp_ereport_ingester
    gateway_url = http://[fd00:1122:3344:108::2]:12225
    result = Ok(Response { url: "http://[fd00:1122:3344:108::2]:12225/sp/sled/29/ereports?limit=255&restart_id=00000000-0000-0000-0000-000000000000", status: 503, headers: {"content-type": "application/json", "x-request-id": "35390a4a-6d3a-4683-be88-217267b46da0", "content-length": "224", "date": "Mon, 28 Jul 2025 20:58:04 GMT"} })
20:58:04.603Z WARN 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): ereport collection: unanticipated MGS request error: Error Response: status: 503 Service Unavailable; headers: {"content-type": "application/json", "x-request-id": "35390a4a-6d3a-4683-be88-217267b46da0", "content-length": "224", "date": "Mon, 28 Jul 2025 20:58:04 GMT"}; value: Error { error_code: Some("SpCommunicationFailed"), message: "error communicating with SP SpIdentifier { typ: Sled, slot: 29 }: RPC call failed (gave up after 5 attempts)", request_id: "35390a4a-6d3a-4683-be88-217267b46da0" }
    background_task = sp_ereport_ingester
    committed_ena = None
    error = Error Response: status: 503 Service Unavailable; headers: {"content-type": "application/json", "x-request-id": "35390a4a-6d3a-4683-be88-217267b46da0", "content-length": "224", "date": "Mon, 28 Jul 2025 20:58:04 GMT"}; value: Error { error_code: Some("SpCommunicationFailed"), message: "error communicating with SP SpIdentifier { typ: Sled, slot: 29 }: RPC call failed (gave up after 5 attempts)", request_id: "35390a4a-6d3a-4683-be88-217267b46da0" }
    file = nexus/src/app/background/tasks/ereport_ingester.rs:380
    gateway_addr = [fd00:1122:3344:108::2]:12225
    restart_id = 00000000-0000-0000-0000-000000000000 (ereporter_restart)
    slot = 29
    sp_type = sled
    start_ena = None
```

Similarly, MGS will also have a bunch of noisy complaints about these
requests failing.

The consequences of this are really not terrible: it just means we'll be
logging a lot of errors. But it seems mildly unfortunate to be
constantly trying to do something that's invariably doomed to failure,
and then yelling about how it didn't work. So, this commit adds a config
flag for disabling the whole thing, which we can turn on for R16's
production Nexus config and then turn back off when the Hubris changes
make it in. I did this using a config setting, rather than hard-coding
it to always be disabled, because there are also integration tests for
this stuff, which will break if we disabled it everywhere.

Author: hawkw

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -2787,7 +2787,7 @@ fn print_task_sp_ereport_ingester(details: &serde_json::Value) {
     use nexus_types::internal_api::background::SpEreportIngesterStatus;
     use nexus_types::internal_api::background::SpEreporterStatus;
 
-    let SpEreportIngesterStatus { sps, errors } =
+    let SpEreportIngesterStatus { sps, errors, disabled } =
         match serde_json::from_value(details.clone()) {
             Err(error) => {
                 eprintln!(
@@ -2813,9 +2813,19 @@ fn print_task_sp_ereport_ingester(details: &serde_json::Value) {
         }
     }
 
-    print_ereporter_status_totals(sps.iter().map(|sp| &sp.status));
+    if disabled {
+        println!("    SP ereport ingestion explicitly disabled by config!");
+    } else {
+        print_ereporter_status_totals(sps.iter().map(|sp| &sp.status));
+    }
 
     if !sps.is_empty() {
+        if disabled {
+            println!(
+                "/!\\ WEIRD: SP ereport ingestion disabled by config, but \
+                 some SP statuses were recorded!"
+            )
+        }
         println!("\n    service processors:");
         for SpEreporterStatus { sp_type, slot, status } in &sps {
             println!(

nexus-config/src/nexus_config.rs

@@ -815,11 +815,20 @@ pub struct SpEreportIngesterConfig {
     /// period (in seconds) for periodic activations of this background task
     #[serde_as(as = "DurationSeconds<u64>")]
     pub period_secs: Duration,
+
+    /// disable ereport collection altogether
+    ///
+    /// This is an emergency lever for support / operations.  It should never be
+    /// necessary.
+    ///
+    /// Default: Off
+    #[serde(default)]
+    pub disable: bool,
 }
 
 impl Default for SpEreportIngesterConfig {
     fn default() -> Self {
-        Self { period_secs: Duration::from_secs(30) }
+        Self { period_secs: Duration::from_secs(30), disable: false }
     }
 }
 
@@ -1320,6 +1329,7 @@ mod test {
                         },
                         sp_ereport_ingester: SpEreportIngesterConfig {
                             period_secs: Duration::from_secs(47),
+                            disable: false,
                         },
                     },
                     default_region_allocation_strategy:

nexus/src/app/background/init.rs

@@ -985,7 +985,10 @@ impl BackgroundTasksInitializer {
             description: "collects error reports from service processors",
             period: config.sp_ereport_ingester.period_secs,
             task_impl: Box::new(ereport_ingester::SpEreportIngester::new(
-                datastore, resolver, nexus_id,
+                datastore,
+                resolver,
+                nexus_id,
+                config.sp_ereport_ingester.disable,
             )),
             opctx: opctx.child(BTreeMap::new()),
             watchers: vec![],

nexus/src/app/background/tasks/ereport_ingester.rs

@@ -31,6 +31,7 @@ use std::sync::Arc;
 
 pub struct SpEreportIngester {
     resolver: internal_dns_resolver::Resolver,
+    disabled: bool,
     inner: Ingester,
 }
 
@@ -58,15 +59,24 @@ impl SpEreportIngester {
         datastore: Arc<DataStore>,
         resolver: internal_dns_resolver::Resolver,
         nexus_id: OmicronZoneUuid,
+        disabled: bool,
     ) -> Self {
-        Self { resolver, inner: Ingester { datastore, nexus_id } }
+        Self { resolver, inner: Ingester { datastore, nexus_id }, disabled }
     }
 
     async fn actually_activate(
         &mut self,
         opctx: &OpContext,
     ) -> SpEreportIngesterStatus {
         let mut status = SpEreportIngesterStatus::default();
+        if self.disabled {
+            status.disabled = true;
+            slog::trace!(
+                &opctx.log,
+                "SP ereport ingestion disabled, doing nothing",
+            );
+            return status;
+        }
         // Find MGS clients.
         // TODO(eliza): reuse the same client across activations; qorb, etc.
         let mgs_clients = {
@@ -478,6 +488,7 @@ mod tests {
             datastore.clone(),
             nexus.internal_resolver.clone(),
             nexus.id(),
+            false,
         );
 
         let activation1 = ingester.actually_activate(&opctx).await;

nexus/types/src/internal_api/background.rs

@@ -555,6 +555,9 @@ pub struct ReadOnlyRegionReplacementStartStatus {
 
 #[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq)]
 pub struct SpEreportIngesterStatus {
+    /// If `true`, then ereport ingestion has been explicitly disabled by
+    /// the config file.
+    pub disabled: bool,
     pub sps: Vec<SpEreporterStatus>,
     pub errors: Vec<String>,
 }

smf/nexus/multi-sled/config-partial.toml

@@ -83,6 +83,10 @@ read_only_region_replacement_start.period_secs = 30
 alert_dispatcher.period_secs = 60
 webhook_deliverator.period_secs = 60
 sp_ereport_ingester.period_secs = 30
+# Disabled in R16, as the Hubris task that handles ereport ingestion requests
+# has not merged yet, and trying to ingest them will just result in Nexus
+# logging a bunch of errors.
+sp_ereport_ingester.disable = true
 
 [default_region_allocation_strategy]
 # by default, allocate across 3 distinct sleds

smf/nexus/single-sled/config-partial.toml

@@ -83,6 +83,10 @@ read_only_region_replacement_start.period_secs = 30
 alert_dispatcher.period_secs = 60
 webhook_deliverator.period_secs = 60
 sp_ereport_ingester.period_secs = 30
+# Disabled in R16, as the Hubris task that handles ereport ingestion requests
+# has not merged yet, and trying to ingest them will just result in Nexus
+# logging a bunch of errors.
+sp_ereport_ingester.disable = true
 
 [default_region_allocation_strategy]
 # by default, allocate without requirement for distinct sleds.