Rpws for all networking (#4822)

Overview
---
This PR ensures the rest of our `dpd` configuration is covered by a RPW
to help recover state in the event of `dendrite` crashing, the switch
zone restarting / being replaced, or the sled restarting. This is
accomplished via a background task in Nexus that periodically ensures
`dpd` is up to date with the tables in Nexus. The tradeoffs of this
design is that we don't track versioning and reconcile the entire state
every time, but since the actual number of ports will never be that high
(relative to something like NAT entries) the tradeoff of less efficiency
for much greater simplicity seems to make sense today, and it requires
much less rework in Nexus and Dendrite should we choose to replace this
strategy down the road.

Tasks
---
- [x] Ensure that Service Zones configured during rss, cold boot, and
nexus have their NAT entries added to the NAT RPW table (extracted into
#4857)
- [x] Create background task that periodically reconciles switch port
configuration for dendrite instances
- [x] Move switch zone uplink SMF property updates to RPW
- [x] Move routing updates (via mg) to RPW
    - [x] Static Routing
    - [x] BGP
- [x] Move bootstore updates to RPW
- [x] Move loopback address management to RPW
- [x] Move Nexus-side switch zone service on-demand lookups as outlined
in #5092

Verifications Performed
---
- [x] Basic instance deployment
- [x] Loopback Address Creation
- [x] BGP configuration (a4x2)
- [ ] BGP configuration modification (a4x2)
- [x] Static routing
- [x] Static routing configuration modification

Related
---
Closes #4715
Closes #4650

Depends on https://github.com/oxidecomputer/dendrite/pull/838

---------

Co-authored-by: Levon Tarver <[email protected]>

Author: internet-diglett

.github/buildomat/jobs/deploy.sh

@@ -231,12 +231,10 @@ first = \"$SERVICE_IP_POOL_START\"
 		/^last/c\\
 last = \"$SERVICE_IP_POOL_END\"
 	}
-	/^\\[rack_network_config/,/^$/ {
-		/^infra_ip_first/c\\
+	/^infra_ip_first/c\\
 infra_ip_first = \"$UPLINK_IP\"
-		/^infra_ip_last/c\\
+	/^infra_ip_last/c\\
 infra_ip_last = \"$UPLINK_IP\"
-	}
 	/^\\[\\[rack_network_config.ports/,/^\$/ {
 		/^routes/c\\
 routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\]
@@ -335,6 +333,18 @@ while [[ $(pfexec svcs -z $(zoneadm list -n | grep oxz_ntp) \
 done
 echo "Waited for chrony: ${retry}s"
 
+# Wait for at least one nexus zone to become available
+retry=0
+until zoneadm list | grep nexus; do
+	if [[ $retry -gt 300 ]]; then
+		echo "Failed to start at least one nexus zone after 300 seconds"
+		exit 1
+	fi
+	sleep 1
+	retry=$((retry + 1))
+done
+echo "Waited for nexus: ${retry}s"
+
 export RUST_BACKTRACE=1
 export E2E_TLS_CERT IPPOOL_START IPPOOL_END
 eval "$(./tests/bootstrap)"

clients/mg-admin-client/src/lib.rs

@@ -17,11 +17,13 @@ mod inner {
 }
 
 pub use inner::types;
+use inner::types::Prefix4;
 pub use inner::Error;
 
 use inner::Client as InnerClient;
 use omicron_common::api::external::BgpPeerState;
 use slog::Logger;
+use std::hash::Hash;
 use std::net::Ipv6Addr;
 use std::net::SocketAddr;
 use thiserror::Error;
@@ -81,3 +83,18 @@ impl Client {
         Ok(Self { inner, log })
     }
 }
+
+impl Eq for Prefix4 {}
+
+impl PartialEq for Prefix4 {
+    fn eq(&self, other: &Self) -> bool {
+        self.value == other.value && self.length == other.length
+    }
+}
+
+impl Hash for Prefix4 {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.value.hash(state);
+        self.length.hash(state);
+    }
+}

dev-tools/omdb/tests/env.out

@@ -92,6 +92,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -182,6 +186,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -259,6 +267,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/successes.out

@@ -299,6 +299,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -368,7 +372,7 @@ task: "nat_v4_garbage_collector"
   currently executing: no
   last completed activation: iter 2, triggered by an explicit signal
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
-warning: unknown background task: "nat_v4_garbage_collector" (don't know how to interpret details: Null)
+    last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }
 
 task: "blueprint_loader"
   configured period: every 1m 40s
@@ -389,7 +393,7 @@ task: "bfd_manager"
   currently executing: no
   last completed activation: iter 2, triggered by an explicit signal
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
-warning: unknown background task: "bfd_manager" (don't know how to interpret details: Object {})
+    last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }
 
 task: "external_endpoints"
   configured period: every 1m
@@ -440,6 +444,13 @@ task: "service_zone_nat_tracker"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
     last completion reported error: inventory collection is None
 
+task: "switch_port_config_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: iter 2, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "switch_port_config_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/

nexus-config/src/nexus_config.rs

@@ -367,6 +367,8 @@ pub struct BackgroundTaskConfig {
     pub sync_service_zone_nat: SyncServiceZoneNatConfig,
     /// configuration for the bfd manager task
     pub bfd_manager: BfdManagerConfig,
+    /// configuration for the switch port settings manager task
+    pub switch_port_settings_manager: SwitchPortSettingsManagerConfig,
     /// configuration for region replacement task
     pub region_replacement: RegionReplacementConfig,
 }
@@ -427,6 +429,15 @@ pub struct SyncServiceZoneNatConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct SwitchPortSettingsManagerConfig {
+    /// Interval (in seconds) for periodic activations of this background task.
+    /// This task is also activated on-demand when any of the switch port settings
+    /// api endpoints are called.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct InventoryConfig {
@@ -713,6 +724,7 @@ mod test {
             blueprints.period_secs_load = 10
             blueprints.period_secs_execute = 60
             sync_service_zone_nat.period_secs = 30
+            switch_port_settings_manager.period_secs = 30
             region_replacement.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
@@ -828,6 +840,10 @@ mod test {
                         sync_service_zone_nat: SyncServiceZoneNatConfig {
                             period_secs: Duration::from_secs(30)
                         },
+                        switch_port_settings_manager:
+                            SwitchPortSettingsManagerConfig {
+                                period_secs: Duration::from_secs(30),
+                            },
                         region_replacement: RegionReplacementConfig {
                             period_secs: Duration::from_secs(30),
                         },
@@ -893,6 +909,7 @@ mod test {
             blueprints.period_secs_load = 10
             blueprints.period_secs_execute = 60
             sync_service_zone_nat.period_secs = 30
+            switch_port_settings_manager.period_secs = 30
             region_replacement.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"

nexus/db-model/src/address_lot.rs

@@ -11,8 +11,10 @@ use omicron_common::api::external;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
+pub const INFRA_LOT: &str = "initial-infra";
+
 impl_enum_type!(
-    #[derive(SqlType, Debug, Clone, Copy)]
+    #[derive(SqlType, Debug, Clone, Copy, QueryId)]
     #[diesel(postgres_type(name = "address_lot_kind", schema = "public"))]
     pub struct AddressLotKindEnum;
 
@@ -24,7 +26,7 @@ impl_enum_type!(
         FromSqlRow,
         PartialEq,
         Serialize,
-        Deserialize
+        Deserialize,
     )]
     #[diesel(sql_type = AddressLotKindEnum)]
     pub enum AddressLotKind;

nexus/db-model/src/bootstore.rs

@@ -1,4 +1,5 @@
-use crate::schema::bootstore_keys;
+use crate::schema::{bootstore_config, bootstore_keys};
+use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 
 pub const NETWORK_KEY: &str = "network_key";
@@ -11,3 +12,18 @@ pub struct BootstoreKeys {
     pub key: String,
     pub generation: i64,
 }
+
+/// BootstoreConfig is a key-value store for bootstrapping data.
+/// We serialize the data as json because it is inherently polymorphic and it
+/// is not intended to be queried directly.
+#[derive(
+    Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize,
+)]
+#[diesel(table_name = bootstore_config)]
+pub struct BootstoreConfig {
+    pub key: String,
+    pub generation: i64,
+    pub data: serde_json::Value,
+    pub time_created: DateTime<Utc>,
+    pub time_deleted: Option<DateTime<Utc>>,
+}

nexus/db-model/src/schema.rs

@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(44, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(45, 0, 0);
 
 table! {
     disk (id) {
@@ -1529,6 +1529,16 @@ table! {
     }
 }
 
+table! {
+    bootstore_config (key, generation) {
+        key -> Text,
+        generation -> Int8,
+        data -> Jsonb,
+        time_created -> Timestamptz,
+        time_deleted -> Nullable<Timestamptz>,
+    }
+}
+
 table! {
     bfd_session (remote, switch) {
         id -> Uuid,

nexus/db-model/src/unsigned.rs

@@ -130,6 +130,7 @@ where
     FromSqlRow,
     Serialize,
     Deserialize,
+    QueryId,
 )]
 #[diesel(sql_type = sql_types::BigInt)]
 #[repr(transparent)]