diff --git a/Cargo.lock b/Cargo.lock index 9344da25236..7965fcd9bf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1772,27 +1772,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "common" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=cc0c307c617f2988aafdca4e3bd35ea178b64801#cc0c307c617f2988aafdca4e3bd35ea178b64801" -dependencies = [ - "anyhow", - "chrono", - "oxnet", - "rand 0.9.2", - "schemars 0.8.22", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-bunyan", - "slog-term", - "smf 0.10.0", - "thiserror 1.0.69", - "tokio", -] - [[package]] name = "compact_str" version = "0.8.1" @@ -3340,30 +3319,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "dpd-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=cc0c307c617f2988aafdca4e3bd35ea178b64801#cc0c307c617f2988aafdca4e3bd35ea178b64801" -dependencies = [ - "async-trait", - "chrono", - "common 0.1.0 (git+https://github.com/oxidecomputer/dendrite?rev=cc0c307c617f2988aafdca4e3bd35ea178b64801)", - "crc8", - "futures", - "http", - "oxnet", - "progenitor 0.13.0", - "regress 0.10.5", - "reqwest 0.13.2", - "schemars 0.8.22", - "serde", - "serde_json", - "slog", - "tokio", - "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control?branch=main)", - "uuid", -] - [[package]] name = "drift" version = "0.2.0" @@ -9660,7 +9615,6 @@ dependencies = [ "derive_more 0.99.20", "dice-verifier 0.3.0-pre0 (git+https://github.com/oxidecomputer/dice-util?branch=main)", "display-error-chain", - "dpd-client 0.1.0 (git+https://github.com/oxidecomputer/dendrite?rev=cc0c307c617f2988aafdca4e3bd35ea178b64801)", "dropshot", "expectorate", "flate2", @@ -9726,7 +9680,6 @@ dependencies = [ "sled-agent-api", "sled-agent-client", "sled-agent-config-reconciler", - "sled-agent-early-networking", "sled-agent-health-monitor", "sled-agent-measurements", "sled-agent-rack-setup", @@ -14102,22 +14055,6 @@ dependencies = [ "zone", ] -[[package]] -name = "sled-agent-early-networking" -version = "0.1.0" -dependencies = [ - "anyhow", - "futures", - "gateway-client", - "internal-dns-resolver", - "internal-dns-types", - "omicron-common", - "omicron-workspace-hack", - "sled-agent-types", - "slog", - "tokio", -] - [[package]] name = "sled-agent-health-monitor" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 74254298895..875766e06df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -151,7 +151,6 @@ members = [ "sled-agent/bootstrap-agent-lockstep-api", "sled-agent/bootstrap-agent-lockstep-types", "sled-agent/config-reconciler", - "sled-agent/early-networking", "sled-agent/health-monitor", "sled-agent/measurements", "sled-agent/rack-setup", @@ -345,7 +344,6 @@ default-members = [ "sled-agent/bootstrap-agent-lockstep-api", "sled-agent/bootstrap-agent-lockstep-types", "sled-agent/config-reconciler", - "sled-agent/early-networking", "sled-agent/health-monitor", "sled-agent/measurements", "sled-agent/rack-setup", @@ -813,7 +811,6 @@ sled = "=0.34.7" sled-agent-api = { path = "sled-agent/api" } sled-agent-client = { path = "clients/sled-agent-client" } sled-agent-config-reconciler = { path = "sled-agent/config-reconciler" } -sled-agent-early-networking = { path = "sled-agent/early-networking" } sled-agent-health-monitor = { path = "sled-agent/health-monitor" } sled-agent-measurements = { path = "sled-agent/measurements" } sled-agent-rack-setup = { path = "sled-agent/rack-setup" } diff --git a/dev-tools/ls-apis/api-manifest.toml b/dev-tools/ls-apis/api-manifest.toml index 44445adf01a..34685d5a1f6 100644 --- a/dev-tools/ls-apis/api-manifest.toml +++ b/dev-tools/ls-apis/api-manifest.toml @@ -790,47 +790,30 @@ permalinks = [ "https://github.com/oxidecomputer/dice-util/blob/d7472bfa91aee859c3fe0bdc1dbb1e320285228e/verifier/src/sled_agent.rs#L19-L25", ] -# NOTE: The following sled-agent edges are bugs: they cross deployment units. -# They are kept here to avoid breaking the build, but need to be fixed, either -# through client-side versioning or by some other means. - [[intra_deployment_unit_only_edges]] server = "omicron-sled-agent" client = "gateway-client" note = """ -BUG: Sled Agent creates two MGS clients. One of them (in the -sled-agent-scrimlet-reconcilers crate) queries the switch zone on the same sled, -which is within the same deployment unit as the global zone, so this is okay. - -The other one (early-networking/lib.rs:259, called from services.rs:1095) -queries both switch zones, so it crosses deployment units. This needs to be -fixed. - -Reference: https://github.com/oxidecomputer/omicron/issues/9708 +Sled Agent on scrimlets talks to MGS within its own sled's switch zone. The +`sled-agent-scrimlet-reconcilers` crate attempts to go out of its way to ensure +it's only ever used in this way. The switch zone is within the same deployment +unit as the global zone on its own sled, so this is okay. """ permalinks = [ "https://github.com/oxidecomputer/omicron/blob/fbc091a3/sled-agent/scrimlet-reconcilers/src/switch_zone_slot.rs#L46-L59", - "https://github.com/oxidecomputer/omicron/blob/32de20c/sled-agent/src/services.rs#L1095-L1102", - "https://github.com/oxidecomputer/omicron/blob/32de20c/sled-agent/early-networking/src/lib.rs#L259-L262", ] [[intra_deployment_unit_only_edges]] server = "omicron-sled-agent" client = "dpd-client" note = """ -BUG: Sled Agent creates two DPD clients. One of them (in the -sled-agent-scrimlet-reconcilers crate) is always to the switch zone on the same -sled, which is in the same deployment unit. - -The other one (services.rs:1104, iterating addresses returned by -EarlyNetworkSetup::lookup_uplinked_switch_zone_underlay_addrs) queries both -switch zones, so it crosses deployment units. This needs to be fixed. - -Reference: https://github.com/oxidecomputer/omicron/issues/9708 +Sled Agent on scrimlets talks to dendrite within its own sled's switch zone. The +`sled-agent-scrimlet-reconcilers` crate attempts to go out of its way to ensure +it's only ever used in this way. The switch zone is within the same deployment +unit as the global zone on its own sled, so this is okay. """ permalinks = [ "https://github.com/oxidecomputer/omicron/blob/fbc091a3/sled-agent/scrimlet-reconcilers/src/switch_zone_slot.rs#L46-L59", - "https://github.com/oxidecomputer/omicron/blob/32de20c/sled-agent/src/services.rs#L1104-L1115", ] # Note: sled-agent-rack-setup (RSS) is marked with `lifecycle = "rack-init"` diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index d7c9c9671a6..637c0a97484 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -38,7 +38,7 @@ Dendrite DPD (client: dpd-client) consumed by: lldpd (lldp/lldpd) via 2 paths consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 2 paths - consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path consumed by: tfportd (dendrite/tfportd) via 2 paths consumed by: wicketd (omicron/wicketd) via 2 paths @@ -49,7 +49,7 @@ Management Gateway Service (client: gateway-client) consumed by: lldpd (lldp/lldpd) via 1 path consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 6 paths - consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path consumed by: wicketd (omicron/wicketd) via 3 paths Wicketd Installinator (client: installinator-client) diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 4eff4a407f3..990d856dc9c 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -31,10 +31,6 @@ clickhouse-admin-types.workspace = true derive_more.workspace = true dice-verifier = { workspace = true, features = ["ipcc", "mock"] } display-error-chain.workspace = true -# XXX NOTE this is due to https://github.com/oxidecomputer/omicron/issues/9704 -# This is the R20 dpd client -# dpd-client.workspace = true -dpd-client = { git = "https://github.com/oxidecomputer/dendrite", rev = "cc0c307c617f2988aafdca4e3bd35ea178b64801" } dropshot.workspace = true flate2.workspace = true flume.workspace = true @@ -88,7 +84,6 @@ signal-hook.workspace = true sled-agent-api.workspace = true sled-agent-client.workspace = true sled-agent-config-reconciler.workspace = true -sled-agent-early-networking.workspace = true sled-agent-health-monitor.workspace = true sled-agent-measurements.workspace = true sled-agent-rack-setup.workspace = true diff --git a/sled-agent/early-networking/Cargo.toml b/sled-agent/early-networking/Cargo.toml deleted file mode 100644 index 5d177d7bed5..00000000000 --- a/sled-agent/early-networking/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "sled-agent-early-networking" -version = "0.1.0" -edition.workspace = true -license = "MPL-2.0" - -[lints] -workspace = true - -[dependencies] -anyhow.workspace = true -futures.workspace = true -gateway-client.workspace = true -internal-dns-resolver.workspace = true -internal-dns-types.workspace = true -omicron-common.workspace = true -omicron-workspace-hack.workspace = true -sled-agent-types.workspace = true -slog.workspace = true -tokio = { workspace = true, features = ["full"] } diff --git a/sled-agent/early-networking/src/lib.rs b/sled-agent/early-networking/src/lib.rs deleted file mode 100644 index 762c6c1255e..00000000000 --- a/sled-agent/early-networking/src/lib.rs +++ /dev/null @@ -1,249 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Network setup required to bring up the control plane - -use anyhow::{Context, anyhow}; -use futures::future; -use gateway_client::Client as MgsClient; -use internal_dns_resolver::Resolver as DnsResolver; -use internal_dns_types::names::ServiceName; -use omicron_common::address::MGS_PORT; -use omicron_common::backoff::{ - BackoffError, ExponentialBackoff, ExponentialBackoffBuilder, retry_notify, -}; -use sled_agent_types::early_networking::SwitchSlot; -use sled_agent_types::system_networking::SystemNetworkingConfig; -use slog::Logger; -use slog::info; -use slog::o; -use slog::warn; -use std::collections::{HashMap, HashSet}; -use std::net::Ipv6Addr; -use std::time::{Duration, Instant}; -use tokio::sync::watch; - -enum LookupSwitchZoneAddrsResult { - // We found every switch zone reported by internal DNS. - TotalSuccess(HashMap), - // We found some (but not all) switch zones reported by internal DNS. - PartialSuccess(HashMap), -} - -/// Code for configuring the necessary network bits to bring up the control -/// plane -pub struct EarlyNetworkSetup<'a> { - log: &'a Logger, -} - -/// `EarlyNetworkSetup` will be deleted soon as a part of -/// . -/// -/// It still exists to provide `lookup_uplinked_switch_zone_underlay_addrs()`, -/// which is used when sled-agent starts zones that require NAT entries. This -/// is only "early network" in the sense that it's part of the requirement for -/// cold booting; the name of this type is largely historical, where it -/// contained considerably more functionality to handle additional early rack -/// networking setup. We don't bother with a rename because we're going to -/// delete it soon anyway. -impl<'a> EarlyNetworkSetup<'a> { - pub fn new(log: &'a Logger) -> Self { - EarlyNetworkSetup { log } - } - - /// Dynamically looks up (via internal DNS and queries to MGS) the underlay - /// addresses of the switch zone(s) that have uplinks configured. - /// - /// The `RackNetworkConfig` inside `config_rx` always has at least one uplink - /// port, so there is always at least one switch zone to find: - /// - /// * If the config specifies only one switch with an uplink, blocks until - /// we can find that switch zone's underlay address. - /// * If the config specifies two switches with uplinks, we will block up - /// to the `wait_for_at_least_one` duration trying to find both - /// corresponding switch zones. If we pass the deadline without having - /// found both, we will return as soon after that as we can find one of - /// the switch zone's addresses. - pub async fn lookup_uplinked_switch_zone_underlay_addrs( - &self, - resolver: &DnsResolver, - config_rx: &watch::Receiver, - wait_for_at_least_one: Duration, - ) -> HashMap { - let query_start = Instant::now(); - let uplinked_switch_zone_addrs = retry_notify( - retry_policy_switch_mapping(), - || async { - // Which switches have configured ports? The list of ports is - // guaranteed to be non-empty by the type system, so this set - // always has at least one switch. - let uplinked_switches = config_rx - .borrow() - .rack_network_config - .ports - .iter() - .map(|port_config| port_config.switch) - .collect::>(); - - match self - .lookup_switch_zone_underlay_addrs_one_attempt( - resolver, - &uplinked_switches, - ) - .await? - { - LookupSwitchZoneAddrsResult::TotalSuccess(map) => { - info!( - self.log, - "Successfully looked up all expected switch zone \ - underlay addresses"; - "addrs" => ?map, - ); - Ok(map) - } - LookupSwitchZoneAddrsResult::PartialSuccess(map) => { - let elapsed = query_start.elapsed(); - if elapsed >= wait_for_at_least_one { - // We only found one switch when we are expecting - // two, but we've been waiting for too long: go with - // just one. - warn!( - self.log, - "Only found one switch (expected two), \ - but passed requested wait time: returning"; - "switch_found" => ?map, - "requested_wait_time" => ?wait_for_at_least_one, - "total_elapsed" => ?elapsed, - ); - Ok(map) - } else { - // We only found one switch when we are expecting - // two; retry after a backoff. Our logging closure - // below will `warn!` with this error. - Err(BackoffError::transient(format!( - "Only found one switch (expected two): {map:?}" - ))) - } - } - } - }, - |error, delay| { - let elapsed = query_start.elapsed(); - warn!( - self.log, - "Failed to look up switch zone slots"; - "error" => #%error, - "retry_after" => ?delay, - "requested_wait_time" => ?wait_for_at_least_one, - "total_elapsed" => ?elapsed, - ); - }, - ) - .await - .expect("Expected an infinite retry loop finding switch zones"); - - assert!( - !uplinked_switch_zone_addrs.is_empty(), - "retry loop found at least one switch zone" - ); - - uplinked_switch_zone_addrs - } - - // TODO: #3601 Audit switch slot discovery logic for robustness - // in multi-rack deployments. Query MGS servers in each switch zone to - // determine which switch slot they are managing. This logic does not handle - // an event where there are multiple racks. Is that ok? - async fn lookup_switch_zone_underlay_addrs_one_attempt( - &self, - resolver: &DnsResolver, - switches_to_find: &HashSet, - ) -> Result> { - // We should only be called with a nonempty `switches_to_find`; - // otherwise we'll never return: we always want to find at least one - // of these switches. - assert!(!switches_to_find.is_empty()); - - // We might have stale DNS results; clear our resolver's cache. - resolver.clear_cache(); - - info!(self.log, "Resolving switch zone addresses in DNS"); - let switch_zone_addrs = resolver - .lookup_all_ipv6(ServiceName::Dendrite) - .await - .map_err(|err| { - BackoffError::transient(format!( - "Error resolving dendrite services in internal DNS: {err}", - )) - })?; - - let mgs_query_futures = - switch_zone_addrs.iter().copied().map(|addr| async move { - let mgs_client = MgsClient::new( - &format!("http://[{}]:{}", addr, MGS_PORT), - self.log.new(o!("component" => "MgsClient")), - ); - - info!( - self.log, "Querying MGS to determine switch slot"; - "addr" => %addr, - ); - let switch_slot = mgs_client - .sp_local_switch_id() - .await - .with_context(|| format!("Failed to query MGS at {addr}"))? - .into_inner() - .slot; - - match switch_slot { - 0 => Ok((SwitchSlot::Switch0, addr)), - 1 => Ok((SwitchSlot::Switch1, addr)), - _ => Err(anyhow!( - "Nonsense switch slot returned by MGS at \ - {addr}: {switch_slot}" - )), - } - }); - - let mut switch_slot_map = HashMap::new(); - for mgs_query_result in future::join_all(mgs_query_futures).await { - match mgs_query_result { - Ok((switch_slot, addr)) => { - info!(self.log, "Found {switch_slot:?} at {addr}"); - switch_slot_map.insert(switch_slot, addr); - } - Err(err) => { - warn!(self.log, "{err:#}"); - } - } - } - - // Filter `switch_slot_map` down to just the ones we care about, - // and then return total/partial/no success based on what's left. - switch_slot_map.retain(|switch_slot, _addr| { - switches_to_find.contains(switch_slot) - }); - - if switch_slot_map.is_empty() { - Err(BackoffError::transient("No switch slots found".to_string())) - } else if switch_slot_map.len() == switches_to_find.len() { - Ok(LookupSwitchZoneAddrsResult::TotalSuccess(switch_slot_map)) - } else { - Ok(LookupSwitchZoneAddrsResult::PartialSuccess(switch_slot_map)) - } - } -} - -// This is derived from `retry_policy_internal_service_aggressive` with a -// much lower `max_interval`, because we are not going to retry forever: once we -// pass `MAX_SWITCH_ZONE_WAIT_TIME` we will stop as soon as we can talk to _any_ -// switch zone (whereas we stop earlier if we can talk to _all_ switch zones). -fn retry_policy_switch_mapping() -> ExponentialBackoff { - ExponentialBackoffBuilder::new() - .with_initial_interval(Duration::from_millis(100)) - .with_multiplier(1.2) - .with_max_interval(Duration::from_secs(15)) - .with_max_elapsed_time(None) - .build() -} diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index f88b6d975c0..b273025d380 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -32,7 +32,6 @@ use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_DIR; use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_FILE; use clickhouse_admin_types::CLICKHOUSE_SERVER_CONFIG_DIR; use clickhouse_admin_types::CLICKHOUSE_SERVER_CONFIG_FILE; -use dpd_client::{Client as DpdClient, Error as DpdError, types as DpdTypes}; use dropshot::HandlerTaskMode; use illumos_utils::addrobj::AddrObject; use illumos_utils::addrobj::IPV6_LINK_LOCAL_ADDROBJ_NAME; @@ -59,10 +58,8 @@ use internal_dns_types::names::DNS_ZONE; use nexus_config::{ConfigDropshotWithTls, DeploymentConfig}; use omicron_common::address::AZ_PREFIX_LENGTH; use omicron_common::address::BOOTSTRAP_AGENT_LOCKSTEP_PORT; -use omicron_common::address::ConcreteIp; use omicron_common::address::DENDRITE_PORT; use omicron_common::address::LLDP_PORT; -use omicron_common::address::MAX_PORT; use omicron_common::address::MGS_PORT; use omicron_common::address::NTP_ADMIN_PORT; use omicron_common::address::RACK_PREFIX_LENGTH; @@ -79,19 +76,14 @@ use omicron_common::address::{ use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT}; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::{PrivateIpConfig, SledIdentifiers}; -use omicron_common::backoff::{ - BackoffError, retry_notify, retry_policy_internal_service_aggressive, -}; use omicron_common::disk::{DatasetKind, DatasetName}; use omicron_ddm_admin_client::DdmError; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::RackUuid; -use sled_agent_early_networking::EarlyNetworkSetup; use sled_agent_resolvable_files::{ ZoneImageSourceResolver, ramdisk_file_source, }; use sled_agent_types::instance::ExternalIpConfig; -use sled_agent_types::instance::ExternalIps; use sled_agent_types::inventory::{ OmicronZoneConfig, OmicronZoneType, ZoneKind, }; @@ -99,7 +91,6 @@ use sled_agent_types::resolvable_files::{ MupdateOverrideReadError, PreparedOmicronZone, }; use sled_agent_types::sled::ThisSledSwitchZoneUnderlayIpAddr; -use sled_agent_types::system_networking::SystemNetworkingConfig; use sled_hardware::DendriteAsic; use sled_hardware::SledMode; use sled_hardware::underlay; @@ -113,7 +104,6 @@ use std::time::Duration; use tokio::io::AsyncWriteExt; use tokio::sync::Mutex; use tokio::sync::oneshot; -use tokio::sync::watch; use tokio::task::JoinHandle; use tufaceous_artifact::ArtifactHash; use uuid::Uuid; @@ -210,9 +200,6 @@ pub enum Error { err: Box, }, - #[error("Error contacting dpd")] - DpdError(#[from] DpdError), - #[error("Failed to create Vnic in the switch zone")] SwitchZoneVnicCreation(#[source] illumos_utils::dladm::CreateVnicError), @@ -624,11 +611,9 @@ pub struct ServiceManagerInner { pub(crate) struct SledAgentInfo { pub(crate) config: Config, pub(crate) port_manager: PortManager, - pub(crate) resolver: Resolver, pub(crate) underlay_address: Ipv6Addr, pub(crate) local_switch_zone_ip: ThisSledSwitchZoneUnderlayIpAddr, pub(crate) rack_id: RackUuid, - pub(crate) network_config_rx: watch::Receiver, pub(crate) metrics_queue: MetricsRequestQueue, } @@ -637,100 +622,6 @@ pub struct ServiceManager { inner: Arc, } -/// Ensure that a NAT entry exists, overwriting a previous conflicting entry if -/// applicable. -/// -/// nat_ipv\[46\]_create are not idempotent (see oxidecomputer/dendrite#343), -/// but this wrapper function is. Call this from sagas instead. -#[allow(clippy::too_many_arguments)] -async fn dpd_ensure_nat_entry( - client: &DpdClient, - log: &Logger, - target_ip: IpAddr, - target_mac: DpdTypes::MacAddr, - target_first_port: u16, - target_last_port: u16, - target_vni: u32, - sled_ip_address: &std::net::Ipv6Addr, -) -> Result<(), Error> { - let existing_nat = match &target_ip { - IpAddr::V4(ip) => client.nat_ipv4_get(ip, target_first_port).await, - IpAddr::V6(ip) => client.nat_ipv6_get(ip, target_first_port).await, - }; - - // If a NAT entry already exists, but has the wrong internal - // IP address, delete the old entry before continuing (the - // DPD entry-creation API won't replace an existing entry). - // If the entry exists and has the right internal IP, there's - // no more work to do for this external IP. - match existing_nat { - Ok(existing) => { - let existing = existing.into_inner(); - if existing.internal_ip != *sled_ip_address { - info!(log, "deleting old nat entry"; - "target_ip" => ?target_ip); - - match &target_ip { - IpAddr::V4(ip) => { - client.nat_ipv4_delete(ip, target_first_port).await - } - IpAddr::V6(ip) => { - client.nat_ipv6_delete(ip, target_first_port).await - } - }?; - } else { - info!(log, - "nat entry with expected internal ip exists"; - "target_ip" => ?target_ip, - "existing_entry" => ?existing); - - return Ok(()); - } - } - Err(e) => { - if e.status() == Some(http::StatusCode::NOT_FOUND) { - info!(log, "no nat entry found for: {target_ip:#?}"); - } else { - return Err(Error::DpdError(e)); - } - } - } - - info!(log, "creating nat entry for: {target_ip:#?}"); - let nat_target = DpdTypes::NatTarget { - inner_mac: target_mac, - internal_ip: *sled_ip_address, - vni: target_vni.into(), - }; - - match &target_ip { - IpAddr::V4(ip) => { - client - .nat_ipv4_create( - ip, - target_first_port, - target_last_port, - &nat_target, - ) - .await - } - IpAddr::V6(ip) => { - client - .nat_ipv6_create( - ip, - target_first_port, - target_last_port, - &nat_target, - ) - .await - } - }?; - - info!(log, "creation of nat entry successful for: {target_ip:#?}"); - - Ok(()) -} - impl ServiceManager { /// Creates a service manager. /// @@ -1046,32 +937,6 @@ impl ServiceManager { &self, zone_args: &ZoneArgs<'_>, ) -> Result, Error> { - // As a part of setting up OPTE ports, we notify dendrite on all - // switches that have an uplink about the new required NAT entries. This - // requires finding the switch zone IP addresses. We currently block - // until either: - // - // 1. We find all switch zone IPs. - // 2. We find at least one switch zone IP and this timeout elapses. - // - // This should be unnecessary: the scrimlets now run reconcilers that - // set up NAT entries for all services on the rack. We keep it here for - // now for update safety to the release in which we shipped said - // reconcilers; we still need sled-agent to be able to set up NAT - // entries on its own in some (unlikely) scenarios where a rack loses - // power partway through an update. This function will be removed once - // we've shipped the release containing the reconcilers. - // - // The choice of timeout here is a tension between wanting to wait for - // both switches and not wanting to block zone startup indefinitely if - // one of the scrimlets or switches is unavailable for an extended - // period of time. We'll pick 5 minutes, which has historically been the - // timeout here and should hopefully give enough time for a "just - // rebooted" scrimlet to bring its switch zone up, if we get unlucky in - // coincidental timings. - const WAIT_FOR_ALL_SWITCH_ZONES_TIMEOUT: Duration = - Duration::from_secs(5 * 60); - if !matches!( zone_args.omicron_type(), Some(OmicronZoneType::ExternalDns { .. }) @@ -1081,37 +946,12 @@ impl ServiceManager { return Ok(vec![]); } - let SledAgentInfo { - port_manager, - underlay_address, - resolver, - network_config_rx, - .. - } = &self.inner.sled_info.get().ok_or(Error::SledAgentNotReady)?; - - let uplinked_switch_zone_addrs = - EarlyNetworkSetup::new(&self.inner.log) - .lookup_uplinked_switch_zone_underlay_addrs( - resolver, - network_config_rx, - WAIT_FOR_ALL_SWITCH_ZONES_TIMEOUT, - ) - .await; - - let dpd_clients: Vec = uplinked_switch_zone_addrs - .values() - .map(|addr| { - DpdClient::new( - &format!("http://[{}]:{}", addr, DENDRITE_PORT), - dpd_client::ClientState { - tag: "sled-agent".to_string(), - log: self.inner.log.new(o!( - "component" => "DpdClient" - )), - }, - ) - }) - .collect(); + let port_manager = &self + .inner + .sled_info + .get() + .ok_or(Error::SledAgentNotReady)? + .port_manager; let (zone_kind, nic, external_ips) = match &zone_args.omicron_type() { Some( @@ -1190,49 +1030,7 @@ impl ServiceManager { service: zone_kind, err: Box::new(err), })?; - let nat_data = extract_nat_data_for_external_ip_config(&external_ips); - - for dpd_client in &dpd_clients { - // TODO-correctness(#2933): If we fail part-way we need to - // clean up previous entries instead of leaking them. - let nat_create = || async { - info!( - self.inner.log, "creating NAT entry for service"; - "zone_type" => zone_kind.report_str(), - ); - for data in nat_data.iter() { - dpd_ensure_nat_entry( - dpd_client, - &self.inner.log, - data.ip, - dpd_client::types::MacAddr { - a: port.0.mac().into_array(), - }, - data.first_port, - data.last_port, - port.0.vni().as_u32(), - underlay_address, - ) - .await - .map_err(BackoffError::::transient)?; - } - Ok::<(), BackoffError>(()) - }; - let log_failure = |error, _| { - warn!( - self.inner.log, "failed to create NAT entry for service"; - InlineErrorChain::new(&error), - "zone_type" => zone_kind.report_str(), - ); - }; - retry_notify( - retry_policy_internal_service_aggressive(), - nat_create, - log_failure, - ) - .await?; - } Ok(vec![port]) } @@ -3995,59 +3793,6 @@ impl ServiceManager { } } -struct NatData { - ip: IpAddr, - first_port: u16, - last_port: u16, -} - -// Construct a list of IP address and port-ranges needed to update -// Dendrite wtih the NAT mappings. This handles dual-stack and mulitple -// addresses. -fn extract_nat_data_for_external_ip_config( - external_ips: &ExternalIpConfig, -) -> Vec { - let mut nat_data = Vec::new(); - if let Some(cfg) = external_ips.v4.as_ref() { - nat_data - .append(&mut extract_nat_data_for_concrete_external_ip_config(cfg)); - } - if let Some(cfg) = external_ips.v6.as_ref() { - nat_data - .append(&mut extract_nat_data_for_concrete_external_ip_config(cfg)); - } - nat_data -} - -fn extract_nat_data_for_concrete_external_ip_config( - cfg: &ExternalIps, -) -> Vec { - let mut nat_data = Vec::new(); - if let Some(snat) = cfg.source_nat.as_ref() { - let (first_port, last_port) = snat.port_range_raw(); - nat_data.push(NatData { - ip: snat.ip.into_ipaddr(), - first_port, - last_port, - }); - } - if let Some(ip) = cfg.ephemeral_ip.as_ref() { - nat_data.push(NatData { - ip: ip.into_ipaddr(), - first_port: 0, - last_port: MAX_PORT, - }); - } - for ip in cfg.floating_ips.iter() { - nat_data.push(NatData { - ip: ip.into_ipaddr(), - first_port: 0, - last_port: MAX_PORT, - }); - } - nat_data -} - fn internal_dns_addrobj_name(gz_address_index: u32) -> String { format!("internaldns{gz_address_index}") } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 5df84469d1d..9dd57485026 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -44,7 +44,6 @@ use illumos_utils::zfs::SizeDetails; use illumos_utils::zfs::Zfs; use illumos_utils::zpool::PathInPool; use illumos_utils::zpool::ZpoolOrRamdisk; -use internal_dns_resolver::Resolver; use itertools::Itertools as _; use omicron_common::address::BOOTSTRAP_AGENT_RACK_INIT_PORT; use omicron_common::address::{ @@ -698,7 +697,7 @@ impl SledAgent { long_running_task_handles .scrimlet_reconcilers .set_sled_agent_networking_info_once(SledAgentNetworkingInfo { - system_networking_config_rx: network_config_rx.clone(), + system_networking_config_rx: network_config_rx, mode: ScrimletReconcilersMode::SwitchZone( this_sled_switch_zone_ip, ), @@ -722,14 +721,9 @@ impl SledAgent { .sled_agent_started(SledAgentInfo { config: svc_config, port_manager: port_manager.clone(), - resolver: Resolver::new_from_ip( - parent_log.new(o!("component" => "DnsResolver")), - *sled_address.ip(), - )?, underlay_address: *sled_address.ip(), local_switch_zone_ip: this_sled_switch_zone_ip, rack_id: request.body.rack_id, - network_config_rx, metrics_queue: metrics_manager.request_queue(), }) .await?;