[nexus][sled-agent] Use IPv4 addresses for Nexus' external interface

docs/how-to-run.adoc

@@ -113,22 +113,30 @@ we'll assign addresses as per RFD 63 as well as incorporating DNS based
 service discovery.
 
 For the purposes of local development today, we specify some hardcoded IPv6
-unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`:
+unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`.
+
+If you'd like to modify these values to suit your local network, you can modify
+them within the https://github.com/oxidecomputer/omicron/tree/main/smf[`smf/` subdirectory].
+Notably, Nexus is being served from IPv4 address, which may be configured to be
+external. By default, it uses a private IPv4 address and no Internet gateway, but may
+be configured to use a public-facing IP address with an Internet gateway that may
+be set as a default route for the Nexus zone.
 
 [options="header"]
 |===================================================================================================
 | Service                    | Endpoint
 | Sled Agent: Bootstrap      | Derived from MAC address of physical data link.
 | Sled Agent: Dropshot API   | `[fd00:1122:3344:0101::1]:12345`
 | Cockroach DB               | `[fd00:1122:3344:0101::2]:32221`
-| Nexus: External API        | `[fd00:1122:3344:0101::3]:12220`
 | Nexus: Internal API        | `[fd00:1122:3344:0101::3]:12221`
 | Oximeter                   | `[fd00:1122:3344:0101::4]:12223`
 | Clickhouse                 | `[fd00:1122:3344:0101::5]:8123`
 | Crucible Downstairs 1      | `[fd00:1122:3344:0101::6]:32345`
 | Crucible Downstairs 2      | `[fd00:1122:3344:0101::7]:32345`
 | Crucible Downstairs 3      | `[fd00:1122:3344:0101::8]:32345`
 | Internal DNS Service       | `[fd00:1122:3344:0001::1]:5353`
+| Nexus: External API        | `192.168.1.20:80`
+| Internet Gateway           | None, but can be set in `smf/sled-agent/config.toml`
 |===================================================================================================
 
 Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other

sled-agent/src/bootstrap/agent.rs

@@ -150,15 +150,15 @@ impl Agent {
                 err,
             })?;
 
-        let etherstub = Dladm::create_etherstub().map_err(|e| {
+        let etherstub = Dladm::ensure_etherstub().map_err(|e| {
             BootstrapError::SledError(format!(
                 "Can't access etherstub device: {}",
                 e
             ))
         })?;
 
         let etherstub_vnic =
-            Dladm::create_etherstub_vnic(&etherstub).map_err(|e| {
+            Dladm::ensure_etherstub_vnic(&etherstub).map_err(|e| {
                 BootstrapError::SledError(format!(
                     "Can't access etherstub VNIC device: {}",
                     e

sled-agent/src/config.rs

@@ -9,7 +9,7 @@ use crate::illumos::dladm::{self, Dladm, PhysicalLink};
 use crate::illumos::zpool::ZpoolName;
 use dropshot::ConfigLogging;
 use serde::Deserialize;
-use std::net::SocketAddr;
+use std::net::{Ipv4Addr, SocketAddr};
 use std::path::{Path, PathBuf};
 use uuid::Uuid;
 
@@ -27,6 +27,10 @@ pub struct Config {
     /// Optional list of zpools to be used as "discovered disks".
     pub zpools: Option<Vec<ZpoolName>>,
 
+    /// Address of the Internet gateway, which is particularly
+    /// relevant for external-facing services (such as Nexus).
+    pub gateway_address: Option<Ipv4Addr>,
+
     /// The data link on which we infer the bootstrap address.
     ///
     /// If unsupplied, we default to the first physical device.

sled-agent/src/illumos/dladm.rs

@@ -124,7 +124,7 @@ pub struct Dladm {}
 #[cfg_attr(test, mockall::automock, allow(dead_code))]
 impl Dladm {
     /// Creates an etherstub, or returns one which already exists.
-    pub fn create_etherstub() -> Result<Etherstub, ExecutionError> {
+    pub fn ensure_etherstub() -> Result<Etherstub, ExecutionError> {
         if let Ok(stub) = Self::get_etherstub() {
             return Ok(stub);
         }
@@ -147,7 +147,7 @@ impl Dladm {
     ///
     /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because
     /// it is expected to exist for the lifetime of the sled.
-    pub fn create_etherstub_vnic(
+    pub fn ensure_etherstub_vnic(
         source: &Etherstub,
     ) -> Result<EtherstubVnic, CreateVnicError> {
         if let Ok(vnic) = Self::get_etherstub_vnic() {

sled-agent/src/illumos/running_zone.rs

@@ -5,13 +5,14 @@
 //! Utilities to manage running zones.
 
 use crate::illumos::addrobj::AddrObject;
+use crate::illumos::dladm::Etherstub;
 use crate::illumos::svc::wait_for_service;
 use crate::illumos::vnic::{Vnic, VnicAllocator};
 use crate::illumos::zone::{AddressRequest, ZONE_PREFIX};
 use crate::opte::OptePort;
 use ipnetwork::IpNetwork;
 use slog::Logger;
-use std::net::Ipv6Addr;
+use std::net::{Ipv4Addr, Ipv6Addr};
 use std::path::PathBuf;
 
 #[cfg(test)]
@@ -153,14 +154,22 @@ impl RunningZone {
         &self,
         addrtype: AddressRequest,
     ) -> Result<IpNetwork, EnsureAddressError> {
-        info!(self.inner.log, "Adding address: {:?}", addrtype);
         let name = match addrtype {
             AddressRequest::Dhcp => "omicron",
             AddressRequest::Static(net) => match net.ip() {
                 std::net::IpAddr::V4(_) => "omicron4",
                 std::net::IpAddr::V6(_) => "omicron6",
             },
         };
+        self.ensure_address_with_name(addrtype, name).await
+    }
+
+    pub async fn ensure_address_with_name(
+        &self,
+        addrtype: AddressRequest,
+        name: &str,
+    ) -> Result<IpNetwork, EnsureAddressError> {
+        info!(self.inner.log, "Adding address: {:?}", addrtype);
         let addrobj = AddrObject::new(self.inner.control_vnic.name(), name)
             .map_err(|err| EnsureAddressError::AddrObject {
                 request: addrtype,
@@ -172,6 +181,31 @@ impl RunningZone {
         Ok(network)
     }
 
+    // TODO: Remove once Nexus uses OPTE - external addresses should generally
+    // be served via OPTE.
+    pub async fn ensure_external_address_with_name(
+        &self,
+        addrtype: AddressRequest,
+        name: &str,
+    ) -> Result<IpNetwork, EnsureAddressError> {
+        info!(self.inner.log, "Adding address: {:?}", addrtype);
+        let addrobj = AddrObject::new(
+                self.inner.physical_nic
+                    .as_ref()
+                    .expect("Cannot allocate external address on zone without physical NIC")
+                    .name(),
+                name
+            )
+            .map_err(|err| EnsureAddressError::AddrObject {
+                request: addrtype,
+                zone: self.inner.name.clone(),
+                err,
+            })?;
+        let network =
+            Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?;
+        Ok(network)
+    }
+
     pub async fn add_default_route(
         &self,
         gateway: Ipv6Addr,
@@ -187,6 +221,19 @@ impl RunningZone {
         Ok(())
     }
 
+    pub async fn add_default_route4(
+        &self,
+        gateway: Ipv4Addr,
+    ) -> Result<(), RunCommandError> {
+        self.run_cmd(&[
+            "/usr/sbin/route",
+            "add",
+            "default",
+            &gateway.to_string(),
+        ])?;
+        Ok(())
+    }
+
     /// Looks up a running zone based on the `zone_prefix`, if one already exists.
     ///
     /// - If the zone was found, is running, and has a network interface, it is
@@ -249,6 +296,7 @@ impl RunningZone {
                 //
                 // Re-initialize guest_vnic state by inspecting the zone.
                 opte_ports: vec![],
+                physical_nic: None,
             },
         })
     }
@@ -301,6 +349,10 @@ pub struct InstalledZone {
 
     // OPTE devices for the guest network interfaces
     opte_ports: Vec<OptePort>,
+
+    // Physical NIC possibly provisioned to the zone.
+    // TODO: Remove once Nexus traffic is transmitted over OPTE.
+    physical_nic: Option<Vnic>,
 }
 
 impl InstalledZone {
@@ -325,14 +377,16 @@ impl InstalledZone {
         zone_name
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub async fn install(
         log: &Logger,
-        vnic_allocator: &VnicAllocator,
+        vnic_allocator: &VnicAllocator<Etherstub>,
         service_name: &str,
         unique_name: Option<&str>,
         datasets: &[zone::Dataset],
         devices: &[zone::Device],
         opte_ports: Vec<OptePort>,
+        physical_nic: Option<Vnic>,
     ) -> Result<InstalledZone, InstallZoneError> {
         let control_vnic = vnic_allocator.new_control(None).map_err(|err| {
             InstallZoneError::CreateVnic {
@@ -349,6 +403,7 @@ impl InstalledZone {
             .iter()
             .map(|port| port.vnic().name().to_string())
             .chain(std::iter::once(control_vnic.name().to_string()))
+            .chain(physical_nic.as_ref().map(|vnic| vnic.name().to_string()))
             .collect();
 
         Zones::install_omicron_zone(
@@ -370,6 +425,7 @@ impl InstalledZone {
             name: zone_name,
             control_vnic,
             opte_ports,
+            physical_nic,
         })
     }
 }

sled-agent/src/illumos/vnic.rs

@@ -5,7 +5,7 @@
 //! API for controlling a single instance.
 
 use crate::illumos::dladm::{
-    CreateVnicError, DeleteVnicError, Etherstub, VNIC_PREFIX,
+    CreateVnicError, DeleteVnicError, VnicSource, VNIC_PREFIX,
     VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST,
 };
 use omicron_common::api::external::MacAddr;
@@ -23,13 +23,13 @@ use crate::illumos::dladm::MockDladm as Dladm;
 /// May be used to allocate runtime-unique IDs for objects
 /// which have naming constraints - such as VNICs.
 #[derive(Clone, Debug)]
-pub struct VnicAllocator {
+pub struct VnicAllocator<DL: VnicSource + 'static> {
     value: Arc<AtomicU64>,
     scope: String,
-    data_link: Etherstub,
+    data_link: DL,
 }
 
-impl VnicAllocator {
+impl<DL: VnicSource + Clone> VnicAllocator<DL> {
     /// Creates a new Vnic name allocator with a particular scope.
     ///
     /// The intent with varying scopes is to create non-overlapping
@@ -41,11 +41,11 @@ impl VnicAllocator {
     ///
     /// VnicAllocator::new("Storage") produces
     /// - oxControlStorage[NNN]
-    pub fn new<S: AsRef<str>>(scope: S, etherstub: Etherstub) -> Self {
+    pub fn new<S: AsRef<str>>(scope: S, data_link: DL) -> Self {
         Self {
             value: Arc::new(AtomicU64::new(0)),
             scope: scope.as_ref().to_string(),
-            data_link: etherstub,
+            data_link,
         }
     }
 
@@ -167,6 +167,7 @@ impl Drop for Vnic {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::illumos::dladm::Etherstub;
 
     #[test]
     fn test_allocate() {

sled-agent/src/illumos/zone.rs

@@ -101,8 +101,8 @@ pub struct EnsureAddressError {
 #[derive(thiserror::Error, Debug)]
 #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")]
 pub struct EnsureGzAddressError {
-    address: Ipv6Addr,
-    link: EtherstubVnic,
+    address: IpAddr,
+    link: String,
     name: String,
     #[source]
     err: anyhow::Error,
@@ -580,8 +580,8 @@ impl Zones {
             Ok(())
         }(link.clone(), address, name)
         .map_err(|err| EnsureGzAddressError {
-            address,
-            link,
+            address: IpAddr::V6(address),
+            link: link.0.clone(),
             name: name.to_string(),
             err,
             extra_note:

sled-agent/src/instance.rs

@@ -5,6 +5,7 @@
 //! API for controlling a single instance.
 
 use crate::common::instance::{Action as InstanceAction, InstanceStates};
+use crate::illumos::dladm::Etherstub;
 use crate::illumos::running_zone::{
     InstalledZone, RunCommandError, RunningZone,
 };
@@ -204,7 +205,7 @@ struct InstanceInner {
     propolis_ip: IpAddr,
 
     // NIC-related properties
-    vnic_allocator: VnicAllocator,
+    vnic_allocator: VnicAllocator<Etherstub>,
 
     // OPTE port related properties
     underlay_addr: Ipv6Addr,
@@ -400,7 +401,7 @@ mockall::mock! {
         pub fn new(
             log: Logger,
             id: Uuid,
-            vnic_allocator: VnicAllocator,
+            vnic_allocator: VnicAllocator<Etherstub>,
             underlay_addr: Ipv6Addr,
             port_allocator: OptePortAllocator,
             initial: InstanceHardware,
@@ -446,7 +447,7 @@ impl Instance {
     pub fn new(
         log: Logger,
         id: Uuid,
-        vnic_allocator: VnicAllocator,
+        vnic_allocator: VnicAllocator<Etherstub>,
         underlay_addr: Ipv6Addr,
         port_allocator: OptePortAllocator,
         initial: InstanceHardware,
@@ -534,6 +535,8 @@ impl Instance {
                 zone::Device { name: "/dev/viona".to_string() },
             ],
             opte_ports,
+            // physical_nic=
+            None,
         )
         .await?;
 

sled-agent/src/instance_manager.rs

@@ -44,7 +44,7 @@ struct InstanceManagerInternal {
     /// A mapping from a Sled Agent "Instance ID" to ("Propolis ID", [Instance]).
     instances: Mutex<BTreeMap<Uuid, (Uuid, Instance)>>,
 
-    vnic_allocator: VnicAllocator,
+    vnic_allocator: VnicAllocator<Etherstub>,
     underlay_addr: Ipv6Addr,
     port_allocator: OptePortAllocator,
 }

sled-agent/src/params.rs

@@ -303,7 +303,7 @@ impl From<DatasetEnsureBody> for sled_agent_client::types::DatasetEnsureBody {
 )]
 #[serde(tag = "type", rename_all = "snake_case")]
 pub enum ServiceType {
-    Nexus { internal_address: SocketAddrV6, external_address: SocketAddrV6 },
+    Nexus { internal_address: SocketAddrV6, external_address: SocketAddr },
     InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 },
     Oximeter,
 }