[nexus][sled-agent] Use IPv4 addresses for Nexus' external interface

common/src/sql/dbwipe.sql

@@ -8,5 +8,15 @@
  * NOTE: the database and user names MUST be kept in sync with the
  * initialization code and dbwipe.sql.
  */
+
+/*
+ * This is silly, but we throw an error if the user was already deleted.
+ * Create the user so we can always delete it.
+ */
+CREATE DATABASE IF NOT EXISTS omicron;
+CREATE USER IF NOT EXISTS omicron;
+
+ALTER DEFAULT PRIVILEGES FOR ROLE root REVOKE ALL ON TABLES FROM omicron;
+
 DROP DATABASE IF EXISTS omicron;
 DROP USER IF EXISTS omicron;

sled-agent/src/illumos/running_zone.rs

@@ -11,7 +11,7 @@ use crate::illumos::zone::{AddressRequest, ZONE_PREFIX};
 use crate::opte::OptePort;
 use ipnetwork::IpNetwork;
 use slog::Logger;
-use std::net::Ipv6Addr;
+use std::net::{Ipv4Addr, Ipv6Addr};
 use std::path::PathBuf;
 
 #[cfg(test)]
@@ -153,14 +153,22 @@ impl RunningZone {
         &self,
         addrtype: AddressRequest,
     ) -> Result<IpNetwork, EnsureAddressError> {
-        info!(self.inner.log, "Adding address: {:?}", addrtype);
         let name = match addrtype {
             AddressRequest::Dhcp => "omicron",
             AddressRequest::Static(net) => match net.ip() {
                 std::net::IpAddr::V4(_) => "omicron4",
                 std::net::IpAddr::V6(_) => "omicron6",
             },
         };
+        self.ensure_address_with_name(addrtype, name).await
+    }
+
+    pub async fn ensure_address_with_name(
+        &self,
+        addrtype: AddressRequest,
+        name: &str,
+    ) -> Result<IpNetwork, EnsureAddressError> {
+        info!(self.inner.log, "Adding address: {:?}", addrtype);
         let addrobj = AddrObject::new(self.inner.control_vnic.name(), name)
             .map_err(|err| EnsureAddressError::AddrObject {
                 request: addrtype,
@@ -187,6 +195,19 @@ impl RunningZone {
         Ok(())
     }
 
+    pub async fn add_default_route4(
+        &self,
+        gateway: Ipv4Addr,
+    ) -> Result<(), RunCommandError> {
+        self.run_cmd(&[
+            "/usr/sbin/route",
+            "add",
+            "default",
+            &gateway.to_string(),
+        ])?;
+        Ok(())
+    }
+
     /// Looks up a running zone based on the `zone_prefix`, if one already exists.
     ///
     /// - If the zone was found, is running, and has a network interface, it is

sled-agent/src/illumos/zone.rs

@@ -7,7 +7,7 @@
 use anyhow::anyhow;
 use ipnetwork::IpNetwork;
 use slog::Logger;
-use std::net::{IpAddr, Ipv6Addr};
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 
 use crate::illumos::addrobj::AddrObject;
 use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL};
@@ -101,7 +101,7 @@ pub struct EnsureAddressError {
 #[derive(thiserror::Error, Debug)]
 #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")]
 pub struct EnsureGzAddressError {
-    address: Ipv6Addr,
+    address: IpAddr,
     link: EtherstubVnic,
     name: String,
     #[source]
@@ -541,6 +541,37 @@ impl Zones {
         Ok(())
     }
 
+    // TODO: Remove once Nexus traffic is transmitted over OPTE.
+    pub fn ensure_has_global_zone_v4_address(
+        link: EtherstubVnic,
+        address: Ipv4Addr,
+        name: &str,
+    ) -> Result<(), EnsureGzAddressError> {
+        // Call the guts of this function within a closure to make it easier
+        // to wrap the error with appropriate context.
+        |link: EtherstubVnic, address, name| -> Result<(), anyhow::Error> {
+            // Ensure that a static IPv4 address has been allocated
+            // to the Global Zone. Without this, we don't have a way
+            // to route to IP addresses that we want to create in
+            // the non-GZ.
+            Self::ensure_address(
+                None,
+                &AddrObject::new(&link.0, name).unwrap(),
+                AddressRequest::new_static(IpAddr::V4(address), None),
+            )
+            .map_err(|err| anyhow!(err))?;
+            Ok(())
+        }(link.clone(), address, name)
+        .map_err(|err| EnsureGzAddressError {
+            address: IpAddr::V4(address),
+            link,
+            name: name.to_string(),
+            err,
+            extra_note: "".to_string(),
+        })?;
+        Ok(())
+    }
+
     // TODO(https://github.com/oxidecomputer/omicron/issues/821): We
     // should remove this function when Sled Agents are provided IPv6 addresses
     // from RSS.
@@ -580,7 +611,7 @@ impl Zones {
             Ok(())
         }(link.clone(), address, name)
         .map_err(|err| EnsureGzAddressError {
-            address,
+            address: IpAddr::V6(address),
             link,
             name: name.to_string(),
             err,

sled-agent/src/params.rs

@@ -303,7 +303,7 @@ impl From<DatasetEnsureBody> for sled_agent_client::types::DatasetEnsureBody {
 )]
 #[serde(tag = "type", rename_all = "snake_case")]
 pub enum ServiceType {
-    Nexus { internal_address: SocketAddrV6, external_address: SocketAddrV6 },
+    Nexus { internal_address: SocketAddrV6, external_address: SocketAddr },
     InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 },
     Oximeter,
 }

sled-agent/src/services.rs

@@ -10,6 +10,7 @@ use crate::illumos::running_zone::{InstalledZone, RunningZone};
 use crate::illumos::vnic::VnicAllocator;
 use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT;
 use crate::illumos::zone::AddressRequest;
+use crate::illumos::{execute, PFEXEC};
 use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType};
 use crate::zone::Zones;
 use dropshot::ConfigDropshot;
@@ -58,6 +59,9 @@ pub enum Error {
         err: crate::illumos::running_zone::RunCommandError,
     },
 
+    #[error("Installing GZ route to IPv4 address: {0}")]
+    GzIpv4Route(crate::illumos::ExecutionError),
+
     #[error("Failed to boot zone: {0}")]
     ZoneBoot(#[from] crate::illumos::running_zone::BootError),
 
@@ -254,7 +258,7 @@ impl ServiceManager {
                 &[],
                 // devices=
                 &[],
-                // vnics=
+                // opte_ports=
                 vec![],
             )
             .await?;
@@ -305,7 +309,7 @@ impl ServiceManager {
                 //
                 // This is currently being used for the DNS service.
                 //
-                // TODO: consider limitng the number of GZ addresses which
+                // TODO: consider limiting the number of GZ addresses which
                 // can be supplied - now that we're actively using it, we
                 // aren't really handling the "many GZ addresses" case, and it
                 // doesn't seem necessary now.
@@ -344,13 +348,89 @@ impl ServiceManager {
                 ServiceType::Nexus { internal_address, external_address } => {
                     info!(self.log, "Setting up Nexus service");
 
+                    // The address of Nexus' external interface is a special
+                    // case; it may be an IPv4 address.
+                    let addr_request =
+                        AddressRequest::new_static(external_address.ip(), None);
+                    running_zone
+                        .ensure_address_with_name(addr_request, "public")
+                        .await?;
+
+                    // TODO: Remove once Nexus traffic is transmitted over OPTE.
+                    match external_address.ip() {
+                        IpAddr::V4(public_addr4) => {
+                            // Create an address which is routable from the GZ's
+                            // Ipv4 address.
+                            let private_addr4 =
+                                crate::sled_agent::get_nexus_private_ipv4(
+                                    self.underlay_address,
+                                );
+                            let addr_request = AddressRequest::new_static(
+                                IpAddr::V4(private_addr4),
+                                None,
+                            );
+                            running_zone
+                                .ensure_address_with_name(
+                                    addr_request,
+                                    "private",
+                                )
+                                .await?;
+
+                            // Create a default route back through the GZ's
+                            // underlay.
+                            let gateway4 = crate::sled_agent::get_gz_ipv4(
+                                self.underlay_address,
+                            );
+                            running_zone
+                                .add_default_route4(gateway4)
+                                .await
+                                .map_err(|err| Error::ZoneCommand {
+                                    intent: "Adding Route".to_string(),
+                                    err,
+                                })?;
+
+                            // Add a route from the GZ to this zone, using the
+                            // private address as a gateway.
+                            let mut command =
+                                std::process::Command::new(PFEXEC);
+                            let cmd = command.args(&[
+                                "/usr/sbin/route",
+                                "add",
+                                &public_addr4.to_string(),
+                                &private_addr4.to_string(),
+                            ]);
+
+                            execute(cmd)
+                                .map(|_| ())
+                                .or_else(|err| {
+                                    // If the command failed because the entry
+                                    // already exists in the global zone, we're
+                                    // good to continue.
+                                    match err {
+                                        crate::illumos::ExecutionError::CommandFailure {
+                                            ref stdout,
+                                            ..
+                                        } => {
+                                            if stdout.contains("entry exists") {
+                                                return Ok(());
+                                            }
+                                        }
+                                        _ => (),
+                                    }
+                                    return Err(err);
+                                })
+                                .map_err(|err| Error::GzIpv4Route(err))?;
+                        }
+                        _ => (),
+                    }
+
                     // Nexus takes a separate config file for parameters which
                     // cannot be known at packaging time.
                     let deployment_config = NexusDeploymentConfig {
                         id: service.id,
                         rack_id: self.rack_id,
                         dropshot_external: ConfigDropshot {
-                            bind_address: SocketAddr::V6(external_address),
+                            bind_address: external_address,
                             request_body_max_bytes: 1048576,
                             ..Default::default()
                         },
@@ -606,7 +686,7 @@ mod test {
         svc,
         zone::MockZones,
     };
-    use std::net::{Ipv6Addr, SocketAddrV6};
+    use std::net::Ipv6Addr;
     use std::os::unix::process::ExitStatusExt;
     use uuid::Uuid;
 
@@ -635,12 +715,20 @@ mod test {
             assert_eq!(name, EXPECTED_ZONE_NAME);
             Ok(())
         });
+
+        // Ensure the address exists
+        let ensure_address_ctx = MockZones::ensure_address_context();
+        ensure_address_ctx.expect().return_once(|_, _, _| {
+            Ok(ipnetwork::IpNetwork::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 64)
+                .unwrap())
+        });
+
         // Wait for the networking service.
         let wait_ctx = svc::wait_for_service_context();
         wait_ctx.expect().return_once(|_, _| Ok(()));
         // Import the manifest, enable the service
         let execute_ctx = crate::illumos::execute_context();
-        execute_ctx.expect().times(3).returning(|_| {
+        execute_ctx.expect().times(..).returning(|_| {
             Ok(std::process::Output {
                 status: std::process::ExitStatus::from_raw(0),
                 stdout: vec![],
@@ -652,6 +740,7 @@ mod test {
             Box::new(create_vnic_ctx),
             Box::new(install_ctx),
             Box::new(boot_ctx),
+            Box::new(ensure_address_ctx),
             Box::new(wait_ctx),
             Box::new(execute_ctx),
         ]
@@ -665,22 +754,9 @@ mod test {
             services: vec![ServiceRequest {
                 id,
                 name: SVC_NAME.to_string(),
-                addresses: vec![],
+                addresses: vec![Ipv6Addr::LOCALHOST],
                 gz_addresses: vec![],
-                service_type: ServiceType::Nexus {
-                    internal_address: SocketAddrV6::new(
-                        Ipv6Addr::LOCALHOST,
-                        0,
-                        0,
-                        0,
-                    ),
-                    external_address: SocketAddrV6::new(
-                        Ipv6Addr::LOCALHOST,
-                        0,
-                        0,
-                        0,
-                    ),
-                },
+                service_type: ServiceType::Oximeter,
             }],
         })
         .await
@@ -694,22 +770,9 @@ mod test {
             services: vec![ServiceRequest {
                 id,
                 name: SVC_NAME.to_string(),
-                addresses: vec![],
+                addresses: vec![Ipv6Addr::LOCALHOST],
                 gz_addresses: vec![],
-                service_type: ServiceType::Nexus {
-                    internal_address: SocketAddrV6::new(
-                        Ipv6Addr::LOCALHOST,
-                        0,
-                        0,
-                        0,
-                    ),
-                    external_address: SocketAddrV6::new(
-                        Ipv6Addr::LOCALHOST,
-                        0,
-                        0,
-                        0,
-                    ),
-                },
+                service_type: ServiceType::Oximeter,
             }],
         })
         .await