Initial integration with Oxide Packet Transformation Engine

- Brings in OPTE via the `opte-ioctl` and `opte` crates.
- Modifies the instance-ensure request from Nexus to the sled agent, to
  carry the actual information required for setting up the guest OPTE
  port. This includes the actual IP subnet and MAC, rather than things
  like the VPC Subnet UUID.
- Adds a database query and method to extract the above information from
  both the network interface and VPC subnet tables.
- Adds OPTE port for the guests (and currently still a VNIC on top),
  with the right OPTE settings for traffic to flow between two guests in
  the same VPC subnet. That's the virtual-to-physical mapping and a
  router entry for the subnet.
- Adds the VNICs over each OPTE port to the running zone. Note that this
  removes the specification of guest NICs for the zone itself as VNICs.
  They are passed as OPTE ports, and the VNIC is pulled out internally,
  so hopefully little will need to change when the VNIC is removed
  entirely.
- Store the main underlay address for the sled agent, currently its
  dropshot server IP address, in the instance manager, and forward to
  each instance. It's then used as the underlay address when setting up
  the OPTE ports for the guest.
- Fixes bad naming of VNICs used to emulate Chelsio NICs in the current
  setup. Removes the unnecessary loopback address, since the sled agent
  provides its main listening address as the underlay address when
  create OPTE ports.
- Adds better installation of OPTE, xde, and the kernel bits they rely
  on. Don't install `opteadm` or `xde` directly, do everything through
  their package repos.

Author: bnaecker

Cargo.lock

@@ -1168,6 +1168,15 @@ impl FromStr for IpNet {
     }
 }
 
+impl From<IpNet> for ipnetwork::IpNetwork {
+    fn from(net: IpNet) -> ipnetwork::IpNetwork {
+        match net {
+            IpNet::V4(net) => ipnetwork::IpNetwork::from(net.0),
+            IpNet::V6(net) => ipnetwork::IpNetwork::from(net.0),
+        }
+    }
+}
+
 /// A `RouteTarget` describes the possible locations that traffic matching a
 /// route destination can be sent.
 #[derive(

common/src/api/external/mod.rs

@@ -64,6 +64,7 @@ use diesel::query_dsl::methods::LoadQuery;
 use diesel::upsert::excluded;
 use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
 use omicron_common::api;
+use omicron_common::api::external;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
@@ -76,6 +77,7 @@ use omicron_common::api::external::{
     CreateResult, IdentityMetadataCreateParams,
 };
 use omicron_common::bail_unless;
+use sled_agent_client::types as sled_client_types;
 use std::convert::{TryFrom, TryInto};
 use std::net::Ipv6Addr;
 use std::sync::Arc;
@@ -1321,6 +1323,83 @@ impl DataStore {
         Ok(())
     }
 
+    /// Return the information about an instance's network interfaces required
+    /// for the sled agent to instantiate it via OPTE.
+    ///
+    /// OPTE requires information that's currently split across the network
+    /// interface and VPC subnet tables. This query just joins those for each
+    /// NIC in the given instance.
+    pub(crate) async fn derive_guest_network_interface_info(
+        &self,
+        opctx: &OpContext,
+        authz_instance: &authz::Instance,
+    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+        opctx.authorize(authz::Action::ListChildren, authz_instance).await?;
+
+        use db::schema::network_interface;
+        use db::schema::vpc_subnet;
+
+        // The record type for the results of the below JOIN query
+        #[derive(Debug, diesel::Queryable)]
+        struct NicInfo {
+            name: db::model::Name,
+            ip: ipnetwork::IpNetwork,
+            mac: db::model::MacAddr,
+            ipv4_block: db::model::Ipv4Net,
+            ipv6_block: db::model::Ipv6Net,
+            vpc_id: Uuid,
+            slot: i16,
+        }
+
+        impl From<NicInfo> for sled_client_types::NetworkInterface {
+            fn from(nic: NicInfo) -> sled_client_types::NetworkInterface {
+                let ip_subnet = if nic.ip.is_ipv4() {
+                    external::IpNet::V4(nic.ipv4_block.0)
+                } else {
+                    external::IpNet::V6(nic.ipv6_block.0)
+                };
+                sled_client_types::NetworkInterface {
+                    name: sled_client_types::Name::from(&nic.name.0),
+                    ip: nic.ip.ip().to_string(),
+                    mac: sled_client_types::MacAddr::from(nic.mac.0),
+                    subnet: sled_client_types::IpNet::from(ip_subnet),
+                    vpc_id: nic.vpc_id,
+                    slot: u8::try_from(nic.slot).unwrap(),
+                }
+            }
+        }
+
+        let rows = network_interface::table
+            .filter(network_interface::instance_id.eq(authz_instance.id()))
+            .filter(network_interface::time_deleted.is_null())
+            .inner_join(
+                vpc_subnet::table
+                    .on(network_interface::subnet_id.eq(vpc_subnet::id)),
+            )
+            .order_by(network_interface::slot)
+            // TODO-cleanup: Having to specify each column again is less than
+            // ideal. However, this type doesn't appear to have the `Row`
+            // associated type in the documentation. DRY this out.
+            .select((
+                network_interface::name,
+                network_interface::ip,
+                network_interface::mac,
+                vpc_subnet::ipv4_block,
+                vpc_subnet::ipv6_block,
+                network_interface::vpc_id,
+                network_interface::slot,
+            ))
+            .get_results_async::<NicInfo>(self.pool_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel_pool(e, ErrorHandler::Server)
+            })?;
+        Ok(rows
+            .into_iter()
+            .map(sled_client_types::NetworkInterface::from)
+            .collect())
+    }
+
     /// List network interfaces associated with a given instance.
     pub async fn instance_list_network_interfaces(
         &self,

nexus/src/db/datastore.rs

@@ -1686,8 +1686,12 @@ impl Nexus {
             });
         }
 
-        let nics: Vec<external::NetworkInterface> = self
+        let nics = self
             .db_datastore
+            .derive_guest_network_interface_info(&opctx, &authz_instance)
+            .await?;
+
+        /*
             .instance_list_network_interfaces(
                 &opctx,
                 &authz_instance,
@@ -1702,6 +1706,7 @@ impl Nexus {
             .iter()
             .map(|x| x.clone().into())
             .collect();
+        */
 
         // Ask the sled agent to begin the state change.  Then update the
         // database to reflect the new intermediate state.  If this update is
@@ -1712,7 +1717,7 @@ impl Nexus {
             runtime: sled_agent_client::types::InstanceRuntimeState::from(
                 db_instance.runtime().clone(),
             ),
-            nics: nics.iter().map(|nic| nic.clone().into()).collect(),
+            nics,
             disks: disk_reqs,
             cloud_init_bytes: Some(base64::encode(
                 db_instance.generate_cidata()?,

nexus/src/nexus.rs

@@ -28,7 +28,6 @@ use omicron_common::api::external::Generation;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::InstanceState;
 use omicron_common::api::external::Name;
-use omicron_common::api::external::NetworkInterface;
 use omicron_common::api::internal::nexus::InstanceRuntimeState;
 use omicron_common::backoff::{self, BackoffError};
 use rand::{rngs::StdRng, RngCore, SeedableRng};
@@ -326,9 +325,9 @@ async fn sic_allocate_network_interface_ids(
 
 async fn sic_create_network_interfaces(
     sagactx: ActionContext<SagaInstanceCreate>,
-) -> Result<Option<Vec<NetworkInterface>>, ActionError> {
+) -> Result<(), ActionError> {
     match sagactx.saga_params().create_params.network_interfaces {
-        params::InstanceNetworkInterfaceAttachment::None => Ok(None),
+        params::InstanceNetworkInterfaceAttachment::None => Ok(()),
         params::InstanceNetworkInterfaceAttachment::Default => {
             sic_create_default_network_interface(&sagactx).await
         }
@@ -345,9 +344,9 @@ async fn sic_create_network_interfaces(
 async fn sic_create_custom_network_interfaces(
     sagactx: &ActionContext<SagaInstanceCreate>,
     interface_params: &[params::NetworkInterfaceCreate],
-) -> Result<Option<Vec<NetworkInterface>>, ActionError> {
+) -> Result<(), ActionError> {
     if interface_params.is_empty() {
-        return Ok(Some(vec![]));
+        return Ok(());
     }
 
     let osagactx = sagactx.user_data();
@@ -381,7 +380,6 @@ async fn sic_create_custom_network_interfaces(
         )));
     }
 
-    let mut interfaces = Vec::with_capacity(interface_params.len());
     if ids.len() != interface_params.len() {
         return Err(ActionError::action_failed(Error::internal_error(
             "found differing number of network interface IDs and interface \
@@ -392,7 +390,7 @@ async fn sic_create_custom_network_interfaces(
         // TODO-correctness: It seems racy to fetch the subnet and create the
         // interface in separate requests, but outside of a transaction. This
         // should probably either be in a transaction, or the
-        // `subnet_create_network_interface` function/query needs some JOIN
+        // `instance_create_network_interface` function/query needs some JOIN
         // on the `vpc_subnet` table.
         let (.., authz_subnet, db_subnet) = LookupPath::new(&opctx, &datastore)
             .vpc_id(authz_vpc.id())
@@ -406,7 +404,7 @@ async fn sic_create_custom_network_interfaces(
             interface_id,
             instance_id,
             authz_vpc.id(),
-            db_subnet,
+            db_subnet.clone(),
             mac,
             params.identity.clone(),
             params.ip,
@@ -422,8 +420,8 @@ async fn sic_create_custom_network_interfaces(
             .await;
 
         use crate::db::subnet_allocation::NetworkInterfaceError;
-        let interface = match result {
-            Ok(interface) => Ok(interface),
+        match result {
+            Ok(_) => Ok(()),
 
             // Detect the specific error arising from this node being partially
             // completed.
@@ -453,30 +451,19 @@ async fn sic_create_custom_network_interfaces(
                     instance_id;
                     "primary_key" => interface_id.to_string(),
                 );
-
-                // Refetch the interface itself, to serialize it for the next
-                // saga node.
-                LookupPath::new(&opctx, &datastore)
-                    .instance_id(authz_instance.id())
-                    .network_interface_name(&db::model::Name(
-                        params.identity.name.clone(),
-                    ))
-                    .fetch()
-                    .await
-                    .map(|(.., db_interface)| db_interface)
+                Ok(())
             }
             Err(e) => Err(e.into_external()),
         }
         .map_err(ActionError::action_failed)?;
-        interfaces.push(NetworkInterface::from(interface))
     }
-    Ok(Some(interfaces))
+    Ok(())
 }
 
 /// Create the default network interface for an instance during the create saga
 async fn sic_create_default_network_interface(
     sagactx: &ActionContext<SagaInstanceCreate>,
-) -> Result<Option<Vec<NetworkInterface>>, ActionError> {
+) -> Result<(), ActionError> {
     let osagactx = sagactx.user_data();
     let datastore = osagactx.datastore();
     let saga_params = sagactx.saga_params();
@@ -519,13 +506,13 @@ async fn sic_create_default_network_interface(
         interface_id,
         instance_id,
         authz_vpc.id(),
-        db_subnet,
+        db_subnet.clone(),
         mac,
         interface_params.identity.clone(),
         interface_params.ip,
     )
     .map_err(ActionError::action_failed)?;
-    let interface = datastore
+    datastore
         .instance_create_network_interface(
             &opctx,
             &authz_subnet,
@@ -535,17 +522,17 @@ async fn sic_create_default_network_interface(
         .await
         .map_err(db::subnet_allocation::NetworkInterfaceError::into_external)
         .map_err(ActionError::action_failed)?;
-    Ok(Some(vec![interface.into()]))
+    Ok(())
 }
 
 async fn sic_create_network_interfaces_undo(
     sagactx: ActionContext<SagaInstanceCreate>,
 ) -> Result<(), anyhow::Error> {
-    // We issue a request to delete any interfaces associated with this
-    // instance.  In the case we failed partway through allocating interfaces,
-    // we won't have cached the interface records in the saga log, but they're
-    // definitely still in the database. Just delete every interface that
-    // exists, even if there are zero such records.
+    // We issue a request to delete any interfaces associated with this instance.
+    // In the case we failed partway through allocating interfaces, we need to
+    // clean up any previously-created interface records from the database.
+    // Just delete every interface that exists, even if there are zero such
+    // records.
     let osagactx = sagactx.user_data();
     let datastore = osagactx.datastore();
     let saga_params = sagactx.saga_params();