Launch Nexus with both and HTTP and HTTPS server

smklein · smklein · commit 1bab1830c9bd · 2022-07-26T16:54:20.000-04:00
diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs
@@ -7,11 +7,11 @@
 
 use super::address::{Ipv6Subnet, RACK_PREFIX};
 use super::postgres_config::PostgresConfigWithUrl;
-use dropshot::ConfigDropshot;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use serde_with::DisplayFromStr;
 use std::fmt;
+use std::net::IpAddr;
 use std::path::{Path, PathBuf};
 use uuid::Uuid;
 
@@ -104,10 +104,10 @@ pub struct DeploymentConfig {
     pub id: Uuid,
     /// Uuid of the Rack where Nexus is executing.
     pub rack_id: Uuid,
-    /// Dropshot configuration for external API server
-    pub dropshot_external: ConfigDropshot,
-    /// Dropshot configuration for internal API server
-    pub dropshot_internal: ConfigDropshot,
+    /// External address of Nexus.
+    pub external_ip: IpAddr,
+    /// Internal address of Nexus.
+    pub internal_ip: IpAddr,
     /// Portion of the IP space to be managed by the Rack.
     pub subnet: Ipv6Subnet<RACK_PREFIX>,
     /// DB configuration.
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
@@ -34,6 +34,8 @@ pub use crucible_agent_client;
 use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use slog::Logger;
+use std::net::SocketAddr;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 #[macro_use]
@@ -71,7 +73,9 @@ pub fn run_openapi_internal() -> Result<(), String> {
 pub struct Server {
     /// shared state used by API request handlers
     pub apictx: Arc<ServerContext>,
-    /// dropshot server for external API
+    /// dropshot server for external API (encrypted)
+    pub https_server_external: dropshot::HttpServer<Arc<ServerContext>>,
+    /// dropshot server for external API (unencrypted)
     pub http_server_external: dropshot::HttpServer<Arc<ServerContext>>,
     /// dropshot server for internal API
     pub http_server_internal: dropshot::HttpServer<Arc<ServerContext>>,
@@ -92,26 +96,76 @@ impl Server {
             ServerContext::new(config.deployment.rack_id, ctxlog, &config)
                 .await?;
 
+        // We launch separate dropshot servers for the "encrypted" and
+        // "unencrypted" ports.
+
+        const HTTPS_PORT: u16 = 443;
+        const HTTP_PORT: u16 = 80;
+
+        let dropshot_external_https_config = dropshot::ConfigDropshot {
+            bind_address: SocketAddr::new(
+                config.deployment.external_ip,
+                HTTPS_PORT,
+            ),
+            request_body_max_bytes: 1048576,
+            tls: Some(dropshot::ConfigTls {
+                cert_file: PathBuf::from("/var/nexus/certs/cert.pem"),
+                key_file: PathBuf::from("/var/nexus/certs/key.pem"),
+            }),
+        };
+        // TODO: Consider removing this interface when all clients are using
+        // https?
+        let dropshot_external_http_config = dropshot::ConfigDropshot {
+            bind_address: SocketAddr::new(
+                config.deployment.external_ip,
+                HTTP_PORT,
+            ),
+            request_body_max_bytes: 1048576,
+            tls: None,
+        };
+
+        let dropshot_internal_config = dropshot::ConfigDropshot {
+            bind_address: SocketAddr::new(
+                config.deployment.internal_ip,
+                omicron_common::address::NEXUS_INTERNAL_PORT,
+            ),
+            request_body_max_bytes: 1048576,
+            ..Default::default()
+        };
+
+        let https_server_starter_external = dropshot::HttpServerStarter::new(
+            &dropshot_external_https_config,
+            external_api(),
+            Arc::clone(&apictx),
+            &log.new(o!("component" => "dropshot_external (encrypted)")),
+        )
+        .map_err(|error| format!("initializing external server: {}", error))?;
+        let https_server_external = https_server_starter_external.start();
+
         let http_server_starter_external = dropshot::HttpServerStarter::new(
-            &config.deployment.dropshot_external,
+            &dropshot_external_http_config,
             external_api(),
             Arc::clone(&apictx),
-            &log.new(o!("component" => "dropshot_external")),
+            &log.new(o!("component" => "dropshot_external (unencrypted)")),
         )
         .map_err(|error| format!("initializing external server: {}", error))?;
+        let http_server_external = http_server_starter_external.start();
 
         let http_server_starter_internal = dropshot::HttpServerStarter::new(
-            &config.deployment.dropshot_internal,
+            &dropshot_internal_config,
             internal_api(),
             Arc::clone(&apictx),
             &log.new(o!("component" => "dropshot_internal")),
         )
         .map_err(|error| format!("initializing internal server: {}", error))?;
-
-        let http_server_external = http_server_starter_external.start();
         let http_server_internal = http_server_starter_internal.start();
 
-        Ok(Server { apictx, http_server_external, http_server_internal })
+        Ok(Server {
+            apictx,
+            https_server_external,
+            http_server_external,
+            http_server_internal,
+        })
     }
 
     /// Wait for the given server to shut down
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
@@ -3,7 +3,9 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use internal_dns_client::names::{BackendName, ServiceName, AAAA, SRV};
-use omicron_common::address::{DENDRITE_PORT, OXIMETER_PORT};
+use omicron_common::address::{
+    DENDRITE_PORT, NEXUS_INTERNAL_PORT, OXIMETER_PORT,
+};
 use omicron_common::api::external;
 use omicron_common::api::internal::nexus::{
     DiskRuntimeState, InstanceRuntimeState,
@@ -342,7 +344,7 @@ impl From<DatasetEnsureBody> for sled_agent_client::types::DatasetEnsureBody {
 )]
 #[serde(tag = "type", rename_all = "snake_case")]
 pub enum ServiceType {
-    Nexus { internal_address: SocketAddrV6, external_address: SocketAddr },
+    Nexus { internal_ip: Ipv6Addr, external_ip: IpAddr },
     InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 },
     Oximeter,
     Dendrite { asic: DendriteAsic },
@@ -354,9 +356,9 @@ impl From<ServiceType> for sled_agent_client::types::ServiceType {
         use ServiceType as St;
 
         match s {
-            St::Nexus { internal_address, external_address } => AutoSt::Nexus {
-                internal_address: internal_address.to_string(),
-                external_address: external_address.to_string(),
+            St::Nexus { internal_ip, external_ip } => AutoSt::Nexus {
+                internal_address: internal_ip.to_string(),
+                external_address: external_ip.to_string(),
             },
             St::InternalDns { server_address, dns_address } => {
                 AutoSt::InternalDns {
@@ -370,7 +372,7 @@ impl From<ServiceType> for sled_agent_client::types::ServiceType {
     }
 }
 
-/// Describes a request to create a service. This information
+/// Describes a request to create a service. This informatios tn
 /// should be sufficient for a Sled Agent to start a zone
 /// containing the requested service.
 #[derive(
@@ -415,7 +417,9 @@ impl ServiceRequest {
     pub fn address(&self) -> SocketAddrV6 {
         match self.service_type {
             ServiceType::InternalDns { server_address, .. } => server_address,
-            ServiceType::Nexus { internal_address, .. } => internal_address,
+            ServiceType::Nexus { internal_ip, .. } => {
+                SocketAddrV6::new(internal_ip, NEXUS_INTERNAL_PORT, 0, 0)
+            }
             ServiceType::Oximeter => {
                 SocketAddrV6::new(self.addresses[0], OXIMETER_PORT, 0, 0)
             }
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
@@ -13,7 +13,6 @@ use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT;
 use crate::illumos::zone::AddressRequest;
 use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType};
 use crate::zone::Zones;
-use dropshot::ConfigDropshot;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::OXIMETER_PORT;
 use omicron_common::address::RACK_PREFIX;
@@ -25,7 +24,7 @@ use omicron_common::postgres_config::PostgresConfigWithUrl;
 use slog::Logger;
 use std::collections::HashSet;
 use std::iter::FromIterator;
-use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use tokio::io::AsyncWriteExt;
@@ -373,21 +372,21 @@ impl ServiceManager {
             let default_smf_name = format!("{}:default", smf_name);
 
             match service.service_type {
-                ServiceType::Nexus { internal_address, external_address } => {
+                ServiceType::Nexus { internal_ip, external_ip } => {
                     info!(self.log, "Setting up Nexus service");
 
                     // The address of Nexus' external interface is a special
                     // case; it may be an IPv4 address.
                     let addr_request =
-                        AddressRequest::new_static(external_address.ip(), None);
+                        AddressRequest::new_static(external_ip, None);
                     running_zone
                         .ensure_external_address_with_name(
                             addr_request,
                             "public",
                         )
                         .await?;
 
-                    if let IpAddr::V4(_public_addr4) = external_address.ip() {
+                    if let IpAddr::V4(_public_addr4) = external_ip {
                         // If requested, create a default route back through
                         // the internet gateway.
                         if let Some(ref gateway) = self.config.gateway_address {
@@ -406,21 +405,8 @@ impl ServiceManager {
                     let deployment_config = NexusDeploymentConfig {
                         id: service.id,
                         rack_id: self.rack_id,
-                        dropshot_external: ConfigDropshot {
-                            bind_address: external_address,
-                            request_body_max_bytes: 1048576,
-                            tls: Some(
-                                dropshot::ConfigTls {
-                                    cert_file: PathBuf::from("/var/nexus/certs/cert.pem"),
-                                    key_file: PathBuf::from("/var/nexus/certs/key.pem"),
-                                }
-                            ),
-                        },
-                        dropshot_internal: ConfigDropshot {
-                            bind_address: SocketAddr::V6(internal_address),
-                            request_body_max_bytes: 1048576,
-                            ..Default::default()
-                        },
+                        external_ip,
+                        internal_ip: IpAddr::V6(internal_ip),
                         subnet: Ipv6Subnet::<RACK_PREFIX>::new(
                             self.underlay_address,
                         ),
diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml
@@ -55,9 +55,9 @@ addresses = [ "fd00:1122:3344:0101::3" ]
 gz_addresses = []
 [request.service.service_type]
 type = "nexus"
-internal_address = "[fd00:1122:3344:0101::3]:12221"
+internal_address = "fd00:1122:3344:0101::3"
 # NOTE: In the lab, use "172.20.15.226"
-external_address = "192.168.1.20:80"
+external_address = "192.168.1.20"
 
 # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus
 # should allocate Oximeter services.
