[nexus] Add HTTPS support, plumbing x509 certificates (#1500)

Another attempt at #1287

In addition to launching an HTTPS server, this also launches an HTTP server so we can smoothly migrate clients (like the CLI).

Part of #249

Author: smklein

.github/buildomat/jobs/package.sh

@@ -17,6 +17,7 @@ cargo --version
 rustc --version
 
 ptime -m ./tools/install_builder_prerequisites.sh -yp
+ptime -m ./tools/create_self_signed_cert.sh -yp
 
 ptime -m cargo run --locked --release --bin omicron-package -- package
 

README.adoc

@@ -81,6 +81,7 @@ Supported config properties include:
 |
 |Yes
 |Dropshot configuration for the external server (i.e., the one that operators and developers using the Oxide rack will use).  Specific properties are documented below, but see the Dropshot README for details.
+| Note that this is an array of external address configurations; multiple may be supplied.
 
 |`dropshot_external.bind_address`
 |`"127.0.0.1:12220"`

common/src/nexus_config.rs

@@ -104,9 +104,12 @@ pub struct DeploymentConfig {
     pub id: Uuid,
     /// Uuid of the Rack where Nexus is executing.
     pub rack_id: Uuid,
-    /// Dropshot configuration for external API server
-    pub dropshot_external: ConfigDropshot,
-    /// Dropshot configuration for internal API server
+    /// Dropshot configurations for external API server.
+    ///
+    /// Multiple configurations may be supplied to request
+    /// combinations of HTTP / HTTPS servers.
+    pub dropshot_external: Vec<ConfigDropshot>,
+    /// Dropshot configuration for internal API server.
     pub dropshot_internal: ConfigDropshot,
     /// Portion of the IP space to be managed by the Rack.
     pub subnet: Ipv6Subnet<RACK_PREFIX>,

docs/how-to-run.adoc

@@ -53,6 +53,13 @@ This script requires Omicron be uninstalled, e.g., with `pfexec
 that is not the case. The script will then remove the file-based vdevs and the
 VNICs created by `create_virtual_hardware.sh`.
 
+=== Make me a certificate!
+
+Nexus's external interface will typically be served using public-facing x.509
+certificate. While we are still configuring the mechanism to integrate this real
+certificate into the package system, `./tools/create_self_signed_cert.sh` can be
+used to generate an equivalent self-signed certificate.
+
 == Deploying Omicron
 
 The control plane repository contains a packaging tool which bundles binaries

nexus/examples/config.toml

@@ -38,15 +38,15 @@ address = "[::1]:8123"
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
 rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
-[deployment.dropshot_external]
-# IP address and TCP port on which to listen for the external API
+[[deployment.dropshot_external]]
+# IP Address and TCP port on which to listen for the external API
 bind_address = "127.0.0.1:12220"
 # Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
 # rule is ~500 bytes)
 request_body_max_bytes = 1048576
 
 [deployment.dropshot_internal]
-# IP address and TCP port on which to listen for the internal API
+# IP Address and TCP port on which to listen for the internal API
 bind_address = "127.0.0.1:12221"
 
 [deployment.subnet]

nexus/src/config.rs

@@ -336,7 +336,7 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
+            [[deployment.dropshot_external]]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
             [deployment.dropshot_internal]
@@ -358,12 +358,12 @@ mod test {
                     rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
                         .parse()
                         .unwrap(),
-                    dropshot_external: ConfigDropshot {
+                    dropshot_external: vec![ConfigDropshot {
                         bind_address: "10.1.2.3:4567"
                             .parse::<SocketAddr>()
                             .unwrap(),
                         ..Default::default()
-                    },
+                    },],
                     dropshot_internal: ConfigDropshot {
                         bind_address: "10.1.2.3:4568"
                             .parse::<SocketAddr>()
@@ -418,7 +418,7 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
+            [[deployment.dropshot_external]]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
             [deployment.dropshot_internal]
@@ -460,7 +460,7 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
+            [[deployment.dropshot_external]]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
             [deployment.dropshot_internal]
@@ -516,7 +516,7 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
+            [[deployment.dropshot_external]]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
             [deployment.dropshot_internal]

nexus/src/lib.rs

@@ -71,8 +71,8 @@ pub fn run_openapi_internal() -> Result<(), String> {
 pub struct Server {
     /// shared state used by API request handlers
     pub apictx: Arc<ServerContext>,
-    /// dropshot server for external API
-    pub http_server_external: dropshot::HttpServer<Arc<ServerContext>>,
+    /// dropshot servers for external API
+    pub http_servers_external: Vec<dropshot::HttpServer<Arc<ServerContext>>>,
     /// dropshot server for internal API
     pub http_server_internal: dropshot::HttpServer<Arc<ServerContext>>,
 }
@@ -92,26 +92,36 @@ impl Server {
             ServerContext::new(config.deployment.rack_id, ctxlog, &config)
                 .await?;
 
-        let http_server_starter_external = dropshot::HttpServerStarter::new(
-            &config.deployment.dropshot_external,
-            external_api(),
-            Arc::clone(&apictx),
-            &log.new(o!("component" => "dropshot_external")),
-        )
-        .map_err(|error| format!("initializing external server: {}", error))?;
-
-        let http_server_starter_internal = dropshot::HttpServerStarter::new(
+        // Launch the internal server.
+        let server_starter_internal = dropshot::HttpServerStarter::new(
             &config.deployment.dropshot_internal,
             internal_api(),
             Arc::clone(&apictx),
             &log.new(o!("component" => "dropshot_internal")),
         )
         .map_err(|error| format!("initializing internal server: {}", error))?;
-
-        let http_server_external = http_server_starter_external.start();
-        let http_server_internal = http_server_starter_internal.start();
-
-        Ok(Server { apictx, http_server_external, http_server_internal })
+        let http_server_internal = server_starter_internal.start();
+
+        // Launch the external server(s).
+        let http_servers_external = config
+            .deployment
+            .dropshot_external
+            .iter()
+            .map(|cfg| {
+                let server_starter_external = dropshot::HttpServerStarter::new(
+                    &cfg,
+                    external_api(),
+                    Arc::clone(&apictx),
+                    &log.new(o!("component" => "dropshot_external")),
+                )
+                .map_err(|error| {
+                    format!("initializing external server: {}", error)
+                })?;
+                Ok(server_starter_external.start())
+            })
+            .collect::<Result<Vec<dropshot::HttpServer<_>>, String>>()?;
+
+        Ok(Server { apictx, http_servers_external, http_server_internal })
     }
 
     /// Wait for the given server to shut down
@@ -120,18 +130,20 @@ impl Server {
     /// immediately after calling `start()`, the program will block indefinitely
     /// or until something else initiates a graceful shutdown.
     pub async fn wait_for_finish(self) -> Result<(), String> {
-        let errors = vec![
-            self.http_server_external
-                .await
-                .map_err(|e| format!("external: {}", e)),
+        let mut errors = vec![];
+        for server in self.http_servers_external {
+            errors.push(server.await.map_err(|e| format!("external: {}", e)));
+        }
+        errors.push(
             self.http_server_internal
                 .await
                 .map_err(|e| format!("internal: {}", e)),
-        ]
-        .into_iter()
-        .filter(Result::is_err)
-        .map(|r| r.unwrap_err())
-        .collect::<Vec<String>>();
+        );
+        let errors = errors
+            .into_iter()
+            .filter(Result::is_err)
+            .map(|r| r.unwrap_err())
+            .collect::<Vec<String>>();
 
         if errors.len() > 0 {
             let msg = format!("errors shutting down: ({})", errors.join(", "));

nexus/test-utils/src/lib.rs

@@ -46,7 +46,9 @@ pub struct ControlPlaneTestContext {
 
 impl ControlPlaneTestContext {
     pub async fn teardown(mut self) {
-        self.server.http_server_external.close().await.unwrap();
+        for server in self.server.http_servers_external {
+            server.close().await.unwrap();
+        }
         self.server.http_server_internal.close().await.unwrap();
         self.database.cleanup().await.unwrap();
         self.clickhouse.cleanup().await.unwrap();
@@ -119,7 +121,7 @@ pub async fn test_setup_with_config(
         .expect("Nexus never loaded users");
 
     let testctx_external = ClientTestContext::new(
-        server.http_server_external.local_addr(),
+        server.http_servers_external[0].local_addr(),
         logctx.log.new(o!("component" => "external client test context")),
     );
     let testctx_internal = ClientTestContext::new(

nexus/tests/config.test.toml

@@ -41,16 +41,13 @@ max_vpc_ipv4_subnet_prefix = 29
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
 rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
-#
+[[deployment.dropshot_external]]
 # NOTE: for the test suite, the port MUST be 0 (in order to bind to any
 # available port) because the test suite will be running many servers
 # concurrently.
-#
-[deployment.dropshot_external]
 bind_address = "127.0.0.1:0"
 request_body_max_bytes = 1048576
 
-# port must be 0. see above
 [deployment.dropshot_internal]
 bind_address = "127.0.0.1:0"
 request_body_max_bytes = 1048576

nexus/tests/integration_tests/authn_http.rs

@@ -299,7 +299,7 @@ async fn start_whoami_server(
     TestContext::new(
         whoami_api,
         server_state,
-        &config.deployment.dropshot_external,
+        &config.deployment.dropshot_external[0],
         Some(logctx),
         log,
     )