diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 39d7badb8bb..f753abfb754 100644 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -17,6 +17,7 @@ cargo --version rustc --version ptime -m ./tools/install_builder_prerequisites.sh -yp +ptime -m ./tools/create_self_signed_cert.sh -yp ptime -m cargo run --locked --release --bin omicron-package -- package diff --git a/README.adoc b/README.adoc index 60f27194bdf..e25835e2b2e 100644 --- a/README.adoc +++ b/README.adoc @@ -81,6 +81,7 @@ Supported config properties include: | |Yes |Dropshot configuration for the external server (i.e., the one that operators and developers using the Oxide rack will use). Specific properties are documented below, but see the Dropshot README for details. +| Note that this is an array of external address configurations; multiple may be supplied. |`dropshot_external.bind_address` |`"127.0.0.1:12220"` diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index a18454e02d0..13a6a7b057b 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -104,9 +104,12 @@ pub struct DeploymentConfig { pub id: Uuid, /// Uuid of the Rack where Nexus is executing. pub rack_id: Uuid, - /// Dropshot configuration for external API server - pub dropshot_external: ConfigDropshot, - /// Dropshot configuration for internal API server + /// Dropshot configurations for external API server. + /// + /// Multiple configurations may be supplied to request + /// combinations of HTTP / HTTPS servers. + pub dropshot_external: Vec, + /// Dropshot configuration for internal API server. pub dropshot_internal: ConfigDropshot, /// Portion of the IP space to be managed by the Rack. pub subnet: Ipv6Subnet, diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 09b158238e4..ac4ee63e97f 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -53,6 +53,13 @@ This script requires Omicron be uninstalled, e.g., with `pfexec that is not the case. The script will then remove the file-based vdevs and the VNICs created by `create_virtual_hardware.sh`. +=== Make me a certificate! + +Nexus's external interface will typically be served using public-facing x.509 +certificate. While we are still configuring the mechanism to integrate this real +certificate into the package system, `./tools/create_self_signed_cert.sh` can be +used to generate an equivalent self-signed certificate. + == Deploying Omicron The control plane repository contains a packaging tool which bundles binaries diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 94cb06acc98..c9139d8c948 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -38,15 +38,15 @@ address = "[::1]:8123" id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc" -[deployment.dropshot_external] -# IP address and TCP port on which to listen for the external API +[[deployment.dropshot_external]] +# IP Address and TCP port on which to listen for the external API bind_address = "127.0.0.1:12220" # Allow larger request bodies (1MiB) to accomodate firewall endpoints (one # rule is ~500 bytes) request_body_max_bytes = 1048576 [deployment.dropshot_internal] -# IP address and TCP port on which to listen for the internal API +# IP Address and TCP port on which to listen for the internal API bind_address = "127.0.0.1:12221" [deployment.subnet] diff --git a/nexus/src/config.rs b/nexus/src/config.rs index 7266a3abd10..d622368ef1b 100644 --- a/nexus/src/config.rs +++ b/nexus/src/config.rs @@ -336,7 +336,7 @@ mod test { [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" - [deployment.dropshot_external] + [[deployment.dropshot_external]] bind_address = "10.1.2.3:4567" request_body_max_bytes = 1024 [deployment.dropshot_internal] @@ -358,12 +358,12 @@ mod test { rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f" .parse() .unwrap(), - dropshot_external: ConfigDropshot { + dropshot_external: vec![ConfigDropshot { bind_address: "10.1.2.3:4567" .parse::() .unwrap(), ..Default::default() - }, + },], dropshot_internal: ConfigDropshot { bind_address: "10.1.2.3:4568" .parse::() @@ -418,7 +418,7 @@ mod test { [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" - [deployment.dropshot_external] + [[deployment.dropshot_external]] bind_address = "10.1.2.3:4567" request_body_max_bytes = 1024 [deployment.dropshot_internal] @@ -460,7 +460,7 @@ mod test { [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" - [deployment.dropshot_external] + [[deployment.dropshot_external]] bind_address = "10.1.2.3:4567" request_body_max_bytes = 1024 [deployment.dropshot_internal] @@ -516,7 +516,7 @@ mod test { [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" - [deployment.dropshot_external] + [[deployment.dropshot_external]] bind_address = "10.1.2.3:4567" request_body_max_bytes = 1024 [deployment.dropshot_internal] diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index fd7b50802e9..e16037a91bf 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -71,8 +71,8 @@ pub fn run_openapi_internal() -> Result<(), String> { pub struct Server { /// shared state used by API request handlers pub apictx: Arc, - /// dropshot server for external API - pub http_server_external: dropshot::HttpServer>, + /// dropshot servers for external API + pub http_servers_external: Vec>>, /// dropshot server for internal API pub http_server_internal: dropshot::HttpServer>, } @@ -92,26 +92,36 @@ impl Server { ServerContext::new(config.deployment.rack_id, ctxlog, &config) .await?; - let http_server_starter_external = dropshot::HttpServerStarter::new( - &config.deployment.dropshot_external, - external_api(), - Arc::clone(&apictx), - &log.new(o!("component" => "dropshot_external")), - ) - .map_err(|error| format!("initializing external server: {}", error))?; - - let http_server_starter_internal = dropshot::HttpServerStarter::new( + // Launch the internal server. + let server_starter_internal = dropshot::HttpServerStarter::new( &config.deployment.dropshot_internal, internal_api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot_internal")), ) .map_err(|error| format!("initializing internal server: {}", error))?; - - let http_server_external = http_server_starter_external.start(); - let http_server_internal = http_server_starter_internal.start(); - - Ok(Server { apictx, http_server_external, http_server_internal }) + let http_server_internal = server_starter_internal.start(); + + // Launch the external server(s). + let http_servers_external = config + .deployment + .dropshot_external + .iter() + .map(|cfg| { + let server_starter_external = dropshot::HttpServerStarter::new( + &cfg, + external_api(), + Arc::clone(&apictx), + &log.new(o!("component" => "dropshot_external")), + ) + .map_err(|error| { + format!("initializing external server: {}", error) + })?; + Ok(server_starter_external.start()) + }) + .collect::>, String>>()?; + + Ok(Server { apictx, http_servers_external, http_server_internal }) } /// Wait for the given server to shut down @@ -120,18 +130,20 @@ impl Server { /// immediately after calling `start()`, the program will block indefinitely /// or until something else initiates a graceful shutdown. pub async fn wait_for_finish(self) -> Result<(), String> { - let errors = vec![ - self.http_server_external - .await - .map_err(|e| format!("external: {}", e)), + let mut errors = vec![]; + for server in self.http_servers_external { + errors.push(server.await.map_err(|e| format!("external: {}", e))); + } + errors.push( self.http_server_internal .await .map_err(|e| format!("internal: {}", e)), - ] - .into_iter() - .filter(Result::is_err) - .map(|r| r.unwrap_err()) - .collect::>(); + ); + let errors = errors + .into_iter() + .filter(Result::is_err) + .map(|r| r.unwrap_err()) + .collect::>(); if errors.len() > 0 { let msg = format!("errors shutting down: ({})", errors.join(", ")); diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 73279124b53..82b01c3a0ff 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -46,7 +46,9 @@ pub struct ControlPlaneTestContext { impl ControlPlaneTestContext { pub async fn teardown(mut self) { - self.server.http_server_external.close().await.unwrap(); + for server in self.server.http_servers_external { + server.close().await.unwrap(); + } self.server.http_server_internal.close().await.unwrap(); self.database.cleanup().await.unwrap(); self.clickhouse.cleanup().await.unwrap(); @@ -119,7 +121,7 @@ pub async fn test_setup_with_config( .expect("Nexus never loaded users"); let testctx_external = ClientTestContext::new( - server.http_server_external.local_addr(), + server.http_servers_external[0].local_addr(), logctx.log.new(o!("component" => "external client test context")), ); let testctx_internal = ClientTestContext::new( diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index fdfeb5effb4..59c8dc4614d 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -41,16 +41,13 @@ max_vpc_ipv4_subnet_prefix = 29 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc" -# +[[deployment.dropshot_external]] # NOTE: for the test suite, the port MUST be 0 (in order to bind to any # available port) because the test suite will be running many servers # concurrently. -# -[deployment.dropshot_external] bind_address = "127.0.0.1:0" request_body_max_bytes = 1048576 -# port must be 0. see above [deployment.dropshot_internal] bind_address = "127.0.0.1:0" request_body_max_bytes = 1048576 diff --git a/nexus/tests/integration_tests/authn_http.rs b/nexus/tests/integration_tests/authn_http.rs index 0ae6e3fd3cf..a79fd1f56d0 100644 --- a/nexus/tests/integration_tests/authn_http.rs +++ b/nexus/tests/integration_tests/authn_http.rs @@ -299,7 +299,7 @@ async fn start_whoami_server( TestContext::new( whoami_api, server_state, - &config.deployment.dropshot_external, + &config.deployment.dropshot_external[0], Some(logctx), log, ) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index ceb8c4ac342..ee9a1a1709b 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1136,11 +1136,13 @@ { "type": "object", "properties": { - "external_address": { - "type": "string" + "external_ip": { + "type": "string", + "format": "ip" }, - "internal_address": { - "type": "string" + "internal_ip": { + "type": "string", + "format": "ipv6" }, "type": { "type": "string", @@ -1150,8 +1152,8 @@ } }, "required": [ - "external_address", - "internal_address", + "external_ip", + "internal_ip", "type" ] }, diff --git a/package-manifest.toml b/package-manifest.toml index 72f8b6f0a44..10582ddd8c1 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -19,6 +19,7 @@ zone = true setup_hint = """ - Run `./tools/ci_download_console` to download the web console assets - Run `pkg install library/postgresql-13` to download Postgres libraries +- Run `./tools/create_self_signed_cert.sh` to generate a certificate """ [[package.omicron-nexus.paths]] @@ -30,6 +31,14 @@ to = "/var/svc/manifest/site/nexus" [[package.omicron-nexus.paths]] from = "out/console-assets" to = "/var/nexus/static" +# Note, we could just map the whole "out/certs" directory, but this ensures +# both files exist. +[[package.omicron-nexus.paths]] +from = "out/certs/cert.pem" +to = "/var/nexus/certs/cert.pem" +[[package.omicron-nexus.paths]] +from = "out/certs/key.pem" +to = "/var/nexus/certs/key.pem" [package.oximeter-collector] rust.binary_names = ["oximeter"] diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 494a901be5d..a5a34bd9179 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -3,7 +3,9 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use internal_dns_client::names::{BackendName, ServiceName, AAAA, SRV}; -use omicron_common::address::{DENDRITE_PORT, OXIMETER_PORT}; +use omicron_common::address::{ + DENDRITE_PORT, NEXUS_INTERNAL_PORT, OXIMETER_PORT, +}; use omicron_common::api::external; use omicron_common::api::internal::nexus::{ DiskRuntimeState, InstanceRuntimeState, @@ -342,7 +344,7 @@ impl From for sled_agent_client::types::DatasetEnsureBody { )] #[serde(tag = "type", rename_all = "snake_case")] pub enum ServiceType { - Nexus { internal_address: SocketAddrV6, external_address: SocketAddr }, + Nexus { internal_ip: Ipv6Addr, external_ip: IpAddr }, InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 }, Oximeter, Dendrite { asic: DendriteAsic }, @@ -354,10 +356,9 @@ impl From for sled_agent_client::types::ServiceType { use ServiceType as St; match s { - St::Nexus { internal_address, external_address } => AutoSt::Nexus { - internal_address: internal_address.to_string(), - external_address: external_address.to_string(), - }, + St::Nexus { internal_ip, external_ip } => { + AutoSt::Nexus { internal_ip, external_ip } + } St::InternalDns { server_address, dns_address } => { AutoSt::InternalDns { server_address: server_address.to_string(), @@ -415,7 +416,9 @@ impl ServiceRequest { pub fn address(&self) -> SocketAddrV6 { match self.service_type { ServiceType::InternalDns { server_address, .. } => server_address, - ServiceType::Nexus { internal_address, .. } => internal_address, + ServiceType::Nexus { internal_ip, .. } => { + SocketAddrV6::new(internal_ip, NEXUS_INTERNAL_PORT, 0, 0) + } ServiceType::Oximeter => { SocketAddrV6::new(self.addresses[0], OXIMETER_PORT, 0, 0) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 8404965ec26..19efed5b701 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -13,8 +13,8 @@ use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType}; use crate::zone::Zones; -use dropshot::ConfigDropshot; use omicron_common::address::Ipv6Subnet; +use omicron_common::address::NEXUS_INTERNAL_PORT; use omicron_common::address::OXIMETER_PORT; use omicron_common::address::RACK_PREFIX; use omicron_common::address::SLED_PREFIX; @@ -373,13 +373,13 @@ impl ServiceManager { let default_smf_name = format!("{}:default", smf_name); match service.service_type { - ServiceType::Nexus { internal_address, external_address } => { + ServiceType::Nexus { internal_ip, external_ip } => { info!(self.log, "Setting up Nexus service"); // The address of Nexus' external interface is a special // case; it may be an IPv4 address. let addr_request = - AddressRequest::new_static(external_address.ip(), None); + AddressRequest::new_static(external_ip, None); running_zone .ensure_external_address_with_name( addr_request, @@ -387,7 +387,7 @@ impl ServiceManager { ) .await?; - if let IpAddr::V4(_public_addr4) = external_address.ip() { + if let IpAddr::V4(_public_addr4) = external_ip { // If requested, create a default route back through // the internet gateway. if let Some(ref gateway) = self.config.gateway_address { @@ -401,18 +401,31 @@ impl ServiceManager { } } + let cert_file = PathBuf::from("/var/nexus/certs/cert.pem"); + let key_file = PathBuf::from("/var/nexus/certs/key.pem"); + // Nexus takes a separate config file for parameters which // cannot be known at packaging time. let deployment_config = NexusDeploymentConfig { id: service.id, rack_id: self.rack_id, - dropshot_external: ConfigDropshot { - bind_address: external_address, - request_body_max_bytes: 1048576, - ..Default::default() - }, - dropshot_internal: ConfigDropshot { - bind_address: SocketAddr::V6(internal_address), + + // Request two dropshot servers: One for HTTP (port 80), + // one for HTTPS (port 443). + dropshot_external: vec![ + dropshot::ConfigDropshot { + bind_address: SocketAddr::new(external_ip, 443), + request_body_max_bytes: 1048576, + tls: Some(dropshot::ConfigTls { cert_file, key_file }), + }, + dropshot::ConfigDropshot { + bind_address: SocketAddr::new(external_ip, 80), + request_body_max_bytes: 1048576, + ..Default::default() + }, + ], + dropshot_internal: dropshot::ConfigDropshot { + bind_address: SocketAddr::new(IpAddr::V6(internal_ip), NEXUS_INTERNAL_PORT), request_body_max_bytes: 1048576, ..Default::default() }, diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 27b828da7bd..98b03cdcce0 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -71,9 +71,9 @@ addresses = [ "fd00:1122:3344:0101::3" ] gz_addresses = [] [request.service.service_type] type = "nexus" -internal_address = "[fd00:1122:3344:0101::3]:12221" +internal_ip = "fd00:1122:3344:0101::3" # NOTE: In the lab, use "172.20.15.226" -external_address = "192.168.1.20:80" +external_ip = "192.168.1.20" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. diff --git a/tools/create_self_signed_cert.sh b/tools/create_self_signed_cert.sh new file mode 100755 index 00000000000..783c8eaf708 --- /dev/null +++ b/tools/create_self_signed_cert.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Creates a self-signed certificate. +# +# For those with access, certificates are available in: +# +# https://github.com/oxidecomputer/configs/tree/master/nginx/ssl/wildcard.oxide-preview.com + +set -eu + +# Set the CWD to Omicron's source. +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd "${SOURCE_DIR}/.." + +OUTPUT_DIR="out/certs" +CERT_PATH="$OUTPUT_DIR/cert.pem" +KEY_PATH="$OUTPUT_DIR/key.pem" + +mkdir -p "$OUTPUT_DIR" + +openssl req -newkey rsa:4096 \ + -x509 \ + -sha256 \ + -days 3650 \ + -nodes \ + -out "$CERT_PATH" \ + -keyout "$KEY_PATH" \ + -subj '/CN=localhost'