[sled agent] Overhaul error handling to be more informative

common/src/address.rs

@@ -0,0 +1,163 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Common IP addressing functionality.
+//!
+//! This addressing functionality is shared by both initialization services
+//! and Nexus, who need to agree upon addressing schemes.
+
+use ipnetwork::Ipv6Network;
+use serde::{Deserialize, Serialize};
+use std::net::{Ipv6Addr, SocketAddrV6};
+
+pub const AZ_PREFIX: u8 = 48;
+pub const RACK_PREFIX: u8 = 56;
+pub const SLED_PREFIX: u8 = 64;
+
+/// The amount of redundancy for DNS servers.
+///
+/// Must be less than MAX_DNS_REDUNDANCY.
+pub const DNS_REDUNDANCY: usize = 1;
+/// The maximum amount of redundancy for DNS servers.
+///
+/// This determines the number of addresses which are
+/// reserved for DNS servers.
+pub const MAX_DNS_REDUNDANCY: usize = 5;
+
+pub const DNS_PORT: u16 = 53;
+pub const DNS_SERVER_PORT: u16 = 5353;
+pub const SLED_AGENT_PORT: u16 = 12345;
+
+/// Represents a subnet which may be used for contacting DNS services.
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+pub struct DnsSubnet {
+    network: Ipv6Network,
+}
+
+impl DnsSubnet {
+    /// Returns the DNS server address within the subnet.
+    ///
+    /// This is the first address within the subnet.
+    pub fn dns_address(&self) -> Ipv6Network {
+        let mut iter = self.network.iter();
+        let _anycast_ip = iter.next().unwrap();
+        let dns_ip = iter.next().unwrap();
+        Ipv6Network::new(dns_ip, SLED_PREFIX).unwrap()
+    }
+
+    /// Returns the address which the Global Zone should create
+    /// to be able to contact the DNS server.
+    ///
+    /// This is the second address within the subnet.
+    pub fn gz_address(&self) -> Ipv6Network {
+        let mut iter = self.network.iter();
+        let _anycast_ip = iter.next().unwrap();
+        let _dns_ip = iter.next().unwrap();
+        Ipv6Network::new(iter.next().unwrap(), SLED_PREFIX).unwrap()
+    }
+}
+
+/// A wrapper around an IPv6 network, indicating it is a "reserved" rack
+/// subnet which can be used for AZ-wide services.
+#[derive(Debug, Clone)]
+pub struct ReservedRackSubnet(pub Ipv6Network);
+
+impl ReservedRackSubnet {
+    /// Returns the subnet for the reserved rack subnet.
+    pub fn new(subnet: Ipv6Network) -> Self {
+        let net = Ipv6Network::new(subnet.network(), AZ_PREFIX).unwrap();
+        ReservedRackSubnet(
+            Ipv6Network::new(net.network(), RACK_PREFIX).unwrap(),
+        )
+    }
+
+    /// Given a particular rack subnet, return the DNS addresses.
+    ///
+    /// These addresses will come from the first [`DNS_REDUNDANCY`] `/64s` of the
+    /// [`RACK_PREFIX`] subnet.
+    pub fn get_dns_subnets(&self) -> Vec<DnsSubnet> {
+        assert_eq!(self.0.prefix(), RACK_PREFIX);
+
+        (0..DNS_REDUNDANCY)
+            .map(|idx| {
+                let network =
+                    get_64_subnet(self.0, u8::try_from(idx + 1).unwrap());
+
+                DnsSubnet { network }
+            })
+            .collect()
+    }
+}
+
+/// Return the sled agent address for a subnet.
+///
+/// This address will come from the first address of the [`SLED_PREFIX`] subnet.
+pub fn get_sled_address(sled_subnet: Ipv6Network) -> SocketAddrV6 {
+    assert_eq!(sled_subnet.prefix(), SLED_PREFIX);
+
+    let mut iter = sled_subnet.iter();
+    let _anycast_ip = iter.next().unwrap();
+    let sled_agent_ip = iter.next().unwrap();
+    SocketAddrV6::new(sled_agent_ip, SLED_AGENT_PORT, 0, 0)
+}
+
+/// Returns a sled subnet within a rack subnet.
+///
+/// The subnet at index == 0 is used for rack-local services.
+pub fn get_64_subnet(rack_subnet: Ipv6Network, index: u8) -> Ipv6Network {
+    assert_eq!(rack_subnet.prefix(), RACK_PREFIX);
+
+    let mut rack_network = rack_subnet.network().octets();
+
+    // To set bits distinguishing the /64 from the /56, we modify the 7th octet.
+    rack_network[7] = index;
+    Ipv6Network::new(Ipv6Addr::from(rack_network), 64).unwrap()
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_dns_subnets() {
+        let subnet = "fd00:1122:3344:0100::/64".parse::<Ipv6Network>().unwrap();
+        let rack_subnet = ReservedRackSubnet::new(subnet);
+
+        assert_eq!(
+            //              Note that these bits (indicating the rack) are zero.
+            //              vv
+            "fd00:1122:3344:0000::/56".parse::<Ipv6Network>().unwrap(),
+            rack_subnet.0,
+        );
+
+        // Observe the first DNS subnet within this reserved rack subnet.
+        let dns_subnets = rack_subnet.get_dns_subnets();
+        assert_eq!(DNS_REDUNDANCY, dns_subnets.len());
+
+        // The DNS address and GZ address should be only differing by one.
+        assert_eq!(
+            "fd00:1122:3344:0001::1/64".parse::<Ipv6Network>().unwrap(),
+            dns_subnets[0].dns_address(),
+        );
+        assert_eq!(
+            "fd00:1122:3344:0001::2/64".parse::<Ipv6Network>().unwrap(),
+            dns_subnets[0].gz_address(),
+        );
+    }
+
+    #[test]
+    fn test_sled_address() {
+        let subnet = "fd00:1122:3344:0101::/64".parse::<Ipv6Network>().unwrap();
+        assert_eq!(
+            "[fd00:1122:3344:0101::1]:12345".parse::<SocketAddrV6>().unwrap(),
+            get_sled_address(subnet)
+        );
+
+        let subnet = "fd00:1122:3344:0308::/64".parse::<Ipv6Network>().unwrap();
+        assert_eq!(
+            "[fd00:1122:3344:0308::1]:12345".parse::<SocketAddrV6>().unwrap(),
+            get_sled_address(subnet)
+        );
+    }
+}

common/src/lib.rs

@@ -20,6 +20,7 @@
 // TODO(#32): Remove this exception once resolved.
 #![allow(clippy::field_reassign_with_default)]
 
+pub mod address;
 pub mod api;
 pub mod backoff;
 pub mod cmd;

docs/how-to-run.adoc

@@ -119,15 +119,16 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1:
 |===================================================================================================
 | Service                    | Endpoint
 | Sled Agent: Bootstrap      | Derived from MAC address of physical data link.
-| Sled Agent: Dropshot API   | `[fd00:1122:3344:1::1]:12345`
-| Cockroach DB               | `[fd00:1122:3344:1::2]:32221`
-| Nexus: External API        | `[fd00:1122:3344:1::3]:12220`
-| Nexus: Internal API        | `[fd00:1122:3344:1::3]:12221`
-| Oximeter                   | `[fd00:1122:3344:1::4]:12223`
-| Clickhouse                 | `[fd00:1122:3344:1::5]:8123`
-| Crucible Downstairs 1      | `[fd00:1122:3344:1::6]:32345`
-| Crucible Downstairs 2      | `[fd00:1122:3344:1::7]:32345`
-| Crucible Downstairs 3      | `[fd00:1122:3344:1::8]:32345`
+| Sled Agent: Dropshot API   | `[fd00:1122:3344:0101::1]:12345`
+| Cockroach DB               | `[fd00:1122:3344:0101::2]:32221`
+| Nexus: External API        | `[fd00:1122:3344:0101::3]:12220`
+| Nexus: Internal API        | `[fd00:1122:3344:0101::3]:12221`
+| Oximeter                   | `[fd00:1122:3344:0101::4]:12223`
+| Clickhouse                 | `[fd00:1122:3344:0101::5]:8123`
+| Crucible Downstairs 1      | `[fd00:1122:3344:0101::6]:32345`
+| Crucible Downstairs 2      | `[fd00:1122:3344:0101::7]:32345`
+| Crucible Downstairs 3      | `[fd00:1122:3344:0101::8]:32345`
+| Internal DNS Service       | `[fd00:1122:3344:0001::1]:5353`
 |===================================================================================================
 
 Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other

internal-dns-client/Cargo.toml

@@ -5,15 +5,9 @@ edition = "2021"
 license = "MPL-2.0"
 
 [dependencies]
-anyhow = "1.0"
-clap = { version = "3.1", features = [ "derive" ] }
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"
 slog = { version = "2.5.0", features = [ "max_level_trace", "release_max_level_debug" ] }
-slog-term = "2.7"
-slog-async = "2.7"
-slog-envlogger = "2.2"
 structopt = "0.3"
-tokio = { version = "1.17", features = [ "full" ] }
 reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] }

internal-dns/Cargo.toml

@@ -7,6 +7,7 @@ license = "MPL-2.0"
 [dependencies]
 anyhow = "1.0"
 clap = { version = "3.1", features = [ "derive" ] }
+internal-dns-client = { path = "../internal-dns-client" }
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
 pretty-hex = "0.3.0"
 schemars = "0.8"
@@ -26,7 +27,6 @@ trust-dns-server = "0.21"
 
 [dev-dependencies]
 expectorate = "1.0.5"
-internal-dns-client = { path = "../internal-dns-client" }
 omicron-test-utils = { path = "../test-utils" }
 openapiv3 = "1.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }

internal-dns/src/bin/dns-server.rs

@@ -13,23 +13,34 @@
 use anyhow::anyhow;
 use anyhow::Context;
 use clap::Parser;
+use std::net::{SocketAddr, SocketAddrV6};
 use std::path::PathBuf;
 use std::sync::Arc;
 
 #[derive(Parser, Debug)]
 struct Args {
     #[clap(long)]
     config_file: PathBuf,
+
+    #[clap(long)]
+    server_address: SocketAddrV6,
+
+    #[clap(long)]
+    dns_address: SocketAddrV6,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
     let args = Args::parse();
     let config_file = &args.config_file;
+    let dns_address = &args.dns_address;
     let config_file_contents = std::fs::read_to_string(config_file)
         .with_context(|| format!("read config file {:?}", config_file))?;
-    let config: internal_dns::Config = toml::from_str(&config_file_contents)
-        .with_context(|| format!("parse config file {:?}", config_file))?;
+    let mut config: internal_dns::Config =
+        toml::from_str(&config_file_contents)
+            .with_context(|| format!("parse config file {:?}", config_file))?;
+
+    config.dropshot.bind_address = SocketAddr::V6(args.server_address);
     eprintln!("{:?}", config);
 
     let log = config
@@ -42,10 +53,11 @@ async fn main() -> Result<(), anyhow::Error> {
     {
         let db = db.clone();
         let log = log.clone();
-        let config = config.dns.clone();
-
+        let dns_config = internal_dns::dns_server::Config {
+            bind_address: dns_address.to_string(),
+        };
         tokio::spawn(async move {
-            internal_dns::dns_server::run(log, db, config).await
+            internal_dns::dns_server::run(log, db, dns_config).await
         });
     }
 

internal-dns/src/lib.rs

@@ -18,7 +18,6 @@ pub struct Config {
     pub log: dropshot::ConfigLogging,
     pub dropshot: dropshot::ConfigDropshot,
     pub data: dns_data::Config,
-    pub dns: dns_server::Config,
 }
 
 pub async fn start_server(

internal-dns/tests/basic_test.rs

@@ -140,10 +140,12 @@ async fn init_client_server(
     {
         let db = db.clone();
         let log = log.clone();
-        let config = config.dns.clone();
+        let dns_config = internal_dns::dns_server::Config {
+            bind_address: format!("127.0.0.1:{}", dns_port),
+        };
 
         tokio::spawn(async move {
-            internal_dns::dns_server::run(log, db, config).await
+            internal_dns::dns_server::run(log, db, dns_config).await
         });
     }
 
@@ -184,9 +186,6 @@ fn test_config() -> Result<(internal_dns::Config, u16, u16), anyhow::Error> {
             nmax_messages: 16,
             storage_path,
         },
-        dns: internal_dns::dns_server::Config {
-            bind_address: format!("127.0.0.1:{}", dns_port).parse().unwrap(),
-        },
     };
 
     Ok((config, dropshot_port, dns_port))

openapi/sled-agent.json

@@ -965,7 +965,15 @@
           "addresses": {
             "type": "array",
             "items": {
-              "type": "string"
+              "type": "string",
+              "format": "ipv6"
+            }
+          },
+          "gz_addresses": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "format": "ipv6"
             }
           },
           "name": {

package-manifest.toml

@@ -58,6 +58,15 @@ to = "/opt/oxide/cockroachdb/sql"
 from = "smf/cockroachdb"
 to = "/var/svc/manifest/site/cockroachdb"
 
+[package.internal-dns]
+rust.binary_names = ["dnsadm", "dns-server"]
+rust.release = true
+service_name = "internal-dns"
+zone = true
+[[package.internal-dns.paths]]
+from = "smf/internal-dns"
+to = "/var/svc/manifest/site/internal-dns"
+
 # Packages not built within Omicron, but which must be imported.
 
 # Refer to