[internal-dns][sled agent] Assign IPs to DNS service from reserved subnet

common/src/address.rs

@@ -0,0 +1,200 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Common IP addressing functionality.
+//!
+//! This addressing functionality is shared by both initialization services
+//! and Nexus, who need to agree upon addressing schemes.
+
+use crate::api::external::Ipv6Net;
+use ipnetwork::Ipv6Network;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::net::{Ipv6Addr, SocketAddrV6};
+
+pub const AZ_PREFIX: u8 = 48;
+pub const RACK_PREFIX: u8 = 56;
+pub const SLED_PREFIX: u8 = 64;
+
+/// The amount of redundancy for DNS servers.
+///
+/// Must be less than MAX_DNS_REDUNDANCY.
+pub const DNS_REDUNDANCY: usize = 1;
+/// The maximum amount of redundancy for DNS servers.
+///
+/// This determines the number of addresses which are
+/// reserved for DNS servers.
+pub const MAX_DNS_REDUNDANCY: usize = 5;
+
+pub const DNS_PORT: u16 = 53;
+pub const DNS_SERVER_PORT: u16 = 5353;
+pub const SLED_AGENT_PORT: u16 = 12345;
+
+// Anycast is a mechanism in which a single IP address is shared by multiple
+// devices, and the destination is located based on routing distance.
+//
+// This is covered by RFC 4291 in much more detail:
+// <https://datatracker.ietf.org/doc/html/rfc4291#section-2.6>
+//
+// Anycast addresses are always the "zeroeth" address within a subnet.  We
+// always explicitly skip these addresses within our network.
+const _ANYCAST_ADDRESS_INDEX: usize = 0;
+const DNS_ADDRESS_INDEX: usize = 1;
+const GZ_ADDRESS_INDEX: usize = 2;
+
+/// Wraps an [`Ipv6Network`] with a compile-time prefix length.
+#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize, PartialEq)]
+pub struct Ipv6Subnet<const N: u8> {
+    net: Ipv6Net,
+}
+
+impl<const N: u8> Ipv6Subnet<N> {
+    pub fn new(addr: Ipv6Addr) -> Self {
+        // Create a network with the compile-time prefix length.
+        let net = Ipv6Network::new(addr, N).unwrap();
+        // Ensure the address is set to within-prefix only components.
+        let net = Ipv6Network::new(net.network(), N).unwrap();
+        Self { net: Ipv6Net(net) }
+    }
+
+    /// Returns the underlying network.
+    pub fn net(&self) -> Ipv6Network {
+        self.net.0
+    }
+}
+
+/// Represents a subnet which may be used for contacting DNS services.
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+pub struct DnsSubnet {
+    subnet: Ipv6Subnet<SLED_PREFIX>,
+}
+
+impl DnsSubnet {
+    /// Returns the DNS server address within the subnet.
+    ///
+    /// This is the first address within the subnet.
+    pub fn dns_address(&self) -> Ipv6Network {
+        Ipv6Network::new(
+            self.subnet.net().iter().nth(DNS_ADDRESS_INDEX).unwrap(),
+            SLED_PREFIX,
+        )
+        .unwrap()
+    }
+
+    /// Returns the address which the Global Zone should create
+    /// to be able to contact the DNS server.
+    ///
+    /// This is the second address within the subnet.
+    pub fn gz_address(&self) -> Ipv6Network {
+        Ipv6Network::new(
+            self.subnet.net().iter().nth(GZ_ADDRESS_INDEX).unwrap(),
+            SLED_PREFIX,
+        )
+        .unwrap()
+    }
+}
+
+/// A wrapper around an IPv6 network, indicating it is a "reserved" rack
+/// subnet which can be used for AZ-wide services.
+#[derive(Debug, Clone)]
+pub struct ReservedRackSubnet(pub Ipv6Subnet<RACK_PREFIX>);
+
+impl ReservedRackSubnet {
+    /// Returns the subnet for the reserved rack subnet.
+    pub fn new(subnet: Ipv6Subnet<AZ_PREFIX>) -> Self {
+        ReservedRackSubnet(Ipv6Subnet::<RACK_PREFIX>::new(subnet.net().ip()))
+    }
+
+    /// Returns the DNS addresses from this reserved rack subnet.
+    ///
+    /// These addresses will come from the first [`DNS_REDUNDANCY`] `/64s` of the
+    /// [`RACK_PREFIX`] subnet.
+    pub fn get_dns_subnets(&self) -> Vec<DnsSubnet> {
+        (0..DNS_REDUNDANCY)
+            .map(|idx| {
+                let subnet =
+                    get_64_subnet(self.0, u8::try_from(idx + 1).unwrap());
+                DnsSubnet { subnet }
+            })
+            .collect()
+    }
+}
+
+const SLED_AGENT_ADDRESS_INDEX: usize = 1;
+
+/// Return the sled agent address for a subnet.
+///
+/// This address will come from the first address of the [`SLED_PREFIX`] subnet.
+pub fn get_sled_address(sled_subnet: Ipv6Subnet<SLED_PREFIX>) -> SocketAddrV6 {
+    let sled_agent_ip =
+        sled_subnet.net().iter().nth(SLED_AGENT_ADDRESS_INDEX).unwrap();
+    SocketAddrV6::new(sled_agent_ip, SLED_AGENT_PORT, 0, 0)
+}
+
+/// Returns a sled subnet within a rack subnet.
+///
+/// The subnet at index == 0 is used for rack-local services.
+pub fn get_64_subnet(
+    rack_subnet: Ipv6Subnet<RACK_PREFIX>,
+    index: u8,
+) -> Ipv6Subnet<SLED_PREFIX> {
+    let mut rack_network = rack_subnet.net().network().octets();
+
+    // To set bits distinguishing the /64 from the /56, we modify the 7th octet.
+    rack_network[7] = index;
+    Ipv6Subnet::<SLED_PREFIX>::new(Ipv6Addr::from(rack_network))
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_dns_subnets() {
+        let subnet = Ipv6Subnet::<AZ_PREFIX>::new(
+            "fd00:1122:3344:0100::".parse::<Ipv6Addr>().unwrap(),
+        );
+        let rack_subnet = ReservedRackSubnet::new(subnet);
+
+        assert_eq!(
+            //              Note that these bits (indicating the rack) are zero.
+            //              vv
+            "fd00:1122:3344:0000::/56".parse::<Ipv6Network>().unwrap(),
+            rack_subnet.0.net(),
+        );
+
+        // Observe the first DNS subnet within this reserved rack subnet.
+        let dns_subnets = rack_subnet.get_dns_subnets();
+        assert_eq!(DNS_REDUNDANCY, dns_subnets.len());
+
+        // The DNS address and GZ address should be only differing by one.
+        assert_eq!(
+            "fd00:1122:3344:0001::1/64".parse::<Ipv6Network>().unwrap(),
+            dns_subnets[0].dns_address(),
+        );
+        assert_eq!(
+            "fd00:1122:3344:0001::2/64".parse::<Ipv6Network>().unwrap(),
+            dns_subnets[0].gz_address(),
+        );
+    }
+
+    #[test]
+    fn test_sled_address() {
+        let subnet = Ipv6Subnet::<SLED_PREFIX>::new(
+            "fd00:1122:3344:0101::".parse::<Ipv6Addr>().unwrap(),
+        );
+        assert_eq!(
+            "[fd00:1122:3344:0101::1]:12345".parse::<SocketAddrV6>().unwrap(),
+            get_sled_address(subnet)
+        );
+
+        let subnet = Ipv6Subnet::<SLED_PREFIX>::new(
+            "fd00:1122:3344:0308::".parse::<Ipv6Addr>().unwrap(),
+        );
+        assert_eq!(
+            "[fd00:1122:3344:0308::1]:12345".parse::<SocketAddrV6>().unwrap(),
+            get_sled_address(subnet)
+        );
+    }
+}

common/src/lib.rs

@@ -20,6 +20,7 @@
 // TODO(#32): Remove this exception once resolved.
 #![allow(clippy::field_reassign_with_default)]
 
+pub mod address;
 pub mod api;
 pub mod backoff;
 pub mod cmd;

docs/how-to-run.adoc

@@ -119,16 +119,16 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1:
 |===================================================================================================
 | Service                    | Endpoint
 | Sled Agent: Bootstrap      | Derived from MAC address of physical data link.
-| Sled Agent: Dropshot API   | `[fd00:1122:3344:1::1]:12345`
-| Cockroach DB               | `[fd00:1122:3344:1::2]:32221`
-| Nexus: External API        | `[fd00:1122:3344:1::3]:12220`
-| Nexus: Internal API        | `[fd00:1122:3344:1::3]:12221`
-| Oximeter                   | `[fd00:1122:3344:1::4]:12223`
-| Clickhouse                 | `[fd00:1122:3344:1::5]:8123`
-| Crucible Downstairs 1      | `[fd00:1122:3344:1::6]:32345`
-| Crucible Downstairs 2      | `[fd00:1122:3344:1::7]:32345`
-| Crucible Downstairs 3      | `[fd00:1122:3344:1::8]:32345`
-| Internal DNS               | `[fd00:1122:3344:1::9]:5353`
+| Sled Agent: Dropshot API   | `[fd00:1122:3344:0101::1]:12345`
+| Cockroach DB               | `[fd00:1122:3344:0101::2]:32221`
+| Nexus: External API        | `[fd00:1122:3344:0101::3]:12220`
+| Nexus: Internal API        | `[fd00:1122:3344:0101::3]:12221`
+| Oximeter                   | `[fd00:1122:3344:0101::4]:12223`
+| Clickhouse                 | `[fd00:1122:3344:0101::5]:8123`
+| Crucible Downstairs 1      | `[fd00:1122:3344:0101::6]:32345`
+| Crucible Downstairs 2      | `[fd00:1122:3344:0101::7]:32345`
+| Crucible Downstairs 3      | `[fd00:1122:3344:0101::8]:32345`
+| Internal DNS Service       | `[fd00:1122:3344:0001::1]:5353`
 |===================================================================================================
 
 Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other

internal-dns/src/bin/dns-server.rs

@@ -13,23 +13,34 @@
 use anyhow::anyhow;
 use anyhow::Context;
 use clap::Parser;
+use std::net::{SocketAddr, SocketAddrV6};
 use std::path::PathBuf;
 use std::sync::Arc;
 
 #[derive(Parser, Debug)]
 struct Args {
     #[clap(long)]
     config_file: PathBuf,
+
+    #[clap(long)]
+    server_address: SocketAddrV6,
+
+    #[clap(long)]
+    dns_address: SocketAddrV6,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
     let args = Args::parse();
     let config_file = &args.config_file;
+    let dns_address = &args.dns_address;
     let config_file_contents = std::fs::read_to_string(config_file)
         .with_context(|| format!("read config file {:?}", config_file))?;
-    let config: internal_dns::Config = toml::from_str(&config_file_contents)
-        .with_context(|| format!("parse config file {:?}", config_file))?;
+    let mut config: internal_dns::Config =
+        toml::from_str(&config_file_contents)
+            .with_context(|| format!("parse config file {:?}", config_file))?;
+
+    config.dropshot.bind_address = SocketAddr::V6(args.server_address);
     eprintln!("{:?}", config);
 
     let log = config
@@ -42,10 +53,11 @@ async fn main() -> Result<(), anyhow::Error> {
     {
         let db = db.clone();
         let log = log.clone();
-        let config = config.dns.clone();
-
+        let dns_config = internal_dns::dns_server::Config {
+            bind_address: dns_address.to_string(),
+        };
         tokio::spawn(async move {
-            internal_dns::dns_server::run(log, db, config).await
+            internal_dns::dns_server::run(log, db, dns_config).await
         });
     }
 

internal-dns/src/lib.rs

@@ -18,7 +18,6 @@ pub struct Config {
     pub log: dropshot::ConfigLogging,
     pub dropshot: dropshot::ConfigDropshot,
     pub data: dns_data::Config,
-    pub dns: dns_server::Config,
 }
 
 pub async fn start_server(

internal-dns/tests/basic_test.rs

@@ -2,15 +2,14 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
+use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6};
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use internal_dns_client::{
     types::{DnsKv, DnsRecord, DnsRecordKey, Srv},
     Client,
 };
-use std::net::Ipv6Addr;
 use trust_dns_resolver::config::{
     NameServerConfig, Protocol, ResolverConfig, ResolverOpts,
 };
@@ -135,16 +134,16 @@ async fn init_client_server() -> Result<TestContext, anyhow::Error> {
     let db = Arc::new(sled::open(&config.data.storage_path)?);
     db.clear()?;
 
-    let client = Client::new(
-        &format!("http://127.0.0.1:{}", dropshot_port),
-        log.clone(),
-    );
+    let client =
+        Client::new(&format!("http://[::1]:{}", dropshot_port), log.clone());
 
     let mut rc = ResolverConfig::new();
     rc.add_name_server(NameServerConfig {
-        socket_addr: SocketAddr::V4(SocketAddrV4::new(
-            Ipv4Addr::new(127, 0, 0, 1),
+        socket_addr: SocketAddr::V6(SocketAddrV6::new(
+            Ipv6Addr::LOCALHOST,
             dns_port,
+            0,
+            0,
         )),
         protocol: Protocol::Udp,
         tls_dns_name: None,
@@ -159,10 +158,12 @@ async fn init_client_server() -> Result<TestContext, anyhow::Error> {
     {
         let db = db.clone();
         let log = log.clone();
-        let config = config.dns.clone();
+        let dns_config = internal_dns::dns_server::Config {
+            bind_address: format!("[::1]:{}", dns_port),
+        };
 
         tokio::spawn(async move {
-            internal_dns::dns_server::run(log, db, config).await
+            internal_dns::dns_server::run(log, db, dns_config).await
         });
     }
 
@@ -189,19 +190,14 @@ fn test_config(
             level: dropshot::ConfigLoggingLevel::Info,
         },
         dropshot: dropshot::ConfigDropshot {
-            bind_address: format!("127.0.0.1:{}", dropshot_port)
-                .parse()
-                .unwrap(),
+            bind_address: format!("[::1]:{}", dropshot_port).parse().unwrap(),
             request_body_max_bytes: 1024,
             ..Default::default()
         },
         data: internal_dns::dns_data::Config {
             nmax_messages: 16,
             storage_path,
         },
-        dns: internal_dns::dns_server::Config {
-            bind_address: format!("127.0.0.1:{}", dns_port).parse().unwrap(),
-        },
     };
 
     Ok((tmp_dir, config, dropshot_port, dns_port))