oxidecomputer · smklein · Jun 24, 2022 · Jun 8, 2022 · Jun 8, 2022 · Jun 8, 2022
diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs
@@ -102,6 +102,8 @@ pub enum Database {
 pub struct DeploymentConfig {
     /// Uuid of the Nexus instance
     pub id: Uuid,
+    /// Uuid of the Rack where Nexus is executing.
+    pub rack_id: Uuid,
 let rack_id = Uuid::new_v4(); 
 let rack_id = Uuid::new_v4(); 
     /// Dropshot configuration for external API server
     pub dropshot_external: ConfigDropshot,
     /// Dropshot configuration for internal API server

diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql
@@ -75,6 +75,9 @@ CREATE TABLE omicron.public.sled (
     time_deleted TIMESTAMPTZ,
     rcgen INT NOT NULL,
 
+    /* FK into the Rack table */
+    rack_id UUID NOT NULL,
+
     /* The IP address and bound port of the sled agent server. */
     ip INET NOT NULL,
     port INT4 CHECK (port BETWEEN 0 AND 65535) NOT NULL,
@@ -83,6 +86,11 @@ CREATE TABLE omicron.public.sled (
     last_used_address INET NOT NULL
 );
 
+/* Add an index which lets us look up sleds on a rack */
+CREATE INDEX ON omicron.public.sled (
+    rack_id
+) WHERE time_deleted IS NULL;
+
 /*
  * Services
  */

diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml
@@ -36,6 +36,7 @@ address = "[::1]:8123"
 [deployment]
 # Identifier for this instance of Nexus
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
 [deployment.dropshot_external]
 # IP address and TCP port on which to listen for the external API

diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
@@ -222,6 +222,16 @@ impl Nexus {
         &self.opctx_external_authn
     }
 
+    /// Returns an [`OpContext`] used for balancing services.
+    pub fn opctx_for_service_balancer(&self) -> OpContext {
+        OpContext::for_background(
+            self.log.new(o!("component" => "ServiceBalancer")),
+            Arc::clone(&self.authz),
+            authn::Context::internal_service_balancer(),
+            Arc::clone(&self.db_datastore),
+        )
+    }
+
     /// Used as the body of a "stub" endpoint -- one that's currently
     /// unimplemented but that we eventually intend to implement
     ///

diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
@@ -31,7 +31,7 @@ impl super::Nexus {
         address: SocketAddrV6,
     ) -> Result<(), Error> {
         info!(self.log, "registered sled agent"; "sled_uuid" => id.to_string());
-        let sled = db::model::Sled::new(id, address);
+        let sled = db::model::Sled::new(id, address, self.rack_id);
         self.db_datastore.sled_upsert(sled).await?;
         Ok(())
     }

diff --git a/nexus/src/authn/mod.rs b/nexus/src/authn/mod.rs
@@ -35,6 +35,7 @@ pub use crate::db::fixed_data::user_builtin::USER_EXTERNAL_AUTHN;
 pub use crate::db::fixed_data::user_builtin::USER_INTERNAL_API;
 pub use crate::db::fixed_data::user_builtin::USER_INTERNAL_READ;
 pub use crate::db::fixed_data::user_builtin::USER_SAGA_RECOVERY;
+pub use crate::db::fixed_data::user_builtin::USER_SERVICE_BALANCER;
 use crate::db::model::ConsoleSession;
 
 use crate::authz;
@@ -170,6 +171,11 @@ impl Context {
         Context::context_for_builtin_user(USER_DB_INIT.id)
     }
 
+    /// Returns an authenticated context for Nexus-driven service balancing.
+    pub fn internal_service_balancer() -> Context {
+        Context::context_for_builtin_user(USER_SERVICE_BALANCER.id)
+    }
+
     fn context_for_builtin_user(user_builtin_id: Uuid) -> Context {
         Context {
             kind: Kind::Authenticated(Details {
@@ -217,6 +223,7 @@ mod test {
     use super::USER_INTERNAL_API;
     use super::USER_INTERNAL_READ;
     use super::USER_SAGA_RECOVERY;
+    use super::USER_SERVICE_BALANCER;
     use super::USER_TEST_PRIVILEGED;
     use super::USER_TEST_UNPRIVILEGED;
     use crate::db::fixed_data::user_builtin::USER_EXTERNAL_AUTHN;
@@ -251,6 +258,10 @@ mod test {
         let actor = authn.actor().unwrap();
         assert_eq!(actor.actor_id(), USER_DB_INIT.id);
 
+        let authn = Context::internal_service_balancer();
+        let actor = authn.actor().unwrap();
+        assert_eq!(actor.actor_id(), USER_SERVICE_BALANCER.id);
+
         let authn = Context::internal_saga_recovery();
         let actor = authn.actor().unwrap();
         assert_eq!(actor.actor_id(), USER_SAGA_RECOVERY.id);

diff --git a/nexus/src/config.rs b/nexus/src/config.rs
@@ -329,6 +329,7 @@ mod test {
             max_vpc_ipv4_subnet_prefix = 27
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
@@ -348,6 +349,9 @@ mod test {
             Config {
                 deployment: DeploymentConfig {
                     id: "28b90dc4-c22a-65ba-f49a-f051fe01208f".parse().unwrap(),
+                    rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
+                        .parse()
+                        .unwrap(),
                     dropshot_external: ConfigDropshot {
                         bind_address: "10.1.2.3:4567"
                             .parse::<SocketAddr>()
@@ -407,6 +411,7 @@ mod test {
             address = "[::1]:8123"
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
@@ -448,6 +453,7 @@ mod test {
             address = "[::1]:8123"
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
@@ -503,6 +509,7 @@ mod test {
             max_vpc_ipv4_subnet_prefix = 100
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024

diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs
@@ -2990,6 +2990,7 @@ impl DataStore {
         let builtin_users = [
             // Note: "db_init" is also a builtin user, but that one by necessity
             // is created with the database.
+            &*authn::USER_SERVICE_BALANCER,
             &*authn::USER_INTERNAL_API,
             &*authn::USER_INTERNAL_READ,
             &*authn::USER_EXTERNAL_AUTHN,
@@ -4034,8 +4035,9 @@ mod test {
             0,
             0,
         );
+        let rack_id = Uuid::new_v4();
         let sled_id = Uuid::new_v4();
-        let sled = Sled::new(sled_id, bogus_addr.clone());
+        let sled = Sled::new(sled_id, bogus_addr.clone(), rack_id);
         datastore.sled_upsert(sled).await.unwrap();
         sled_id
     }
@@ -4391,14 +4393,15 @@ mod test {
         let opctx =
             OpContext::for_tests(logctx.log.new(o!()), datastore.clone());
 
+        let rack_id = Uuid::new_v4();
         let addr1 = "[fd00:1de::1]:12345".parse().unwrap();
         let sled1_id = "0de4b299-e0b4-46f0-d528-85de81a7095f".parse().unwrap();
-        let sled1 = db::model::Sled::new(sled1_id, addr1);
+        let sled1 = db::model::Sled::new(sled1_id, addr1, rack_id);
         datastore.sled_upsert(sled1).await.unwrap();
 
         let addr2 = "[fd00:1df::1]:12345".parse().unwrap();
         let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap();
-        let sled2 = db::model::Sled::new(sled2_id, addr2);
+        let sled2 = db::model::Sled::new(sled2_id, addr2, rack_id);
         datastore.sled_upsert(sled2).await.unwrap();
 
         let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap();

diff --git a/nexus/src/db/fixed_data/role_assignment.rs b/nexus/src/db/fixed_data/role_assignment.rs
@@ -24,6 +24,13 @@ lazy_static! {
                 *FLEET_ID,
                 role_builtin::FLEET_ADMIN.role_name,
             ),
+            RoleAssignment::new(
+                IdentityType::UserBuiltin,
+                user_builtin::USER_SERVICE_BALANCER.id,
+                role_builtin::FLEET_ADMIN.resource_type,
+                *FLEET_ID,
+                role_builtin::FLEET_ADMIN.role_name,
+            ),
 
             // The "internal-read" user gets the "viewer" role on the sole
             // Fleet.  This will grant them the ability to read various control

diff --git a/nexus/src/db/fixed_data/user_builtin.rs b/nexus/src/db/fixed_data/user_builtin.rs
@@ -39,6 +39,15 @@ lazy_static! {
             "used for seeding initial database data",
         );
 
+    /// Internal user for performing operations to manage the
+    /// provisioning of services across the fleet.
+    pub static ref USER_SERVICE_BALANCER: UserBuiltinConfig =
+        UserBuiltinConfig::new_static(
+            "001de000-05e4-4000-8000-00000000bac3",
+            "service-balancer",
+            "used for Nexus-driven service balancing",
+        );
+
     /// Internal user used by Nexus when handling internal API requests
     pub static ref USER_INTERNAL_API: UserBuiltinConfig =
         UserBuiltinConfig::new_static(
@@ -82,9 +91,11 @@ mod test {
     use super::USER_INTERNAL_API;
     use super::USER_INTERNAL_READ;
     use super::USER_SAGA_RECOVERY;
+    use super::USER_SERVICE_BALANCER;
 
     #[test]
     fn test_builtin_user_ids_are_valid() {
+        assert_valid_uuid(&USER_SERVICE_BALANCER.id);
         assert_valid_uuid(&USER_DB_INIT.id);
         assert_valid_uuid(&USER_INTERNAL_API.id);
         assert_valid_uuid(&USER_EXTERNAL_AUTHN.id);

diff --git a/nexus/src/db/model/sled.rs b/nexus/src/db/model/sled.rs
@@ -21,6 +21,8 @@ pub struct Sled {
     time_deleted: Option<DateTime<Utc>>,
     rcgen: Generation,
 
+    pub rack_id: Uuid,
+
     // ServiceAddress (Sled Agent).
     pub ip: ipv6::Ipv6Addr,
     pub port: SqlU16,
@@ -30,7 +32,7 @@ pub struct Sled {
 }
 
 impl Sled {
-    pub fn new(id: Uuid, addr: SocketAddrV6) -> Self {
+    pub fn new(id: Uuid, addr: SocketAddrV6, rack_id: Uuid) -> Self {
         let last_used_address = {
             let mut segments = addr.ip().segments();
             segments[7] += omicron_common::address::RSS_RESERVED_ADDRESSES;
@@ -40,6 +42,7 @@ impl Sled {
             identity: SledIdentity::new(id),
             time_deleted: None,
             rcgen: Generation::new(),
+            rack_id,
             ip: ipv6::Ipv6Addr::from(addr.ip()),
             port: addr.port().into(),
             last_used_address,

diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs
@@ -297,6 +297,7 @@ table! {
         time_deleted -> Nullable<Timestamptz>,
         rcgen -> Int8,
 
+        rack_id -> Uuid,
         ip -> Inet,
         port -> Int4,
         last_used_address -> Inet,

diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
@@ -36,7 +36,6 @@ use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use slog::Logger;
 use std::sync::Arc;
-use uuid::Uuid;
 
 #[macro_use]
 extern crate slog;
@@ -82,15 +81,15 @@ impl Server {
     /// Start a nexus server.
     pub async fn start(
         config: &Config,
-        rack_id: Uuid,
         log: &Logger,
     ) -> Result<Server, String> {
         let log = log.new(o!("name" => config.deployment.id.to_string()));
         info!(log, "setting up nexus server");
 
         let ctxlog = log.new(o!("component" => "ServerContext"));
 
-        let apictx = ServerContext::new(rack_id, ctxlog, &config)?;
+        let apictx =
+            ServerContext::new(config.deployment.rack_id, ctxlog, &config)?;
 
         let http_server_starter_external = dropshot::HttpServerStarter::new(
             &config.deployment.dropshot_external,
@@ -167,8 +166,7 @@ pub async fn run_server(config: &Config) -> Result<(), String> {
     } else {
         debug!(log, "registered DTrace probes");
     }
-    let rack_id = Uuid::new_v4();
-    let server = Server::start(config, rack_id, &log).await?;
+    let server = Server::start(config, &log).await?;
     server.register_as_producer().await;
     server.wait_for_finish().await
 }
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
@@ -90,7 +90,6 @@ pub async fn test_setup_with_config(
     config: &mut omicron_nexus::Config,
 ) -> ControlPlaneTestContext {
     let logctx = LogContext::new(test_name, &config.pkg.log);
-    let rack_id = Uuid::parse_str(RACK_UUID).unwrap();
     let log = &logctx.log;
 
     // Start up CockroachDB.
@@ -104,9 +103,8 @@ pub async fn test_setup_with_config(
         nexus_config::Database::FromUrl { url: database.pg_config().clone() };
     config.pkg.timeseries_db.address.set_port(clickhouse.port());
 
-    let server = omicron_nexus::Server::start(&config, rack_id, &logctx.log)
-        .await
-        .unwrap();
+    let server =
+        omicron_nexus::Server::start(&config, &logctx.log).await.unwrap();
     server
         .apictx
         .nexus

diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml
@@ -39,6 +39,7 @@ max_vpc_ipv4_subnet_prefix = 29
 # Identifier for this instance of Nexus.
 # NOTE: The test suite always overrides this.
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
 #
 # NOTE: for the test suite, the port MUST be 0 (in order to bind to any

diff --git a/nexus/tests/integration_tests/users_builtin.rs b/nexus/tests/integration_tests/users_builtin.rs
@@ -27,6 +27,9 @@ async fn test_users_builtin(cptestctx: &ControlPlaneTestContext) {
 
     let u = users.remove(&authn::USER_DB_INIT.name.to_string()).unwrap();
     assert_eq!(u.identity.id, authn::USER_DB_INIT.id);
+    let u =
+        users.remove(&authn::USER_SERVICE_BALANCER.name.to_string()).unwrap();
+    assert_eq!(u.identity.id, authn::USER_SERVICE_BALANCER.id);
     let u = users.remove(&authn::USER_INTERNAL_API.name.to_string()).unwrap();
     assert_eq!(u.identity.id, authn::USER_INTERNAL_API.id);
     let u = users.remove(&authn::USER_INTERNAL_READ.name.to_string()).unwrap();

diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs
@@ -245,6 +245,7 @@ impl Agent {
             &self.sled_config,
             self.parent_log.clone(),
             sled_address,
+            request.rack_id,
         )
         .await
         .map_err(|e| {

diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs
@@ -8,13 +8,20 @@ use super::trust_quorum::ShareDistribution;
 use omicron_common::address::{Ipv6Subnet, SLED_PREFIX};
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
+use uuid::Uuid;
 
 /// Configuration information for launching a Sled Agent.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct SledAgentRequest {
+    /// Uuid of the Sled Agent to be created.
+    pub id: Uuid,
+
     /// Portion of the IP space to be managed by the Sled Agent.
     pub subnet: Ipv6Subnet<SLED_PREFIX>,
 
+    /// Uuid of the rack to which this sled agent belongs.
+    pub rack_id: Uuid,
+
     /// Share of the rack secret for this Sled Agent.
     // TODO-cleanup This is currently optional because we don't do trust quorum
     // shares for single-node deployments (i.e., most dev/test environments),

diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs
@@ -357,6 +357,7 @@ impl ServiceInner {
                 (request, (idx, bootstrap_addr))
             });
 
+        let rack_id = Uuid::new_v4();
         let allocations = requests_and_sleds.map(|(request, sled)| {
             let (idx, bootstrap_addr) = sled;
             info!(
@@ -373,7 +374,9 @@ impl ServiceInner {
                 bootstrap_addr,
                 SledAllocation {
                     initialization_request: SledAgentRequest {
+                        id: Uuid::new_v4(),
                         subnet,
+                        rack_id,
                         trust_quorum_share: maybe_rack_secret_shares
                             .as_mut()
                             .map(|shares_iter| {