Leave one Instance as authoritative in sending state updates to Nexus. (#667)

For every Instance sled agent manages, it spawns a task to poll for state updates from propolis. After updating its in-memory state, it notifies Nexus to update the DB representation.

In the Live Migration case, we're in a transitory state wherein there will be two distinct Instance objects (possibly across 2 different Sled Agents). We update Nexus here to only write the changes from the destination SA to the DB.

Along with that, this pulls in the latest propolis side changes. We're still lacking actually transfering over all the necessary state but a lot more scaffolding is coming together.

Author: luqmana

Cargo.lock

@@ -38,6 +38,8 @@ pub struct InstanceRuntimeState {
     pub sled_uuid: Uuid,
     /// which propolis-server is running this Instance
     pub propolis_uuid: Uuid,
+    /// the target propolis-server during a migration of this Instance
+    pub dst_propolis_uuid: Option<Uuid>,
     /// address of propolis-server running this Instance
     pub propolis_addr: Option<SocketAddr>,
     /// migration id (if one in process)

common/src/api/internal/nexus.rs

@@ -91,6 +91,13 @@ impl InstanceStateRequested {
     }
 }
 
+/// Instance runtime state to update for a migration.
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct InstanceRuntimeStateMigrateParams {
+    pub migration_id: Uuid,
+    pub dst_propolis_id: Uuid,
+}
+
 /// Used to request an Instance state change from a sled agent
 ///
 /// Right now, it's only the run state and migration id that can
@@ -99,5 +106,5 @@ impl InstanceStateRequested {
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
 pub struct InstanceRuntimeStateRequested {
     pub run_state: InstanceStateRequested,
-    pub migration_id: Option<Uuid>,
+    pub migration_params: Option<InstanceRuntimeStateMigrateParams>,
 }

common/src/api/internal/sled_agent.rs

@@ -252,9 +252,12 @@ CREATE TABLE omicron.public.instance (
      */
     active_server_id UUID,
     /* Identifies the underlying propolis-server backing the instance. */
-    active_propolis_id UUID,
+    active_propolis_id UUID NOT NULL,
     active_propolis_ip INET,
 
+    /* Identifies the target propolis-server during a migration of the instance. */
+    target_propolis_id UUID,
+
     /*
      * Identifies an ongoing migration for this instance.
      */

common/src/sql/dbinit.sql

@@ -68,6 +68,7 @@ impl From<omicron_common::api::internal::nexus::InstanceRuntimeState>
             run_state: s.run_state.into(),
             sled_uuid: s.sled_uuid,
             propolis_uuid: s.propolis_uuid,
+            dst_propolis_uuid: s.dst_propolis_uuid,
             propolis_addr: s.propolis_addr.map(|addr| addr.to_string()),
             migration_uuid: s.migration_uuid,
             ncpus: s.ncpus.into(),
@@ -89,6 +90,7 @@ impl From<&omicron_common::api::internal::nexus::InstanceRuntimeState>
             run_state: s.run_state.into(),
             sled_uuid: s.sled_uuid,
             propolis_uuid: s.propolis_uuid,
+            dst_propolis_uuid: s.dst_propolis_uuid,
             propolis_addr: s.propolis_addr.map(|addr| addr.to_string()),
             migration_uuid: s.migration_uuid,
             ncpus: s.ncpus.into(),

nexus-client/src/lib.rs

@@ -1118,6 +1118,11 @@ impl DataStore {
             .filter(dsl::time_deleted.is_null())
             .filter(dsl::id.eq(*instance_id))
             .filter(dsl::state_generation.lt(new_runtime.gen))
+            .filter(
+                dsl::migration_id
+                    .is_null()
+                    .or(dsl::target_propolis_id.eq(new_runtime.propolis_uuid)),
+            )
             .set(new_runtime.clone())
             .check_if_exists::<Instance>(*instance_id)
             .execute_and_check(self.pool())

nexus/src/db/datastore.rs

@@ -841,6 +841,8 @@ pub struct InstanceRuntimeState {
     pub sled_uuid: Uuid,
     #[column_name = "active_propolis_id"]
     pub propolis_uuid: Uuid,
+    #[column_name = "target_propolis_id"]
+    pub dst_propolis_uuid: Option<Uuid>,
     #[column_name = "active_propolis_ip"]
     pub propolis_ip: Option<ipnetwork::IpNetwork>,
     #[column_name = "migration_id"]
@@ -871,6 +873,7 @@ impl From<internal::nexus::InstanceRuntimeState> for InstanceRuntimeState {
             state: InstanceState::new(state.run_state),
             sled_uuid: state.sled_uuid,
             propolis_uuid: state.propolis_uuid,
+            dst_propolis_uuid: state.dst_propolis_uuid,
             propolis_ip: state.propolis_addr.map(|addr| addr.ip().into()),
             migration_uuid: state.migration_uuid,
             ncpus: state.ncpus.into(),
@@ -889,6 +892,7 @@ impl Into<internal::nexus::InstanceRuntimeState> for InstanceRuntimeState {
             run_state: *self.state.state(),
             sled_uuid: self.sled_uuid,
             propolis_uuid: self.propolis_uuid,
+            dst_propolis_uuid: self.dst_propolis_uuid,
             propolis_addr: self
                 .propolis_ip
                 .map(|ip| SocketAddr::new(ip.ip(), PROPOLIS_PORT)),

nexus/src/db/model.rs

@@ -39,6 +39,7 @@ table! {
         state_generation -> Int8,
         active_server_id -> Uuid,
         active_propolis_id -> Uuid,
+        target_propolis_id -> Nullable<Uuid>,
         active_propolis_ip -> Nullable<Inet>,
         migration_id -> Nullable<Uuid>,
         ncpus -> Int8,