fix(onboard): revalidate managed sandbox identity without waits

src/lib/onboard/sandbox-create/managed-bootstrap-registration.test.ts

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+import { createOnboardCreatedSandboxRegistration } from "../created-sandbox-finalization";
+import {
+  createCreatedSandboxLifecycle,
+  type SandboxRecreateObservation,
+} from "../sandbox-recreate-transaction";
+import { createOnboardCreatedSandboxRegistrationWithManagedLifecycle } from "./orchestration";
+
+describe("managed bootstrap sandbox registration", () => {
+  const lifecycleGeneration = "generation-1";
+  const durableIdentity = "a".repeat(64);
+  const recordedRegistration = {
+    lifecycleGeneration,
+    lifecycleLiveIdentityFingerprint: durableIdentity,
+  };
+
+  function registrationFixture(managedBootstrap: boolean, observation: SandboxRecreateObservation) {
+    const publish = vi.fn();
+    const runtime = {
+      targetGeneration: undefined,
+      registrationFields: {},
+      recordCreated: vi.fn(),
+    } as never;
+    const completeRegistration = createOnboardCreatedSandboxRegistrationWithManagedLifecycle({
+      sandboxName: "alpha",
+      managedBootstrap,
+      sandboxGpuEnabled: false,
+      createdLifecycle: createCreatedSandboxLifecycle(
+        runtime,
+        { sandboxName: "alpha", gatewayName: "nemoclaw" },
+        () => observation,
+        lifecycleGeneration,
+      ),
+      getRecordedRegistration: () => recordedRegistration,
+      createRegistration: createOnboardCreatedSandboxRegistration,
+      registration: {
+        completion: {
+          complete: async (
+            _created,
+            _configuredReceipt,
+            _providerGpuDisposition,
+            _manageDashboard,
+            resolveLifecycleRegistrationFields,
+            lifecycle,
+          ) => {
+            const verified = lifecycle.revalidate(
+              lifecycle.capture(resolveLifecycleRegistrationFields()),
+            );
+            publish(lifecycle.revalidate(verified));
+          },
+        },
+        cleanupBuildContext: vi.fn(),
+        manageDashboard: false,
+        sandboxGpuEnabled: false,
+      },
+    });
+    return {
+      complete: () =>
+        completeRegistration(
+          { lifecycleRegistrationFields: { lifecycleGeneration } } as never,
+          null,
+        ),
+      publish,
+    };
+  }
+
+  it("publishes a managed sandbox when its not Ready identity matches (#10512)", async () => {
+    const fixture = registrationFixture(true, {
+      state: "not_ready",
+      liveIdentityFingerprint: durableIdentity,
+    });
+
+    await expect(fixture.complete()).resolves.toBeUndefined();
+    expect(fixture.publish).toHaveBeenCalledExactlyOnceWith(recordedRegistration);
+  });
+
+  it.each([
+    ["ordinary", false, { state: "not_ready" as const, liveIdentityFingerprint: durableIdentity }],
+    ["missing", true, { state: "not_ready" as const, liveIdentityFingerprint: null }],
+    [
+      "malformed",
+      true,
+      { state: "not_ready" as const, liveIdentityFingerprint: "not-a-fingerprint" },
+    ],
+    ["changed", true, { state: "not_ready" as const, liveIdentityFingerprint: "b".repeat(64) }],
+  ])(
+    "does not publish a %s sandbox from an invalid not Ready observation (#10512)",
+    async (_case, managedBootstrap, observation) => {
+      const fixture = registrationFixture(managedBootstrap, observation);
+
+      await expect(fixture.complete()).rejects.toThrow();
+      expect(fixture.publish).not.toHaveBeenCalled();
+    },
+  );
+});

src/lib/onboard/sandbox-create/orchestration.ts

@@ -33,6 +33,10 @@ import type {
 } from "../managed-workload/hermes-state-volume";
 import type { OwnedSandboxRecreateRuntime } from "../onboard-recreate-journal";
 import type { SandboxGpuConfig } from "../sandbox-gpu-mode";
+import type {
+  CreatedSandboxLifecycle,
+  CreatedSandboxLifecycleRegistration,
+} from "../sandbox-recreate-transaction";
 import type { PortableOnboardRuntimeContext } from "../session-bootstrap";
 import type {
   InferenceRouteReservationAuthority,
@@ -67,6 +71,43 @@ function cancelRecoveryIdentity(
   };
 }
 
+export function createOnboardCreatedSandboxRegistrationWithManagedLifecycle(input: {
+  readonly sandboxName: string;
+  readonly managedBootstrap: boolean;
+  readonly sandboxGpuEnabled: boolean;
+  readonly createdLifecycle: CreatedSandboxLifecycle;
+  readonly getRecordedRegistration: () => CreatedSandboxLifecycleRegistration;
+  readonly createRegistration: SandboxCreateOrchestrationRuntime["createOnboardCreatedSandboxRegistration"];
+  readonly registration: Omit<
+    Parameters<SandboxCreateOrchestrationRuntime["createOnboardCreatedSandboxRegistration"]>[0],
+    "createdLifecycle"
+  >;
+}) {
+  let createdLifecycle = input.createdLifecycle;
+  if (input.managedBootstrap) {
+    const capture = input.sandboxGpuEnabled
+      ? input.createdLifecycle.capture
+      : ({ lifecycleGeneration }: Pick<SandboxEntry, "lifecycleGeneration">) => {
+          const recordedRegistration = input.getRecordedRegistration();
+          if (lifecycleGeneration !== recordedRegistration.lifecycleGeneration) {
+            throw new Error(
+              `Cannot register sandbox '${input.sandboxName}': lifecycle setup did not preserve its generation.`,
+            );
+          }
+          return recordedRegistration;
+        };
+    createdLifecycle = {
+      ...input.createdLifecycle,
+      capture,
+      revalidate: (registration) =>
+        input.createdLifecycle.revalidate(registration, {
+          allowNotReadyWithMatchingIdentity: true,
+        }),
+    };
+  }
+  return input.createRegistration({ ...input.registration, createdLifecycle });
+}
+
 /** Persist one create-attempt recovery message through the onboard session owner. */
 export function persistRetainedSandboxRecoveryMessage(
   input: {
@@ -2293,6 +2334,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
       }
       return policySourcePath;
     };
+    let managedBootstrapCreateFinished = false;
     const revalidateCreatedSandboxIdentity = (
       expectedIdentity: string,
       operation: string,
@@ -2304,6 +2346,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
           lifecycleLiveIdentityFingerprint: expectedIdentity,
         },
         getSandboxRecreateObservation,
+        { allowNotReadyWithMatchingIdentity: managedBootstrapCreateFinished },
       );
     };
     const requireVerifiedPolicyGate = (): NonNullable<typeof verifiedPolicyGate> => {
@@ -2682,8 +2725,8 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
               await runDeferredProviderEffects(context);
             }
           : undefined,
-        create: (verifyCreatedSandbox) =>
-          sandboxGpuCreateFlow.runSandboxGpuCreateFlow(
+        create: async (verifyCreatedSandbox) => {
+          const created = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow(
             {
               sandboxName,
               ...(resumeVerifiedCreateInput
@@ -2723,6 +2766,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
               terminalAgent: agentDefs.isTerminalAgent(agent),
               managedBootstrap,
               verifyCreatedSandboxBeforeEffects: async (identity) => {
+                managedBootstrapCreateFinished = managedBootstrap !== null;
                 await verifyCreatedSandbox(identity);
               },
               revalidateVerifiedSandboxBeforeEffect: (operation) =>
@@ -2736,7 +2780,9 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
               openshellArgv,
               verifyDirectSandboxGpu: createGpuVerifier,
             },
-          ),
+          );
+          return created;
+        },
       });
     };
 
@@ -2827,13 +2873,27 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
       preparedSandboxWorkload,
       note,
     );
-    const completeCreatedSandboxRegistration = createOnboardCreatedSandboxRegistration({
-      completion: createdSandboxCompletion,
-      createdLifecycle: createdSandboxLifecycle,
-      cleanupBuildContext,
-      manageDashboard,
-      sandboxGpuEnabled: effectiveSandboxGpuConfig.sandboxGpuEnabled,
-    });
+    // Managed bootstrap can invalidate OpenShell's cached Ready state after it
+    // replaces the container. Registry publication stays bound to the durable
+    // sandbox identity recorded before that replacement.
+    const completeCreatedSandboxRegistration =
+      createOnboardCreatedSandboxRegistrationWithManagedLifecycle({
+        sandboxName,
+        managedBootstrap: managedBootstrap !== null,
+        sandboxGpuEnabled: effectiveSandboxGpuConfig.sandboxGpuEnabled,
+        createdLifecycle: createdSandboxLifecycle,
+        getRecordedRegistration: () =>
+          requireDurableCreatedSandboxIdentity(
+            requireVerifiedPolicyGate().lifecycleLiveIdentityFingerprint,
+          ),
+        createRegistration: createOnboardCreatedSandboxRegistration,
+        registration: {
+          completion: createdSandboxCompletion,
+          cleanupBuildContext,
+          manageDashboard,
+          sandboxGpuEnabled: effectiveSandboxGpuConfig.sandboxGpuEnabled,
+        },
+      });
 
     const providerPreparationInput = {
       openshellDriver: sandboxRuntimeFields.openshellDriver,

src/lib/onboard/sandbox-recreate-transaction.ts

@@ -237,6 +237,10 @@ export interface CreatedSandboxLifecycleTarget {
   readonly gatewayName: string;
 }
 
+export interface CreatedSandboxLifecycleRevalidationOptions {
+  readonly allowNotReadyWithMatchingIdentity?: boolean;
+}
+
 type ObserveCreatedSandbox = (
   sandboxName: string,
   gatewayName: string,
@@ -254,22 +258,47 @@ function requireLifecycleGeneration(sandboxName: string, lifecycleGeneration: st
   }
 }
 
+function requireValidLiveIdentity(
+  target: CreatedSandboxLifecycleTarget,
+  observation: SandboxRecreateObservation,
+): string {
+  const fingerprint = observation.liveIdentityFingerprint;
+  if (!fingerprint || !/^[0-9a-f]{64}$/u.test(fingerprint)) {
+    throw new Error(
+      `Cannot register sandbox '${target.sandboxName}': its owning gateway did not report a valid live identity.`,
+    );
+  }
+  return fingerprint;
+}
+
 function requireReadyIdentity(
   target: CreatedSandboxLifecycleTarget,
   observation: SandboxRecreateObservation,
 ): string {
   if (observation.state !== "ready") {
+    // `missing` and `not_ready` need different answers: one says the gateway cannot see the
+    // sandbox at all, the other says it sees it and withholds Ready. Name which one was observed.
     throw new Error(
-      `Cannot register sandbox '${target.sandboxName}': its owning gateway did not report it Ready.`,
+      `Cannot register sandbox '${target.sandboxName}': its owning gateway did not report it Ready (observed ${observation.state}).`,
     );
   }
-  const fingerprint = observation.liveIdentityFingerprint;
-  if (!fingerprint || !/^[0-9a-f]{64}$/u.test(fingerprint)) {
+  return requireValidLiveIdentity(target, observation);
+}
+
+function requireObservedIdentity(
+  target: CreatedSandboxLifecycleTarget,
+  observation: SandboxRecreateObservation,
+  allowNotReadyWithMatchingIdentity: boolean,
+): string {
+  if (
+    observation.state !== "ready" &&
+    !(allowNotReadyWithMatchingIdentity && observation.state === "not_ready")
+  ) {
     throw new Error(
-      `Cannot register sandbox '${target.sandboxName}': its owning gateway did not report a valid live identity.`,
+      `Cannot register sandbox '${target.sandboxName}': its owning gateway did not report it Ready (observed ${observation.state}).`,
     );
   }
-  return fingerprint;
+  return requireValidLiveIdentity(target, observation);
 }
 
 /** Pin the Ready sandbox identity observed from its owning gateway after creation. */
@@ -326,11 +355,13 @@ export function revalidateCreatedSandboxLifecycleRegistration(
   target: CreatedSandboxLifecycleTarget,
   registration: CreatedSandboxLifecycleRegistration,
   observe: ObserveCreatedSandbox,
+  options: CreatedSandboxLifecycleRevalidationOptions = {},
 ): CreatedSandboxLifecycleRegistration {
   requireLifecycleGeneration(target.sandboxName, registration.lifecycleGeneration);
-  const liveIdentityFingerprint = requireReadyIdentity(
+  const liveIdentityFingerprint = requireObservedIdentity(
     target,
     observe(target.sandboxName, target.gatewayName),
+    options.allowNotReadyWithMatchingIdentity === true,
   );
   if (liveIdentityFingerprint !== registration.lifecycleLiveIdentityFingerprint) {
     throw new Error(
@@ -348,6 +379,7 @@ export interface CreatedSandboxLifecycle {
   ): CreatedSandboxLifecycleRegistration;
   revalidate(
     registration: CreatedSandboxLifecycleRegistration,
+    options?: CreatedSandboxLifecycleRevalidationOptions,
   ): CreatedSandboxLifecycleRegistration;
 }
 
@@ -377,8 +409,13 @@ export function createCreatedSandboxLifecycle(
       });
       return captured;
     },
-    revalidate: (registration) => {
-      const verified = revalidateCreatedSandboxLifecycleRegistration(target, registration, observe);
+    revalidate: (registration, options) => {
+      const verified = revalidateCreatedSandboxLifecycleRegistration(
+        target,
+        registration,
+        observe,
+        options,
+      );
       runtime.recordCreated({
         state: "ready",
         liveIdentityFingerprint: verified.lifecycleLiveIdentityFingerprint,