From ac90d694327557da9527513c14b33fcc847e94ba Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:20:37 -0700 Subject: [PATCH 1/4] fix(e2e): verify interrupted sandbox readiness --- src/lib/onboard/sandbox-gpu-create-run-attempt.ts | 8 +++++++- src/lib/onboard/sandbox-readiness-tracing.test.ts | 10 ++++++++++ test/e2e/live/onboard-resume.test.ts | 8 +++++--- test/e2e/mock-parity.json | 2 ++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index f7d8ce601d6..68ae7984769 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -395,7 +395,13 @@ export function createSandboxGpuCreateAttemptRunner( }); if (!readiness.ready) { throw new Error( - `Managed bootstrap incomplete create did not reach authoritative Ready state (${readiness.reason}).`, + sandboxReadinessTracing + .formatCreatedSandboxReadinessFailureMessage( + input.sandboxName, + readiness, + input.sandboxReadyTimeoutSecs, + ) + .trimStart(), ); } } else { diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 03275cf05a8..52b9459fc46 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -521,6 +521,16 @@ describe("DGX Spark fresh-onboard readiness replay (#6043)", () => { ); }); + it("retains the terminal phase in managed-bootstrap readiness diagnostics (#9819)", () => { + expect( + formatCreatedSandboxReadinessFailureMessage( + NAME, + { ready: false, reason: "terminal_failure_phase", failurePhase: "Failed" }, + 1500, + ), + ).toContain("entered Failed phase before it became ready (waited up to 1500s)"); + }); + it("recovers with the shipped default debounce: onboard continues to Ready", () => { const { runCaptureOpenshell, sleep } = replay(reporterSequence); const ready = waitForCreatedSandboxReadyWithTrace({ diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 30627af4da1..427514df62e 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -10,6 +10,7 @@ import { ONBOARD_NO_RECREATE_COMMAND_TIMEOUT_MS, ONBOARD_RESUME_TEST_TIMEOUT_MS, } from "../../../tools/e2e/onboard-timeout-contract.mts"; +import { parseSandboxPhase } from "../../../src/lib/state/gateway.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; @@ -393,9 +394,6 @@ test( // Assertion: interrupted-exit-1. expect(firstRun.exitCode, firstText).toBe(1); - // Assertion: sandbox-created-log. - expect(firstText).toContain(`Sandbox '${SANDBOX_NAME}' created`); - // Assertion: forced-failure-log — failure injection fired at the policies step. expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); @@ -408,6 +406,10 @@ test( timeoutMs: 30_000, }); expect(sandboxAfterInterrupt.exitCode, sandboxAfterInterrupt.stderr).toBe(0); + expect( + parseSandboxPhase(resultText(sandboxAfterInterrupt)), + resultText(sandboxAfterInterrupt), + ).toBe("Ready"); // Exercise the configured route through the sandbox. The OpenShell gateway // must inject the stored compatible-endpoint credential upstream; this POST diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index daf22b89301..26ee607d7d0 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -245,7 +245,9 @@ "src/lib/onboard/machine/handlers/sandbox-recreate-resume.test.ts", "src/lib/onboard/machine/handlers/sandbox-resume.test.ts", "src/lib/onboard/sandbox-create-plan.test.ts", + "src/lib/onboard/sandbox-readiness-tracing.test.ts", "test/onboard-extra-provider-reconciliation.test.ts", + "test/gateway-state.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", "test/e2e/support/onboard-timeout-contract.test.ts" From 0fc604d10afb81340806895031e0ba27826170eb Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:40:09 -0700 Subject: [PATCH 2/4] test(e2e): cover managed readiness failure --- .../onboard/sandbox-gpu-create-flow.test.ts | 101 ++++++++---------- .../sandbox-gpu-create-resolution.test.ts | 73 +++++++++++++ test/e2e/mock-parity.json | 1 + 3 files changed, 120 insertions(+), 55 deletions(-) create mode 100644 src/lib/onboard/sandbox-gpu-create-resolution.test.ts diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index e51f58e3d7d..289d0a1cb58 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -23,7 +23,8 @@ vi.mock("../sandbox/create-stream", () => ({ streamSandboxCreate: mocks.streamSandboxCreate, })); -vi.mock("./sandbox-readiness-tracing", () => ({ +vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({ + ...(await importOriginal()), waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, printReadinessFailure: mocks.printReadinessFailure, })); @@ -52,7 +53,6 @@ vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, })); -import type { AgentDefinition } from "../agent/defs"; import type { CheckpointPortableRuntimeAuthority } from "../state/onboard-checkpoint-types"; import type { SandboxGpuProofResult } from "../state/registry"; import { @@ -81,9 +81,6 @@ import type { import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry"; import { prepareSandboxCreateLaunch } from "./sandbox-create-launch"; import { - resolveExportedPortableRuntimeAuthority, - resolveAgentCreateInput, - resolvePortableLifecycleMode, runSandboxGpuCreateFlow, type SandboxGpuCreateFlowDeps, type SandboxGpuCreateFlowInput, @@ -204,56 +201,6 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); -describe("resolveAgentCreateInput", () => { - it("selects portable lifecycle ownership only for OpenClaw (#9068)", () => { - const env = { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }; - - expect(resolveAgentCreateInput(null, true, env)).toMatchObject({ - persistStartupCommand: false, - portableLifecycle: true, - }); - expect(resolveAgentCreateInput({ name: "hermes" } as AgentDefinition, true, env)).toMatchObject( - { - persistStartupCommand: false, - portableLifecycle: false, - hermesPortableLifecycle: true, - }, - ); - expect(resolvePortableLifecycleMode(null, env)).toBe(true); - expect(resolvePortableLifecycleMode({ name: "hermes" } as AgentDefinition, env)).toBe(false); - }); -}); - -describe("resolveExportedPortableRuntimeAuthority", () => { - it("passes checkpoint-owned authority to exported portable creation (#9070)", () => { - expect( - resolveExportedPortableRuntimeAuthority( - { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, - () => ({ - checkpoint: { - profile: { kind: "selected", value: "portable" }, - runtimeAuthority: { kind: "selected", value: PORTABLE_RUNTIME_AUTHORITY }, - }, - }), - ), - ).toEqual(PORTABLE_RUNTIME_AUTHORITY); - }); - - it("rejects exported portable creation before effects when authority is absent (#9070)", () => { - expect(() => - resolveExportedPortableRuntimeAuthority( - { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, - () => ({ - checkpoint: { - profile: { kind: "selected", value: "portable" }, - runtimeAuthority: { kind: "unset" }, - }, - }), - ), - ).toThrow("requires checkpoint-owned Podman runtime authority before creation begins"); - }); -}); - describe("runSandboxGpuCreateFlow provider-owned managed create", () => { it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => { const input = createInput(); @@ -483,6 +430,50 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { expect(errorOutput()).toContain("Authorization: Bearer "); expect(errorOutput()).not.toContain(recoverySecret); }); + + it("reports the terminal phase when an incomplete managed create cannot become ready (#9819)", async () => { + const input = createInput(); + const bootstrapIdentity = "e".repeat(64); + input.managedBootstrap = { + bootstrapIdentity, + stateRoot: "/tmp/nemoclaw-managed-bootstrap", + runtimeProvider: { + identity: { id: "mxc" }, + bootstrap: { + createOnboardRouting: () => ({ nativeFallbackHasCleanBaseline: false }), + createLifecycle: (options: ManagedBootstrapRuntimeCreateLifecycleInput) => ({ + launchArgv: options.launchArgv, + patch: createPatch(), + recoverUnfinished: async () => null, + prepareNetwork: async () => undefined, + runCreate: async (start) => + ( + await start({ + heldWorkloadArgv: options.heldWorkloadArgv, + bootstrapIdentity: options.bootstrapIdentity, + }) + ).value, + }), + }, + }, + } as unknown as NonNullable; + const deps = createDeps(); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 23, + output: "Created sandbox: alpha", + sawProgress: true, + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValueOnce({ + ready: false, + reason: "terminal_failure_phase", + failurePhase: "Failed", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "Sandbox 'alpha' entered Failed phase before it became ready (waited up to 60s).", + ); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledOnce(); + }); }); describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-resolution.test.ts b/src/lib/onboard/sandbox-gpu-create-resolution.test.ts new file mode 100644 index 00000000000..11b080fb2c6 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-resolution.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { AgentDefinition } from "../agent/defs"; +import type { CheckpointPortableRuntimeAuthority } from "../state/onboard-checkpoint-types"; +import { + resolveAgentCreateInput, + resolveExportedPortableRuntimeAuthority, + resolvePortableLifecycleMode, +} from "./sandbox-gpu-create-flow"; + +const PORTABLE_RUNTIME_AUTHORITY: CheckpointPortableRuntimeAuthority = { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid: 1001, + homeDir: "/home/tester", + configHome: "/home/tester/.config", + runtimeDir: "/run/user/1001", + socketPath: "/run/user/1001/podman/podman.sock", +}; + +describe("resolveAgentCreateInput", () => { + it("selects portable lifecycle ownership only for OpenClaw (#9068)", () => { + const env = { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }; + + expect(resolveAgentCreateInput(null, true, env)).toMatchObject({ + persistStartupCommand: false, + portableLifecycle: true, + }); + expect(resolveAgentCreateInput({ name: "hermes" } as AgentDefinition, true, env)).toMatchObject( + { + persistStartupCommand: false, + portableLifecycle: false, + hermesPortableLifecycle: true, + }, + ); + expect(resolvePortableLifecycleMode(null, env)).toBe(true); + expect(resolvePortableLifecycleMode({ name: "hermes" } as AgentDefinition, env)).toBe(false); + }); +}); + +describe("resolveExportedPortableRuntimeAuthority", () => { + it("passes checkpoint-owned authority to exported portable creation (#9070)", () => { + expect( + resolveExportedPortableRuntimeAuthority( + { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + () => ({ + checkpoint: { + profile: { kind: "selected", value: "portable" }, + runtimeAuthority: { kind: "selected", value: PORTABLE_RUNTIME_AUTHORITY }, + }, + }), + ), + ).toEqual(PORTABLE_RUNTIME_AUTHORITY); + }); + + it("rejects exported portable creation before effects when authority is absent (#9070)", () => { + expect(() => + resolveExportedPortableRuntimeAuthority( + { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + () => ({ + checkpoint: { + profile: { kind: "selected", value: "portable" }, + runtimeAuthority: { kind: "unset" }, + }, + }), + ), + ).toThrow("requires checkpoint-owned Podman runtime authority before creation begins"); + }); +}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 26ee607d7d0..7751e59c525 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -245,6 +245,7 @@ "src/lib/onboard/machine/handlers/sandbox-recreate-resume.test.ts", "src/lib/onboard/machine/handlers/sandbox-resume.test.ts", "src/lib/onboard/sandbox-create-plan.test.ts", + "src/lib/onboard/sandbox-gpu-create-flow.test.ts", "src/lib/onboard/sandbox-readiness-tracing.test.ts", "test/onboard-extra-provider-reconciliation.test.ts", "test/gateway-state.test.ts", From 1ad290b09c42f6b8125b15b46af3fdd87b10bd30 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:48:48 -0700 Subject: [PATCH 3/4] test(e2e): type managed create callback --- src/lib/onboard/sandbox-gpu-create-flow.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 289d0a1cb58..0730e6b44bd 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -446,7 +446,12 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { patch: createPatch(), recoverUnfinished: async () => null, prepareNetwork: async () => undefined, - runCreate: async (start) => + runCreate: async ( + start: (held: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise<{ readonly value: T }>, + ): Promise => ( await start({ heldWorkloadArgv: options.heldWorkloadArgv, From 90edfead7f587c7e205bacd35ef8e6701f7ee099 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:03:02 -0700 Subject: [PATCH 4/4] test(e2e): bound resume readiness diagnostic --- test/e2e/live/onboard-resume.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 427514df62e..b7019c87129 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -407,9 +407,9 @@ test( }); expect(sandboxAfterInterrupt.exitCode, sandboxAfterInterrupt.stderr).toBe(0); expect( - parseSandboxPhase(resultText(sandboxAfterInterrupt)), - resultText(sandboxAfterInterrupt), - ).toBe("Ready"); + parseSandboxPhase(resultText(sandboxAfterInterrupt)) === "Ready", + "OpenShell sandbox did not report Ready after the intended onboarding interruption. Inspect the phase-2-openshell-sandbox-get artifact.", + ).toBe(true); // Exercise the configured route through the sandbox. The OpenShell gateway // must inject the stored compatible-endpoint credential upstream; this POST