Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 51 additions & 55 deletions src/lib/onboard/sandbox-gpu-create-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ vi.mock("../sandbox/create-stream", () => ({
streamSandboxCreate: mocks.streamSandboxCreate,
}));

vi.mock("./sandbox-readiness-tracing", () => ({
vi.mock("./sandbox-readiness-tracing", async (importOriginal) => ({
...(await importOriginal<typeof import("./sandbox-readiness-tracing")>()),
waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace,
printReadinessFailure: mocks.printReadinessFailure,
}));
Expand Down Expand Up @@ -52,7 +53,6 @@ vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({
queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot,
}));

import type { AgentDefinition } from "../agent/defs";
import type { CheckpointPortableRuntimeAuthority } from "../state/onboard-checkpoint-types";
import type { SandboxGpuProofResult } from "../state/registry";
import {
Expand Down Expand Up @@ -81,9 +81,6 @@ import type {
import { createRuntimeProviderBundleRegistry } from "./runtime-provider/registry";
import { prepareSandboxCreateLaunch } from "./sandbox-create-launch";
import {
resolveExportedPortableRuntimeAuthority,
resolveAgentCreateInput,
resolvePortableLifecycleMode,
runSandboxGpuCreateFlow,
type SandboxGpuCreateFlowDeps,
type SandboxGpuCreateFlowInput,
Expand Down Expand Up @@ -204,56 +201,6 @@ function createSourceInput(): SandboxGpuCreateFlowInput {
beforeEach(() => setupGpuFlowMocks(mocks));
afterEach(resetGpuFlowMocks);

describe("resolveAgentCreateInput", () => {
it("selects portable lifecycle ownership only for OpenClaw (#9068)", () => {
const env = { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" };

expect(resolveAgentCreateInput(null, true, env)).toMatchObject({
persistStartupCommand: false,
portableLifecycle: true,
});
expect(resolveAgentCreateInput({ name: "hermes" } as AgentDefinition, true, env)).toMatchObject(
{
persistStartupCommand: false,
portableLifecycle: false,
hermesPortableLifecycle: true,
},
);
expect(resolvePortableLifecycleMode(null, env)).toBe(true);
expect(resolvePortableLifecycleMode({ name: "hermes" } as AgentDefinition, env)).toBe(false);
});
});

describe("resolveExportedPortableRuntimeAuthority", () => {
it("passes checkpoint-owned authority to exported portable creation (#9070)", () => {
expect(
resolveExportedPortableRuntimeAuthority(
{ NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
() => ({
checkpoint: {
profile: { kind: "selected", value: "portable" },
runtimeAuthority: { kind: "selected", value: PORTABLE_RUNTIME_AUTHORITY },
},
}),
),
).toEqual(PORTABLE_RUNTIME_AUTHORITY);
});

it("rejects exported portable creation before effects when authority is absent (#9070)", () => {
expect(() =>
resolveExportedPortableRuntimeAuthority(
{ NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
() => ({
checkpoint: {
profile: { kind: "selected", value: "portable" },
runtimeAuthority: { kind: "unset" },
},
}),
),
).toThrow("requires checkpoint-owned Podman runtime authority before creation begins");
});
});

describe("runSandboxGpuCreateFlow provider-owned managed create", () => {
it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => {
const input = createInput();
Expand Down Expand Up @@ -483,6 +430,55 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => {
expect(errorOutput()).toContain("Authorization: Bearer <REDACTED>");
expect(errorOutput()).not.toContain(recoverySecret);
});

it("reports the terminal phase when an incomplete managed create cannot become ready (#9819)", async () => {
const input = createInput();
const bootstrapIdentity = "e".repeat(64);
input.managedBootstrap = {
bootstrapIdentity,
stateRoot: "/tmp/nemoclaw-managed-bootstrap",
runtimeProvider: {
identity: { id: "mxc" },
bootstrap: {
createOnboardRouting: () => ({ nativeFallbackHasCleanBaseline: false }),
createLifecycle: (options: ManagedBootstrapRuntimeCreateLifecycleInput) => ({
launchArgv: options.launchArgv,
patch: createPatch(),
recoverUnfinished: async () => null,
prepareNetwork: async () => undefined,
runCreate: async <T>(
start: (held: {
readonly heldWorkloadArgv: readonly string[];
readonly bootstrapIdentity: string;
}) => Promise<{ readonly value: T }>,
): Promise<T> =>
(
await start({
heldWorkloadArgv: options.heldWorkloadArgv,
bootstrapIdentity: options.bootstrapIdentity,
})
).value,
}),
},
},
} as unknown as NonNullable<SandboxGpuCreateFlowInput["managedBootstrap"]>;
const deps = createDeps();
mocks.streamSandboxCreate.mockResolvedValueOnce({
status: 23,
output: "Created sandbox: alpha",
sawProgress: true,
});
mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValueOnce({
ready: false,
reason: "terminal_failure_phase",
failurePhase: "Failed",
});

await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow(
"Sandbox 'alpha' entered Failed phase before it became ready (waited up to 60s).",
);
expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledOnce();
});
});
describe("runSandboxGpuCreateFlow proof authorization", () => {
it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => {
Expand Down
73 changes: 73 additions & 0 deletions src/lib/onboard/sandbox-gpu-create-resolution.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import type { AgentDefinition } from "../agent/defs";
import type { CheckpointPortableRuntimeAuthority } from "../state/onboard-checkpoint-types";
import {
resolveAgentCreateInput,
resolveExportedPortableRuntimeAuthority,
resolvePortableLifecycleMode,
} from "./sandbox-gpu-create-flow";

const PORTABLE_RUNTIME_AUTHORITY: CheckpointPortableRuntimeAuthority = {
schemaVersion: 1,
kind: "podman",
ownership: "current-user",
uid: 1001,
homeDir: "/home/tester",
configHome: "/home/tester/.config",
runtimeDir: "/run/user/1001",
socketPath: "/run/user/1001/podman/podman.sock",
};

describe("resolveAgentCreateInput", () => {
it("selects portable lifecycle ownership only for OpenClaw (#9068)", () => {
const env = { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" };

expect(resolveAgentCreateInput(null, true, env)).toMatchObject({
persistStartupCommand: false,
portableLifecycle: true,
});
expect(resolveAgentCreateInput({ name: "hermes" } as AgentDefinition, true, env)).toMatchObject(
{
persistStartupCommand: false,
portableLifecycle: false,
hermesPortableLifecycle: true,
},
);
expect(resolvePortableLifecycleMode(null, env)).toBe(true);
expect(resolvePortableLifecycleMode({ name: "hermes" } as AgentDefinition, env)).toBe(false);
});
});

describe("resolveExportedPortableRuntimeAuthority", () => {
it("passes checkpoint-owned authority to exported portable creation (#9070)", () => {
expect(
resolveExportedPortableRuntimeAuthority(
{ NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
() => ({
checkpoint: {
profile: { kind: "selected", value: "portable" },
runtimeAuthority: { kind: "selected", value: PORTABLE_RUNTIME_AUTHORITY },
},
}),
),
).toEqual(PORTABLE_RUNTIME_AUTHORITY);
});

it("rejects exported portable creation before effects when authority is absent (#9070)", () => {
expect(() =>
resolveExportedPortableRuntimeAuthority(
{ NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
() => ({
checkpoint: {
profile: { kind: "selected", value: "portable" },
runtimeAuthority: { kind: "unset" },
},
}),
),
).toThrow("requires checkpoint-owned Podman runtime authority before creation begins");
});
});
8 changes: 7 additions & 1 deletion src/lib/onboard/sandbox-gpu-create-run-attempt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,13 @@ export function createSandboxGpuCreateAttemptRunner(
});
if (!readiness.ready) {
throw new Error(
`Managed bootstrap incomplete create did not reach authoritative Ready state (${readiness.reason}).`,
sandboxReadinessTracing
.formatCreatedSandboxReadinessFailureMessage(
input.sandboxName,
readiness,
input.sandboxReadyTimeoutSecs,
)
.trimStart(),
);
}
} else {
Expand Down
10 changes: 10 additions & 0 deletions src/lib/onboard/sandbox-readiness-tracing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,16 @@ describe("DGX Spark fresh-onboard readiness replay (#6043)", () => {
);
});

it("retains the terminal phase in managed-bootstrap readiness diagnostics (#9819)", () => {
expect(
formatCreatedSandboxReadinessFailureMessage(
NAME,
{ ready: false, reason: "terminal_failure_phase", failurePhase: "Failed" },
1500,
),
).toContain("entered Failed phase before it became ready (waited up to 1500s)");
});

it("recovers with the shipped default debounce: onboard continues to Ready", () => {
const { runCaptureOpenshell, sleep } = replay(reporterSequence);
const ready = waitForCreatedSandboxReadyWithTrace({
Expand Down
8 changes: 5 additions & 3 deletions test/e2e/live/onboard-resume.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
ONBOARD_NO_RECREATE_COMMAND_TIMEOUT_MS,
ONBOARD_RESUME_TEST_TIMEOUT_MS,
} from "../../../tools/e2e/onboard-timeout-contract.mts";
import { parseSandboxPhase } from "../../../src/lib/state/gateway.ts";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts";
import { resultText } from "../fixtures/clients/command.ts";
Expand Down Expand Up @@ -393,9 +394,6 @@ test(
// Assertion: interrupted-exit-1.
expect(firstRun.exitCode, firstText).toBe(1);

// Assertion: sandbox-created-log.
expect(firstText).toContain(`Sandbox '${SANDBOX_NAME}' created`);

// Assertion: forced-failure-log — failure injection fired at the policies step.
expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'.");

Expand All @@ -408,6 +406,10 @@ test(
timeoutMs: 30_000,
});
expect(sandboxAfterInterrupt.exitCode, sandboxAfterInterrupt.stderr).toBe(0);
expect(
parseSandboxPhase(resultText(sandboxAfterInterrupt)) === "Ready",
"OpenShell sandbox did not report Ready after the intended onboarding interruption. Inspect the phase-2-openshell-sandbox-get artifact.",
).toBe(true);

// Exercise the configured route through the sandbox. The OpenShell gateway
// must inject the stored compatible-endpoint credential upstream; this POST
Expand Down
3 changes: 3 additions & 0 deletions test/e2e/mock-parity.json
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,10 @@
"src/lib/onboard/machine/handlers/sandbox-recreate-resume.test.ts",
"src/lib/onboard/machine/handlers/sandbox-resume.test.ts",
"src/lib/onboard/sandbox-create-plan.test.ts",
"src/lib/onboard/sandbox-gpu-create-flow.test.ts",
"src/lib/onboard/sandbox-readiness-tracing.test.ts",
"test/onboard-extra-provider-reconciliation.test.ts",
"test/gateway-state.test.ts",
Comment thread
rsliter marked this conversation as resolved.
"test/e2e/support/e2e-cleanup-resources.test.ts",
"test/e2e/support/e2e-clients.test.ts",
"test/e2e/support/onboard-timeout-contract.test.ts"
Expand Down
Loading