From d0677ede839837b7c728d52b19249c91c7bfca85 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:09:32 -0400 Subject: [PATCH 01/18] test(e2e): allow whitelisted lifecycle profiles in live runner `liveScenarioSupport` previously rejected any scenario that declared an `environment.lifecycle`, so post-onboard host mutations (reboot, rebuild, upgrade, drift) could not surface in the live Vitest matrix at all. Replace the unconditional reject with a `SUPPORTED_LIFECYCLES` whitelist that starts with the single profile the upcoming post-reboot-recovery fixture dispatches: `post-reboot-recovery`. Future profiles must land the dispatcher branch and an expected-state in the same change set, so the whitelist stays in lockstep with what the runner can actually execute. Prepares the runner for #4423's failing-test-first guard, which needs a post-reboot lifecycle scenario to demonstrate registry preservation + Docker-backed sandbox recovery on Linux/Spark Docker-driver hosts. Refs #4423 --- .../e2e-live-registry-discovery.test.ts | 35 +++++++++++++++++++ .../e2e-scenario/scenarios/runtime-support.ts | 8 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 21cb9b6329e..db1dea7f4d1 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -50,4 +50,39 @@ describe("live Vitest registry discovery support", () => { reasons: ["runtime 'docker-missing' is not wired for live Vitest fixtures"], }); }); + + it("keeps unwhitelisted lifecycle profiles skipped with the lifecycle reason", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-rebuild-openclaw"); + + expect(scenario).toBeTruthy(); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: false, + reasons: [ + "lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures", + ], + }); + }); + + it("accepts whitelisted lifecycle profiles when the rest of the environment matches", () => { + // Synthesised scenario stands in for the not-yet-registered + // post-reboot-recovery scenario so this test pins the whitelist + // contract independently of when the scenario lands. Once the + // post-reboot-recovery scenario is registered, prefer asserting on + // the registry entry directly and remove this synthetic. + const supported = liveScenarioSupport({ + id: "synthetic-post-reboot-recovery", + assertionGroups: [], + expectedStateId: "cloud-openclaw-ready", + environment: { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + lifecycle: "post-reboot-recovery", + }, + }); + + expect(supported.supported).toBe(true); + expect(supported.reasons).toEqual([]); + }); }); diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 24b1e269242..6a9f874264d 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -7,6 +7,12 @@ const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); +// Lifecycle profiles wired into the live Vitest driver. A profile is +// supported only after both (a) `LifecyclePhaseFixture.simulate(profile)` +// dispatches it, and (b) at least one expected-state declares the post- +// lifecycle host invariants the fixture creates. New profiles must add +// the dispatcher branch and an expected-state in the same change set. +const SUPPORTED_LIFECYCLES = new Set(["post-reboot-recovery"]); export interface LiveScenarioSupport { supported: boolean; @@ -43,7 +49,7 @@ export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioS if (!SUPPORTED_ONBOARDING.has(environment.onboarding)) { reasons.push(`onboarding '${environment.onboarding}' is not wired for live Vitest fixtures`); } - if (environment.lifecycle) { + if (environment.lifecycle && !SUPPORTED_LIFECYCLES.has(environment.lifecycle)) { reasons.push(`lifecycle '${environment.lifecycle}' is not wired for live Vitest fixtures`); } } From a220f87fe933ca5c9d20aa58303aedf6e68f52e3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:16:23 -0400 Subject: [PATCH 02/18] test(e2e): add host-side registry and Docker container probes Adds two host-side state-validation probes the live runner needs to express the regression target tracked by #4423: * `local-registry-entry-present` reads `~/.nemoclaw/sandboxes.json` and asserts the scenario's sandbox name is still recorded. This is deliberately orthogonal to `sandbox.expected`: post-reboot bugs can wipe the local registry while the live OpenShell gateway is healthy, and only a host-side probe catches the data-loss regression. * `docker-sandbox-container-present` runs `docker ps -a --filter label=openshell.ai/sandbox-name=` and accepts running, stopped, or `*-nemoclaw-gpu-backup-*` sibling containers. The label filter mirrors `OPENSHELL_SANDBOX_NAME_LABEL` used by `findOpenShellDockerSandboxContainerIds` in `src/lib/onboard/docker-gpu-patch.ts`, so the probe stays in lock- step with how OpenShell labels containers today. Probe wiring: * `StateProbeId` extended with the two new probe ids. * `ExpectedState` gains `localRegistry` and `dockerSandboxContainer` optional dimensions; `probesForState` emits the new probes only for `expected: "present"`. Negative-direction probes are intentionally omitted today and pinned by a probesForState test. * `StateValidationPhaseFixture.from()` now accepts either an expected-state ID or an inline `ExpectedState`, so unit tests can drive new probes without registering synthetic states in the typed registry. The live runner still calls `from(id, instance)`. * Fixture takes an optional `ProbeIO` injection so tests can stub the registry reader without touching `~/.nemoclaw`. No callers of the existing typed registry are affected: every shipped expected-state leaves `localRegistry` and `dockerSandboxContainer` unset, so `probesForState` returns the same probe lists as before. Refs #4423 --- .../e2e-expected-state.test.ts | 35 +++++ .../e2e-phase-state-validation.test.ts | 120 +++++++++++++++++- .../framework/phases/state-validation.ts | 113 ++++++++++++++++- .../e2e-scenario/scenarios/expected-states.ts | 10 ++ test/e2e-scenario/scenarios/types.ts | 25 +++- 5 files changed, 297 insertions(+), 6 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index 2fe76802134..dcf15974923 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -83,6 +83,41 @@ describe("probesForState maps typed expected-state into probe ids", () => { }; expect(probesForState(state)).toEqual([]); }); + + it("localRegistry.expected=present emits the local-registry-entry-present probe", () => { + const state: ExpectedState = { + id: "synthetic-local-registry", + cli: { installed: true }, + localRegistry: { expected: "present" }, + }; + expect(probesForState(state)).toEqual([ + "cli-installed", + "local-registry-entry-present", + ]); + }); + + it("dockerSandboxContainer.expected=present emits the docker-sandbox-container-present probe", () => { + const state: ExpectedState = { + id: "synthetic-docker-container", + cli: { installed: true }, + dockerSandboxContainer: { expected: "present" }, + }; + expect(probesForState(state)).toEqual([ + "cli-installed", + "docker-sandbox-container-present", + ]); + }); + + it("localRegistry/dockerSandboxContainer 'absent' emits no probe today", () => { + // Negative-direction probes haven't landed yet. Pin the gap so a + // future negative-scenario PR is forced to add the absent probes. + const state: ExpectedState = { + id: "synthetic-host-absent", + localRegistry: { expected: "absent" }, + dockerSandboxContainer: { expected: "absent" }, + }; + expect(probesForState(state)).toEqual([]); + }); }); describe("compiler emits state-validation phase actions from expected-state registry", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index f2af06b99f5..50cf1306d59 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -108,9 +108,17 @@ function instance(overrides: Partial = {}): NemoClawInstance { }; } -function fixture(runner: FakeRunner): StateValidationPhaseFixture { +function fixture( + runner: FakeRunner, + io: ConstructorParameters[3] = {}, +): StateValidationPhaseFixture { const host = new HostCliClient(runner); - return new StateValidationPhaseFixture(host, new GatewayClient(host), new SandboxClient(runner)); + return new StateValidationPhaseFixture( + host, + new GatewayClient(host), + new SandboxClient(runner), + io, + ); } describe("state-validation phase fixture", () => { @@ -457,3 +465,111 @@ describe("state-validation phase fixture", () => { >().toEqualTypeOf(); }); }); + +describe("state-validation host-side probes", () => { + const localRegistryState = { + id: "synthetic-local-registry-present", + localRegistry: { expected: "present" as const }, + }; + const dockerContainerState = { + id: "synthetic-docker-container-present", + dockerSandboxContainer: { expected: "present" as const }, + }; + + it("local-registry-entry-present passes when the registry contains the sandbox name", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ + entries: { "e2e-ubuntu-repo-cloud-openclaw": { name: "e2e-ubuntu-repo-cloud-openclaw" } }, + }), + }); + + const result = await fx.from(localRegistryState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["local-registry-entry-present"]); + expect(runner.calls).toEqual([]); + }); + + it("local-registry-entry-present fails when the registry file is missing", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { readRegistry: () => null }); + + await expect(fx.from(localRegistryState, instance())).rejects.toThrow( + /expected local registry entry for 'e2e-ubuntu-repo-cloud-openclaw'.*does not exist/, + ); + }); + + it("local-registry-entry-present fails when the sandbox name is missing from registry", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ entries: { "some-other-sandbox": {} } }), + }); + + await expect(fx.from(localRegistryState, instance())).rejects.toThrow( + /registry contains: some-other-sandbox/, + ); + }); + + it("docker-sandbox-container-present passes when docker ps -a returns labeled names", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "e2e-ubuntu-repo-cloud-openclaw\n")); + const fx = fixture(runner); + + const result = await fx.from(dockerContainerState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["docker-sandbox-container-present"]); + expect(runner.calls).toEqual([ + { + command: "docker", + args: [ + "ps", + "-a", + "--filter", + "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw", + "--format", + "{{.Names}}", + ], + options: { + artifactName: "docker-sandbox-container-present-e2e-ubuntu-repo-cloud-openclaw", + env: expect.objectContaining({ PATH: expect.any(String) }), + timeoutMs: 15_000, + }, + }, + ]); + }); + + it("docker-sandbox-container-present matches *-nemoclaw-gpu-backup-* sibling containers", async () => { + const runner = new FakeRunner(); + runner.enqueue( + shellResult( + 0, + "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n", + ), + ); + const fx = fixture(runner); + + const result = await fx.from(dockerContainerState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["docker-sandbox-container-present"]); + }); + + it("docker-sandbox-container-present fails when docker ps -a returns no labeled container", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "\n")); + const fx = fixture(runner); + + await expect(fx.from(dockerContainerState, instance())).rejects.toThrow( + /docker ps -a returned none/, + ); + }); + + it("docker-sandbox-container-present fails when docker ps -a exits non-zero", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const fx = fixture(runner); + + await expect(fx.from(dockerContainerState, instance())).rejects.toThrow( + /could not query Docker for label.*exit 1/, + ); + }); +}); diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index 4743be9fbe2..e919adc17de 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { trustedProviderEndpoint, @@ -13,6 +17,36 @@ import { probesForState, requireExpectedState } from "../../scenarios/expected-s import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; import type { NemoClawInstance } from "./onboarding.ts"; +// Mirror of `src/lib/state/registry.ts::REGISTRY_FILE`. The fixture +// owns its own copy because the framework code must not import from +// `src/lib/**` (CLI source) — that boundary keeps the live runner +// honest about probing only host-observable state. +const NEMOCLAW_REGISTRY_RELPATH = [".nemoclaw", "sandboxes.json"] as const; +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + +export interface ProbeIO { + readRegistry?(): { entries: Record } | null; +} + +function defaultRegistryPath(): string { + const home = process.env.HOME ?? os.homedir(); + return path.join(home, ...NEMOCLAW_REGISTRY_RELPATH); +} + +function defaultReadRegistry(): { entries: Record } | null { + const file = defaultRegistryPath(); + if (!fs.existsSync(file)) return null; + try { + const raw = fs.readFileSync(file, "utf8"); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object") return { entries: {} }; + const entries = (parsed as { sandboxes?: Record }).sandboxes; + return { entries: entries && typeof entries === "object" ? entries : {} }; + } catch { + return { entries: {} }; + } +} + export interface StateValidationProbeResult { id: StateProbeId; status: "passed"; @@ -74,14 +108,23 @@ function isMissingOpenShellError(error: unknown): boolean { } export class StateValidationPhaseFixture { + private readonly io: ProbeIO; + constructor( private readonly host: HostCliClient, private readonly gateway: GatewayClient, private readonly sandbox: SandboxClient, - ) {} + io: ProbeIO = {}, + ) { + this.io = io; + } - async from(expectedStateId: string, instance?: NemoClawInstance): Promise { - const state = requireExpectedState(expectedStateId); + async from( + expectedState: string | ExpectedState, + instance?: NemoClawInstance, + ): Promise { + const state = + typeof expectedState === "string" ? requireExpectedState(expectedState) : expectedState; const probes: StateValidationProbeResult[] = []; for (const probe of probesForState(state)) { probes.push(await this.runProbe(probe, instance)); @@ -104,6 +147,10 @@ export class StateValidationPhaseFixture { return await this.expectSandboxRunning(requireInstance(probe, instance)); case "sandbox-absent": return await this.expectSandboxAbsent(requireInstance(probe, instance)); + case "local-registry-entry-present": + return this.expectLocalRegistryEntryPresent(requireInstance(probe, instance)); + case "docker-sandbox-container-present": + return await this.expectDockerSandboxContainerPresent(requireInstance(probe, instance)); default: { const _exhaustive: never = probe; throw new Error(`Unsupported state-validation probe '${_exhaustive}'.`); @@ -231,6 +278,66 @@ export class StateValidationPhaseFixture { return { id: "sandbox-running", status: "passed", results: [result] }; } + private expectLocalRegistryEntryPresent( + instance: NemoClawInstance, + ): StateValidationProbeResult { + const reader = this.io.readRegistry ?? defaultReadRegistry; + const registry = reader(); + if (!registry) { + throw new Error( + `state-validation expected local registry entry for '${instance.sandboxName}', ` + + `but ${defaultRegistryPath()} does not exist.`, + ); + } + if (!Object.prototype.hasOwnProperty.call(registry.entries, instance.sandboxName)) { + const present = Object.keys(registry.entries).sort().join(", ") || "(none)"; + throw new Error( + `state-validation expected local registry entry for '${instance.sandboxName}', ` + + `but the registry contains: ${present}.`, + ); + } + return { id: "local-registry-entry-present", status: "passed", results: [] }; + } + + private async expectDockerSandboxContainerPresent( + instance: NemoClawInstance, + ): Promise { + const result = await this.host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: `docker-sandbox-container-present-${instance.sandboxName}`, + env: statusProbeEnv(), + timeoutMs: 15_000, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `state-validation could not query Docker for label '${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' ` + + `(exit ${result.exitCode}).`, + ); + } + const names = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (names.length === 0) { + throw new Error( + `state-validation expected at least one Docker container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (running, stopped, or ` + + `*-nemoclaw-gpu-backup-* sibling), but docker ps -a returned none.`, + ); + } + return { id: "docker-sandbox-container-present", status: "passed", results: [result] }; + } + private async expectSandboxAbsent( instance: NemoClawInstance, ): Promise { diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index 2a740a38cfd..d79b32e185d 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -131,5 +131,15 @@ export function probesForState(state: ExpectedState): readonly StateProbeId[] { } else if (state.sandbox?.expected === "absent") { probes.push("sandbox-absent"); } + // Host-side aspects. "absent" deliberately emits no probe today: it + // would require asserting the registry/container does NOT exist, + // which has no scenario in flight. Add when a negative scenario + // needs it. + if (state.localRegistry?.expected === "present") { + probes.push("local-registry-entry-present"); + } + if (state.dockerSandboxContainer?.expected === "present") { + probes.push("docker-sandbox-container-present"); + } return probes; } diff --git a/test/e2e-scenario/scenarios/types.ts b/test/e2e-scenario/scenarios/types.ts index 693acff04e8..d363e4544a2 100644 --- a/test/e2e-scenario/scenarios/types.ts +++ b/test/e2e-scenario/scenarios/types.ts @@ -16,12 +16,23 @@ export type PhaseResultName = PhaseName | NegativeContractPhase; // nemoclaw_scenarios/probes/. Inference and credentials probes are // declared but not yet implemented; the compiler skips emitting actions // for them until the probe scripts land. +// +// `local-registry-entry-present` and `docker-sandbox-container-present` +// are host-side aspects of the sandbox: the local NemoClaw registry +// (`~/.nemoclaw/sandboxes.json`) and the Docker container labeled with +// `openshell.ai/sandbox-name=` (running OR stopped, including +// `*-nemoclaw-gpu-backup-*` siblings). These probes let scenarios +// assert preservation invariants that diverge from the live gateway +// view of the sandbox, which is precisely the regression class +// covered by the post-reboot recovery work tracked in #4423. export type StateProbeId = | "cli-installed" | "gateway-healthy" | "gateway-absent" | "sandbox-running" - | "sandbox-absent"; + | "sandbox-absent" + | "local-registry-entry-present" + | "docker-sandbox-container-present"; // User-facing phase the negative-scenario contract advertises. Wider // than PhaseName because manifests may declare "preflight" failures, @@ -74,6 +85,18 @@ export interface ExpectedState { credentials?: { expected: ExpectedPresence; }; + // Host-side registry entry for the scenario's sandbox name. + // "present" means `~/.nemoclaw/sandboxes.json` retains the entry, + // even if the live gateway can no longer see the sandbox. This is + // orthogonal to `sandbox.expected`: registry preservation is the + // user-visible regression target for #4423. + localRegistry?: { expected: ExpectedPresence }; + // Host-side Docker container labeled `openshell.ai/sandbox-name=`. + // "present" matches running OR stopped containers, including + // `*-nemoclaw-gpu-backup-*` siblings produced by the GPU patch path. + // Used to assert that recovery information remains available even + // when the live OpenShell gateway returns NotFound. + dockerSandboxContainer?: { expected: ExpectedPresence }; } export type TransientClassifier = From 051611262b96a6b43a9a5f602a983d13e10993df Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:21:20 -0400 Subject: [PATCH 03/18] test(e2e): add LifecyclePhaseFixture for post-reboot recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Vitest phase fixture that mutates host state between onboarding and state-validation, so live scenarios can express post-onboard invariants the legacy bash runner has no equivalent for. `LifecyclePhaseFixture.simulate("post-reboot-recovery", instance, opts)` reproduces the host-side conditions of a DGX Spark / Linux Docker-driver reboot in two modes: * `stop-original` (default) — `openshell gateway stop` + `docker stop` of the labeled sandbox container. Models the common reboot outcome where OpenShell forgets the sandbox while Docker keeps the container exited but labeled. * `rename-to-gpu-backup` — additionally `docker rename`s the container to a `*-nemoclaw-gpu- backup-` sibling, mirroring the GPU-patch reboot path in `src/lib/onboard/docker-gpu-patch.ts`. Both modes register cleanups (in reverse order) to restore the container so test teardown leaves Docker in a usable state. Wiring: * `framework/phases/index.ts` re-exports the fixture and types. * `framework/e2e-test.ts` registers a `lifecycle` Vitest fixture on `E2EScenarioFixtures`, wired with the shared `host`, `sandbox`, and `cleanup` registries. * `live/registry-scenarios.test.ts` invokes `lifecycle.simulate(profile, instance)` between `onboard.from(...)` and `stateValidation.from(...)` whenever the scenario declares a whitelisted `environment.lifecycle`. Scenarios that omit lifecycle are unaffected. A scenario whose lifecycle is whitelisted by `runtime-support.ts` but NOT dispatched by the fixture fails fast with a clear error so the whitelist and dispatcher stay in lock- step. Coverage in `e2e-phase-lifecycle.test.ts` exercises both modes, gateway-stop tolerance, the no-labeled-container failure case, the docker-discover failure case, the unsupported-profile rejection, the cleanup queue order, and `buildBackupContainerName` truncation. The fixture is intentionally narrow on profiles: only `post-reboot-recovery` is dispatched today. Adding rebuild, upgrade, or drift profiles is a separate, equally narrow change set that must land the dispatcher branch and `SUPPORTED_LIFECYCLES` whitelist together. Refs #4423 --- .../e2e-phase-lifecycle.test.ts | 239 ++++++++++++++++++ test/e2e-scenario/framework/e2e-test.ts | 5 + test/e2e-scenario/framework/phases/index.ts | 8 + .../framework/phases/lifecycle.ts | 207 +++++++++++++++ .../live/registry-scenarios.test.ts | 33 ++- 5 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts create mode 100644 test/e2e-scenario/framework/phases/lifecycle.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts new file mode 100644 index 00000000000..b4cae772551 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { + buildBackupContainerName, + LifecyclePhaseFixture, + type LifecycleCleanup, +} from "../framework/phases/lifecycle.ts"; +import type { NemoClawInstance } from "../framework/phases/index.ts"; +import type { + ShellProbeResult, + ShellProbeRunOptions, + TrustedShellCommand, +} from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +interface CleanupCall { + name: string; + run: () => Promise | void; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: exitCode === 0 ? output : "", + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run( + command: TrustedShellCommand, + options?: ShellProbeRunOptions, + ): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift(); + if (!response) { + throw new Error( + `FakeRunner response missing for command: ${command.command} ${command.args.join(" ")}`, + ); + } + return response; + } +} + +class FakeCleanup implements LifecycleCleanup { + readonly calls: CleanupCall[] = []; + + add(name: string, run: () => Promise | void): void { + this.calls.push({ name, run }); + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner, cleanup: FakeCleanup): LifecyclePhaseFixture { + const host = new HostCliClient(runner); + const sandbox = new SandboxClient(runner); + return new LifecyclePhaseFixture(host, sandbox, cleanup); +} + +describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", () => { + it("stops the gateway, discovers the labeled container, and stops it", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // openshell gateway stop + runner.enqueue(shellResult(0, "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance(), + ); + + expect(result.profile).toBe("post-reboot-recovery"); + expect(result.steps.map((step) => step.id)).toEqual([ + "gateway-stop", + "docker-stop:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", + ]); + expect(runner.calls.map((call) => ({ command: call.command, args: call.args }))).toEqual([ + { command: "openshell", args: ["gateway", "stop"] }, + { + command: "docker", + args: [ + "ps", + "-a", + "--filter", + "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw", + "--format", + "{{.Names}}", + ], + }, + { command: "docker", args: ["stop", "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw"] }, + ]); + expect(cleanup.calls.map((call) => call.name)).toEqual([ + "lifecycle.docker-start:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", + ]); + }); + + it("tolerates a non-zero gateway stop (post-reboot fresh runtime)", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "no gateway runtime")); // gateway stop fails + runner.enqueue(shellResult(0, "container-1\n")); + runner.enqueue(shellResult(0)); // docker stop + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance(), + ); + + expect(result.steps.find((step) => step.id === "gateway-stop")).toBeTruthy(); + }); + + it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(0, "\n")); // discover returns nothing + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).simulate("post-reboot-recovery", instance()), + ).rejects.toThrow(/expected at least one Docker container labeled/); + }); + + it("fails when docker discover returns non-zero", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).simulate("post-reboot-recovery", instance()), + ).rejects.toThrow(/could not query Docker for label/); + }); +}); + +describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-backup)", () => { + it("stops, then renames the labeled container to a *-nemoclaw-gpu-backup-* sibling", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // openshell gateway stop + runner.enqueue(shellResult(0, "openshell-cluster-e2e-x\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + runner.enqueue(shellResult(0)); // docker rename + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance({ sandboxName: "e2e-x" }), + { mode: "rename-to-gpu-backup" }, + ); + + expect(result.steps.map((step) => step.id.split("->")[0])).toContain( + "docker-rename:openshell-cluster-e2e-x", + ); + const renameCall = runner.calls.find( + (call) => call.command === "docker" && call.args[0] === "rename", + ); + expect(renameCall).toBeTruthy(); + expect(renameCall!.args[1]).toBe("openshell-cluster-e2e-x"); + expect(renameCall!.args[2]).toMatch(/^openshell-cluster-e2e-x-nemoclaw-gpu-backup-\d+$/); + + // Cleanup queue now has both docker-start and docker-rename-back. + expect(cleanup.calls.map((call) => call.name.split(":")[0])).toEqual([ + "lifecycle.docker-start", + "lifecycle.docker-rename-back", + ]); + }); +}); + +describe("LifecyclePhaseFixture profile dispatch", () => { + it("rejects unknown lifecycle profiles", async () => { + const runner = new FakeRunner(); + const cleanup = new FakeCleanup(); + + await expect( + // @ts-expect-error — exhaustiveness check + fixture(runner, cleanup).simulate("not-a-profile", instance()), + ).rejects.toThrow(/Unsupported lifecycle profile/); + }); + + it("exposes the lifecycle phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); + +describe("buildBackupContainerName", () => { + it("appends -nemoclaw-gpu-backup- to the original name", () => { + expect(buildBackupContainerName("openshell-cluster-foo", 1717280000000)).toBe( + "openshell-cluster-foo-nemoclaw-gpu-backup-1717280000000", + ); + }); + + it("truncates the original name to fit within Docker's 253-char limit", () => { + const longName = "a".repeat(253); + const result = buildBackupContainerName(longName, 1717280000000); + expect(result.length).toBeLessThanOrEqual(253); + expect(result.endsWith("-nemoclaw-gpu-backup-1717280000000")).toBe(true); + }); +}); diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index af31e4560a9..75891b38fa7 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -14,6 +14,7 @@ import { import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; import { EnvironmentPhaseFixture, + LifecyclePhaseFixture, OnboardingPhaseFixture, StateValidationPhaseFixture, } from "./phases/index.ts"; @@ -32,6 +33,7 @@ export interface E2EScenarioFixtures { state: StateClient; environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; + lifecycle: LifecyclePhaseFixture; stateValidation: StateValidationPhaseFixture; } @@ -91,6 +93,9 @@ export const test = base.extend({ onboard: async ({ cleanup, host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets, cleanup)); }, + lifecycle: async ({ cleanup, host, sandbox }, use) => { + await use(new LifecyclePhaseFixture(host, sandbox, cleanup)); + }, stateValidation: async ({ host, gateway, sandbox }, use) => { await use(new StateValidationPhaseFixture(host, gateway, sandbox)); }, diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index f6345ba476f..1905028a0c0 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -7,6 +7,14 @@ export { type DockerRuntimeReady, type EnvironmentReady, } from "./environment.ts"; +export { + LifecyclePhaseFixture, + type LifecycleCleanup, + type LifecycleProfile, + type LifecycleResult, + type PostRebootMode, + type PostRebootOptions, +} from "./lifecycle.ts"; export { OnboardingPhaseFixture, type NemoClawInstance, diff --git a/test/e2e-scenario/framework/phases/lifecycle.ts b/test/e2e-scenario/framework/phases/lifecycle.ts new file mode 100644 index 00000000000..75225c04811 --- /dev/null +++ b/test/e2e-scenario/framework/phases/lifecycle.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { SandboxClient } from "../clients/sandbox.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +// Mirror of `OPENSHELL_SANDBOX_NAME_LABEL` in +// `src/lib/onboard/docker-gpu-patch.ts`. Duplicated here because the +// fixture layer must not import from `src/lib/**` (CLI source) — that +// boundary keeps the live runner honest about probing only host- +// observable state. Drift is caught by the integration test that wires +// a real onboarded sandbox through the docker-sandbox-container-present +// probe. +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +const DOCKER_PROBE_TIMEOUT_MS = 15_000; +const GATEWAY_STOP_TIMEOUT_MS = 60_000; + +export type LifecycleProfile = "post-reboot-recovery"; + +export interface LifecycleCleanup { + add(name: string, run: () => Promise | void): void; +} + +/** + * How the post-reboot-recovery profile leaves Docker before the test + * exits the lifecycle phase: + * + * - `stop-original` — `docker stop` the labeled container in place. + * Matches the common Spark reboot path: the + * container exists, is exited, retains its + * OpenShell labels, but is no longer running. + * + * - `rename-to-gpu-backup` — stop the labeled container, then + * `docker rename` it to `-nemoclaw- + * gpu-backup-`. Reproduces the rarer GPU- + * patch reboot path where only the backup + * sibling survives and recovery has to rename + * it back. Mirrors `buildBackupContainerName()` + * in `src/lib/onboard/docker-gpu-patch.ts`. + */ +export type PostRebootMode = "stop-original" | "rename-to-gpu-backup"; + +export interface PostRebootOptions { + mode?: PostRebootMode; +} + +export interface LifecycleResult { + profile: LifecycleProfile; + steps: Array<{ id: string; results: ShellProbeResult[] }>; +} + +export class LifecyclePhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly sandbox: SandboxClient, + private readonly cleanup: LifecycleCleanup, + ) {} + + async simulate( + profile: LifecycleProfile, + instance: NemoClawInstance, + options: PostRebootOptions = {}, + ): Promise { + switch (profile) { + case "post-reboot-recovery": + return await this.simulatePostReboot(instance, options); + default: { + const _exhaustive: never = profile; + throw new Error(`Unsupported lifecycle profile '${_exhaustive}'.`); + } + } + } + + /** + * Reproduce the host-side conditions of a DGX Spark / Linux Docker-driver + * reboot: + * + * 1. Ask OpenShell to stop its gateway runtime so the in-memory + * sandbox view drops to NotFound. The actual sandbox container + * is unaffected — that is the entire point of the bug class + * tracked by #4423. + * + * 2. Locate the OpenShell-labeled Docker container for the + * scenario's sandbox name and either stop it (default) or + * stop+rename it to a `*-nemoclaw-gpu-backup-*` sibling. + * + * Cleanups (run in reverse order at end of test): + * - rename the backup sibling back to the original name (if we + * created one); + * - `docker start` the labeled container so the sandbox returns + * to a usable state for any teardown that expects it live. + */ + async simulatePostReboot( + instance: NemoClawInstance, + options: PostRebootOptions = {}, + ): Promise { + const mode: PostRebootMode = options.mode ?? "stop-original"; + const steps: LifecycleResult["steps"] = []; + + const gatewayStop = await this.sandbox.openshell(["gateway", "stop"], { + artifactName: "lifecycle-post-reboot-gateway-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: GATEWAY_STOP_TIMEOUT_MS, + }); + // gateway stop is best-effort: a fresh-start/no-runtime gateway + // will exit non-zero with NoSuchProcess, which is exactly the + // post-reboot state we want to simulate. Don't fail the lifecycle + // phase on it. + steps.push({ id: "gateway-stop", results: [gatewayStop] }); + + const containerNames = await this.discoverLabeledContainerNames(instance); + if (containerNames.length === 0) { + throw new Error( + `lifecycle.post-reboot-recovery expected at least one Docker container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}', but docker ps -a returned none. ` + + `Did onboarding create the sandbox?`, + ); + } + const originalName = containerNames[0]; + + const stop = await this.host.command( + "docker", + ["stop", originalName], + { + artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(stop, `docker stop ${originalName}`); + steps.push({ id: `docker-stop:${originalName}`, results: [stop] }); + this.cleanup.add(`lifecycle.docker-start:${originalName}`, async () => { + await this.host.command("docker", ["start", originalName], { + artifactName: `lifecycle-cleanup-docker-start-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); + }); + + if (mode === "rename-to-gpu-backup") { + const backupName = buildBackupContainerName(originalName, Date.now()); + const rename = await this.host.command( + "docker", + ["rename", originalName, backupName], + { + artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(rename, `docker rename ${originalName} ${backupName}`); + steps.push({ id: `docker-rename:${originalName}->${backupName}`, results: [rename] }); + this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { + await this.host.command("docker", ["rename", backupName, originalName], { + artifactName: `lifecycle-cleanup-docker-rename-back-${backupName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); + }); + } + + return { profile: "post-reboot-recovery", steps }; + } + + private async discoverLabeledContainerNames(instance: NemoClawInstance): Promise { + const result = await this.host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: `lifecycle-post-reboot-docker-discover-${instance.sandboxName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `lifecycle.post-reboot-recovery could not query Docker for label ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (exit ${result.exitCode}).`, + ); + } + return result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + } +} + +// Mirror of `MAX_DOCKER_CONTAINER_NAME_LENGTH` in +// `src/lib/onboard/docker-gpu-patch.ts`. +const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; + +export function buildBackupContainerName(originalName: string, nowMs: number): string { + const suffix = `-nemoclaw-gpu-backup-${String(nowMs)}`; + const maxOriginalLength = MAX_DOCKER_CONTAINER_NAME_LENGTH - suffix.length; + return `${originalName.slice(0, Math.max(1, maxOriginalLength))}${suffix}`; +} diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 892cadb8124..3395ac553a8 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -5,9 +5,16 @@ import fs from "node:fs"; import path from "node:path"; import { expect, test } from "../framework/e2e-test.ts"; +import type { LifecycleProfile } from "../framework/phases/index.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; +const LIFECYCLE_PROFILES: ReadonlySet = new Set(["post-reboot-recovery"]); + +function isLifecycleProfile(value: string | undefined): value is LifecycleProfile { + return value !== undefined && LIFECYCLE_PROFILES.has(value as LifecycleProfile); +} + const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); @@ -30,7 +37,7 @@ for (const scenario of listScenarios()) { test( liveScenarioTestName(scenario), - async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + async ({ artifacts, environment, lifecycle, onboard, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } @@ -55,6 +62,27 @@ for (const scenario of listScenarios()) { const ready = await environment.assertReady(scenario.environment); const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); + + // Lifecycle phase runs between onboard and state-validation. + // Scenarios opt in by setting `environment.lifecycle` to a + // whitelisted profile (see SUPPORTED_LIFECYCLES in + // runtime-support.ts). Today only `post-reboot-recovery` is + // wired, and it dispatches through `LifecyclePhaseFixture` to + // mutate host state (gateway runtime, Docker container) before + // the state-validation probes assert preservation invariants. + let lifecycleResult: Awaited> | undefined; + const profile = scenario.environment.lifecycle; + if (profile) { + if (!isLifecycleProfile(profile)) { + throw new Error( + `scenario '${scenario.id}' declares lifecycle '${profile}' which is not ` + + `dispatched by LifecyclePhaseFixture; update the fixture and the ` + + `SUPPORTED_LIFECYCLES whitelist together.`, + ); + } + lifecycleResult = await lifecycle.simulate(profile, instance); + } + const validation = await stateValidation.from(scenario.expectedStateId, instance); await artifacts.writeJson("scenario-result.json", { @@ -62,6 +90,9 @@ for (const scenario of listScenarios()) { expectedStateId: validation.state.id, probes: validation.probes.map((probe) => probe.id), pendingRuntimeSuites: support.pendingRuntimeSuites, + lifecycle: lifecycleResult + ? { profile: lifecycleResult.profile, steps: lifecycleResult.steps.map((s) => s.id) } + : undefined, }); }, ); From 5d132a728bfe32b8c9b2be9f57293f265594625e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 13:54:36 -0400 Subject: [PATCH 04/18] test(e2e): add post-reboot-recovery scenario as #4423 regression guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers the failing-test-first guard for #4423 in the typed scenario registry so the live Vitest matrix from #5006 fans it out as a dedicated CI job. Builds on the framework primitives added earlier in this PR (lifecycle phase fixture, host-side probes, lifecycle whitelist). Additions: * `post-reboot-recovery-ready` expected-state in `scenarios/expected-states.ts` declaring the user-visible invariants that must hold after a `nemoclaw status` call on a freshly-rebooted DGX Spark / Linux Docker-driver host: - cli installed, - gateway healthy (the user-systemd unit from #4580 brings it back up before status runs), - sandbox running (recovery completed in time), - localRegistry entry preserved (the user-visible regression target — destroyed on unfixed `main`), - dockerSandboxContainer present (recovery didn't delete the labeled container or its `*-nemoclaw-gpu-backup-*` sibling). * `ubuntu-repo-docker-post-reboot-recovery` scenario in `scenarios/scenarios/baseline.ts` wiring `ubuntuRepoDockerLifecycle("cloud-openclaw", "post-reboot-recovery")` against the new expected-state and a smoke suite. Carries a description that explains the RED/GREEN contract and points to the PR-A fix landing in `src/lib/`. * `manifests/openclaw-nvidia-post-reboot-recovery.yaml` declares `lifecycle: post-reboot-recovery` and the same NVIDIA_API_KEY credential ref the cloud-openclaw scenarios use. * `.github/workflows/e2e-scenarios.yaml` ROUTES table gains the new scenario so the workflow-boundary test (`e2e-scenarios-workflow.test.ts`) routes every typed id. Test pinning: * `e2e-scenario-matrix.test.ts` updated from a 1-entry to a 2-entry live matrix expectation. The new entry asserts on `expectedStateId: "post-reboot-recovery-ready"` so a future accidental dropped-lifecycle change to the scenario regresses loudly. * `e2e-live-registry-discovery.test.ts` swaps the synthetic whitelist-coverage test for an assertion against the real `ubuntu-repo-docker-post-reboot-recovery` registry entry. Behavior: * On unfixed `main`, the live runner's lifecycle phase stops the OpenShell gateway runtime and `docker stop`s the labeled sandbox container. State-validation then runs `nemoclaw status` (which restarts the gateway via systemd) and the destructive `missing` branch in `src/lib/actions/sandbox/status.ts` wipes the local registry entry. The `local-registry-entry-present` probe fails. Scenario goes RED. * On the PR-A fix branch, the new Docker-driver sandbox recovery helper restarts the labeled container before stale-removal can fire, registry survives, all five probes pass. Scenario flips GREEN. The bash-side legacy compiler emits a `lifecycle.profile.post-reboot-recovery` PhaseAction pointing at `nemoclaw_scenarios/lifecycle/dispatch.sh`, but the legacy bash worker is intentionally not provided: this scenario is Vitest-only. The typed runner's `LifecyclePhaseFixture` handles dispatch directly. If the legacy runner is invoked against this scenario it errors out at the dispatcher; that's the right failure mode while the bash side stays on its own retirement clock. Refs #4423 --- .github/workflows/e2e-scenarios.yaml | 1 + .../e2e-live-registry-discovery.test.ts | 30 +++++--------- .../e2e-scenario-matrix.test.ts | 22 ++++++++++- .../openclaw-nvidia-post-reboot-recovery.yaml | 39 +++++++++++++++++++ .../e2e-scenario/scenarios/expected-states.ts | 26 +++++++++++++ .../scenarios/scenarios/baseline.ts | 32 +++++++++++++++ 6 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 49f317caff2..804426f1331 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -78,6 +78,7 @@ jobs: [ubuntu-repo-cloud-openclaw-slack]=ubuntu-latest [ubuntu-repo-cloud-openclaw-telegram]=ubuntu-latest [ubuntu-repo-cloud-openclaw-token-rotation]=ubuntu-latest + [ubuntu-repo-docker-post-reboot-recovery]=ubuntu-latest [ubuntu-repo-openai-compatible-openclaw]=ubuntu-latest ) selected="" diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index db1dea7f4d1..35f51b78741 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -63,26 +63,16 @@ describe("live Vitest registry discovery support", () => { }); }); - it("accepts whitelisted lifecycle profiles when the rest of the environment matches", () => { - // Synthesised scenario stands in for the not-yet-registered - // post-reboot-recovery scenario so this test pins the whitelist - // contract independently of when the scenario lands. Once the - // post-reboot-recovery scenario is registered, prefer asserting on - // the registry entry directly and remove this synthetic. - const supported = liveScenarioSupport({ - id: "synthetic-post-reboot-recovery", - assertionGroups: [], - expectedStateId: "cloud-openclaw-ready", - environment: { - platform: "ubuntu-local", - install: "repo-current", - runtime: "docker-running", - onboarding: "cloud-openclaw", - lifecycle: "post-reboot-recovery", - }, - }); + it("accepts the whitelisted post-reboot-recovery lifecycle scenario", () => { + const scenario = listScenarios().find( + (entry) => entry.id === "ubuntu-repo-docker-post-reboot-recovery", + ); - expect(supported.supported).toBe(true); - expect(supported.reasons).toEqual([]); + expect(scenario).toBeTruthy(); + expect(scenario!.environment?.lifecycle).toBe("post-reboot-recovery"); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: true, + reasons: [], + }); }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index a6a9bf9d18e..78eb216d53d 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -131,6 +131,7 @@ describe("typed scenario matrix", () => { it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-docker-post-reboot-recovery", ]); expect(buildLiveScenarioMatrix()[0]).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", @@ -145,6 +146,22 @@ describe("typed scenario matrix", () => { supportReasons: [], pendingRuntimeSuites: ["smoke", "inference", "credentials"], }); + // Failing-test-first guard for #4423. Pinned in the matrix to + // confirm the lifecycle whitelist + post-reboot-recovery scenario + // are wired together; the actual RED/GREEN behavior is exercised + // by the live runner (gates on the fix landing in src/lib/). + expect(buildLiveScenarioMatrix()[1]).toMatchObject({ + id: "ubuntu-repo-docker-post-reboot-recovery", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + expectedStateId: "post-reboot-recovery-ready", + requiredSecrets: ["NVIDIA_API_KEY"], + supported: true, + supportReasons: [], + }); }); it("keeps explicitly selected unsupported live scenarios in the matrix with skip reasons", () => { @@ -163,7 +180,10 @@ describe("typed scenario matrix", () => { const lines = result.stdout.trim().split("\n"); expect(lines.length, "live matrix output must be a single line").toBe(1); const parsed = JSON.parse(lines[0]); - expect(parsed.map((entry: { id: string }) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([ + "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-docker-post-reboot-recovery", + ]); }); it("--emit-live-matrix honors explicit scenario selections", () => { diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml new file mode 100644 index 00000000000..571ec5b2f5f --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-post-reboot-recovery +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + # Lifecycle phase opt-in. The Vitest live runner dispatches this + # profile through `LifecyclePhaseFixture.simulate(...)`, which: + # 1. stops the OpenShell gateway runtime, and + # 2. `docker stop`s the labeled sandbox container. + # The host-side state-validation probes + # (`local-registry-entry-present`, `docker-sandbox-container-present`) + # then assert that `nemoclaw status` recovers the sandbox + # without destroying registry state. On unfixed code the registry + # entry is wiped by the `missing` branch in + # `src/lib/actions/sandbox/{status,gateway-state}.ts` and the + # `local-registry-entry-present` probe fails. See #4423. + lifecycle: post-reboot-recovery + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index d79b32e185d..75027f5e238 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -77,6 +77,31 @@ const onboardingFailureGatewayPortConflict: ExpectedState = { sandbox: { expected: "absent" }, }; +// Post-reboot recovery contract for #4423. After the lifecycle phase +// stops the OpenShell gateway runtime + the labeled sandbox container, +// the user-visible invariants are: +// +// * `cli` still installed. +// * `gateway` healthy: the user-systemd unit from #4580 brings the +// gateway back up before status runs. +// * `sandbox` running by the time validation completes: a correct +// fix performs Docker-backed recovery before responding. +// * `localRegistry` entry preserved: this is the user-visible +// regression target. On unfixed code, the destructive `missing` +// branch wipes the entry; on fixed code it survives because +// Docker corroborated the sandbox container existence. +// * `dockerSandboxContainer` still present: the recovery path must +// not delete the labeled container or its `*-nemoclaw-gpu-backup-*` +// sibling as a side effect. +const postRebootRecoveryReady: ExpectedState = { + id: "post-reboot-recovery-ready", + cli: { installed: true }, + gateway: { expected: "present", health: "healthy" }, + sandbox: { expected: "present", status: "running", agent: "openclaw" }, + localRegistry: { expected: "present" }, + dockerSandboxContainer: { expected: "present" }, +}; + const REGISTRY: readonly ExpectedState[] = [ cloudOpenclawReady, cloudOpenclawCustomPoliciesReady, @@ -86,6 +111,7 @@ const REGISTRY: readonly ExpectedState[] = [ preflightFailureNoSandbox, onboardingFailureInvalidNvidiaKey, onboardingFailureGatewayPortConflict, + postRebootRecoveryReady, ]; const BY_ID: ReadonlyMap = new Map( diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index a5b0e050406..4e5befad679 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -148,6 +148,38 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ suiteIds: ["smoke", "rebuild", "upgrade"], requiredSecrets: ["NVIDIA_API_KEY"], }, + { + // Failing-test-first regression guard for #4423. After onboarding, + // the lifecycle phase reproduces the host-side conditions of a + // DGX Spark / Linux Docker-driver reboot: stop the OpenShell + // gateway runtime + `docker stop` the labeled sandbox container. + // The state-validation phase then runs `nemoclaw status` + // and asserts the post-recovery invariants declared by the + // `post-reboot-recovery-ready` expected-state. + // + // On unfixed `main`, the destructive `missing` branch in + // `src/lib/actions/sandbox/status.ts` (and the parallel branch + // reached through `ensureLiveSandboxOrExit` in + // `src/lib/actions/sandbox/gateway-state.ts`) wipes the local + // registry entry once the gateway returns to `healthy_named`, + // so the `local-registry-entry-present` probe fails and this + // scenario goes RED. + // + // The fix lands in PR-A (parts 2 & 3 of ericksoa's plan): add a + // Docker-driver sandbox recovery helper, then tighten + // stale-removal in active paths to require Docker-corroborated + // absence before destroying the registry. PR-A flips this guard + // 🔴 → 🟢. + id: "ubuntu-repo-docker-post-reboot-recovery", + manifestName: "openclaw-nvidia-post-reboot-recovery", + environment: ubuntuRepoDockerLifecycle("cloud-openclaw", "post-reboot-recovery"), + expectedStateId: "post-reboot-recovery-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + description: + "Failing-test-first guard for #4423: post-reboot recovery must preserve " + + "the local registry entry and restart the labeled Docker container.", + }, { id: "ubuntu-repo-openai-compatible-openclaw", manifestName: "openclaw-openai-compatible", From 34466dd271422e450d9038f2bb826494f3a9c090 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:06:13 -0700 Subject: [PATCH 05/18] chore(e2e): scaffold inventory internals migration draft Signed-off-by: Carlos Villela From 54ac3e386a4e8b0f4240190035778d70f27460cd Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:08:14 -0700 Subject: [PATCH 06/18] chore(e2e): scaffold fan-out draft 05 Signed-off-by: Carlos Villela From 57b84ae19338353972bdba141000b84c9ed0f48e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:08:28 -0700 Subject: [PATCH 07/18] chore(e2e): scaffold fan-out draft 06 Signed-off-by: Carlos Villela From 06b84f36b6ef637259468f86e824f1ca188517ce Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 14:22:08 -0400 Subject: [PATCH 08/18] chore(e2e): apply biome formatting Prek hook auto-fixed formatting in 6 files added/touched by this PR. No behavior change. --- node_modules | 1 + .../e2e-expected-state.test.ts | 10 ++----- .../e2e-live-registry-discovery.test.ts | 4 +-- .../e2e-phase-lifecycle.test.ts | 16 ++--------- .../e2e-phase-state-validation.test.ts | 5 +--- .../framework/phases/lifecycle.ts | 28 +++++++------------ .../framework/phases/state-validation.ts | 4 +-- 7 files changed, 19 insertions(+), 49 deletions(-) create mode 120000 node_modules diff --git a/node_modules b/node_modules new file mode 120000 index 00000000000..8b4b1610468 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/Users/jyaunches/Development/NemoClaw/node_modules \ No newline at end of file diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index dcf15974923..173a32a0070 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -90,10 +90,7 @@ describe("probesForState maps typed expected-state into probe ids", () => { cli: { installed: true }, localRegistry: { expected: "present" }, }; - expect(probesForState(state)).toEqual([ - "cli-installed", - "local-registry-entry-present", - ]); + expect(probesForState(state)).toEqual(["cli-installed", "local-registry-entry-present"]); }); it("dockerSandboxContainer.expected=present emits the docker-sandbox-container-present probe", () => { @@ -102,10 +99,7 @@ describe("probesForState maps typed expected-state into probe ids", () => { cli: { installed: true }, dockerSandboxContainer: { expected: "present" }, }; - expect(probesForState(state)).toEqual([ - "cli-installed", - "docker-sandbox-container-present", - ]); + expect(probesForState(state)).toEqual(["cli-installed", "docker-sandbox-container-present"]); }); it("localRegistry/dockerSandboxContainer 'absent' emits no probe today", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 35f51b78741..7158039d2c8 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -57,9 +57,7 @@ describe("live Vitest registry discovery support", () => { expect(scenario).toBeTruthy(); expect(liveScenarioSupport(scenario!)).toMatchObject({ supported: false, - reasons: [ - "lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures", - ], + reasons: ["lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures"], }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts index b4cae772551..5590016373d 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts @@ -3,11 +3,7 @@ import { describe, expect, expectTypeOf, it } from "vitest"; -import { - HostCliClient, - SandboxClient, - type CommandRunner, -} from "../framework/clients/index.ts"; +import { HostCliClient, SandboxClient, type CommandRunner } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; import { buildBackupContainerName, @@ -106,10 +102,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // docker stop const cleanup = new FakeCleanup(); - const result = await fixture(runner, cleanup).simulate( - "post-reboot-recovery", - instance(), - ); + const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance()); expect(result.profile).toBe("post-reboot-recovery"); expect(result.steps.map((step) => step.id)).toEqual([ @@ -143,10 +136,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // docker stop const cleanup = new FakeCleanup(); - const result = await fixture(runner, cleanup).simulate( - "post-reboot-recovery", - instance(), - ); + const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance()); expect(result.steps.find((step) => step.id === "gateway-stop")).toBeTruthy(); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index 50cf1306d59..2d4011061b7 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -541,10 +541,7 @@ describe("state-validation host-side probes", () => { it("docker-sandbox-container-present matches *-nemoclaw-gpu-backup-* sibling containers", async () => { const runner = new FakeRunner(); runner.enqueue( - shellResult( - 0, - "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n", - ), + shellResult(0, "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n"), ); const fx = fixture(runner); diff --git a/test/e2e-scenario/framework/phases/lifecycle.ts b/test/e2e-scenario/framework/phases/lifecycle.ts index 75225c04811..3b130f789fa 100644 --- a/test/e2e-scenario/framework/phases/lifecycle.ts +++ b/test/e2e-scenario/framework/phases/lifecycle.ts @@ -122,15 +122,11 @@ export class LifecyclePhaseFixture { } const originalName = containerNames[0]; - const stop = await this.host.command( - "docker", - ["stop", originalName], - { - artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }, - ); + const stop = await this.host.command("docker", ["stop", originalName], { + artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); assertExitZero(stop, `docker stop ${originalName}`); steps.push({ id: `docker-stop:${originalName}`, results: [stop] }); this.cleanup.add(`lifecycle.docker-start:${originalName}`, async () => { @@ -143,15 +139,11 @@ export class LifecyclePhaseFixture { if (mode === "rename-to-gpu-backup") { const backupName = buildBackupContainerName(originalName, Date.now()); - const rename = await this.host.command( - "docker", - ["rename", originalName, backupName], - { - artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }, - ); + const rename = await this.host.command("docker", ["rename", originalName, backupName], { + artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); assertExitZero(rename, `docker rename ${originalName} ${backupName}`); steps.push({ id: `docker-rename:${originalName}->${backupName}`, results: [rename] }); this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index e919adc17de..d7a9e5de2d1 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -278,9 +278,7 @@ export class StateValidationPhaseFixture { return { id: "sandbox-running", status: "passed", results: [result] }; } - private expectLocalRegistryEntryPresent( - instance: NemoClawInstance, - ): StateValidationProbeResult { + private expectLocalRegistryEntryPresent(instance: NemoClawInstance): StateValidationProbeResult { const reader = this.io.readRegistry ?? defaultReadRegistry; const registry = reader(); if (!registry) { From 839b7bfd874669aed369bcac4ba42cfe7b5ed843 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 14:22:49 -0400 Subject: [PATCH 09/18] chore(e2e): drop accidental node_modules symlink The biome-format commit accidentally added a node_modules symlink alongside the formatting fixes. Remove it; the directory is already in .gitignore. --- node_modules | 1 - 1 file changed, 1 deletion(-) delete mode 120000 node_modules diff --git a/node_modules b/node_modules deleted file mode 120000 index 8b4b1610468..00000000000 --- a/node_modules +++ /dev/null @@ -1 +0,0 @@ -/Users/jyaunches/Development/NemoClaw/node_modules \ No newline at end of file From 5ed17356bc3b4717b39614824c5382383fc641e7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:06:13 -0700 Subject: [PATCH 10/18] test(e2e): extend migration inventory to runner internals --- test/e2e-scenario/docs/MIGRATION.md | 14 +- test/e2e-scenario/docs/README.md | 9 +- .../e2e-migration-inventory.test.ts | 148 +++++++++++++++--- .../migration/legacy-inventory.json | 70 +++++++++ 4 files changed, 209 insertions(+), 32 deletions(-) diff --git a/test/e2e-scenario/docs/MIGRATION.md b/test/e2e-scenario/docs/MIGRATION.md index 0da12a97e59..170b3871d0d 100644 --- a/test/e2e-scenario/docs/MIGRATION.md +++ b/test/e2e-scenario/docs/MIGRATION.md @@ -84,18 +84,22 @@ that owns the work instead. The one repo-local exception is the machine-readable deletion gate inventory at `test/e2e-scenario/migration/legacy-inventory.json`. Keep that file focused on -script-level migration state that prevents accidental legacy E2E deletion. It +deletion-readiness evidence that prevents accidental legacy E2E deletion. It must cover every direct legacy shell entrypoint under `test/e2e/test-*.sh`, -plus any explicitly retained bridge entrypoints such as Brev. It is not a -progress dashboard or owner queue: +plus any explicitly retained bridge entrypoints such as Brev. It also tracks +coarse internal legacy runner surfaces such as the YAML/bash scenario workers, +validation suites, TypeScript shell-runner orchestrators, and runtime helper +libraries so those surfaces cannot be removed without #4357 evidence. It is not +a progress dashboard or owner queue: - `not-migrated`: legacy coverage still has no equivalent Vitest scenario. - `bridge-probe`: coverage is temporarily represented by a bridge path. - `covered`: equivalent Vitest live scenario coverage exists. - `retired`: maintainers agreed the legacy coverage is no longer required. -Do not set `deletionReady: true` unless the entry is `covered` or `retired` and -the deletion approval is recorded through #4357. +Do not set `deletionReady: true` on a script entry or internal surface unless +the record is `covered` or `retired` and the deletion approval is recorded +through #4357. After #4357 completes final legacy E2E reconciliation, remove the inventory if there are no remaining legacy entrypoints to guard. If maintainers keep it, keep diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index 10893cb5d03..8f0f12f704b 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -203,9 +203,12 @@ is tracked in #4941. The narrow repo-local exception is `test/e2e-scenario/migration/legacy-inventory.json`, a machine-readable deletion gate for direct legacy `test/e2e/test-*.sh` entrypoints and explicit bridge -entrypoints. It should prevent accidental deletions, not become a parallel -status table. Remove it after #4357 completes final legacy E2E reconciliation, -or keep it only as an audit artifact if maintainers still need that record. +entrypoints. It also tracks coarse internal legacy runner surfaces such as +scenario shell workers, validation suites, shell-runner orchestrators, and +runtime helper libraries so they cannot be removed without #4357 evidence. It +should prevent accidental deletions, not become a parallel status table. Remove +it after #4357 completes final legacy E2E reconciliation, or keep it only as an +audit artifact if maintainers still need that record. The old workflow-level parity report has been removed. Use scenario framework tests, the coverage report, PR review, and the audit issues to decide what to diff --git a/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts b/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts index fe43f3ca724..14998a690a5 100644 --- a/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts @@ -10,6 +10,14 @@ const INVENTORY_PATH = path.resolve(import.meta.dirname, "../migration/legacy-in const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const LEGACY_E2E_DIR = path.join(REPO_ROOT, "test/e2e"); const EXPECTED_STATUS_VALUES = ["not-migrated", "bridge-probe", "covered", "retired"] as const; +const INTERNAL_SURFACE_ROOTS = [ + "test/e2e-scenario/nemoclaw_scenarios", + "test/e2e-scenario/onboarding_assertions", + "test/e2e-scenario/runtime/lib", + "test/e2e-scenario/runtime/reports", + "test/e2e-scenario/scenarios/orchestrators", + "test/e2e-scenario/validation_suites", +] as const; type MigrationStatus = "not-migrated" | "bridge-probe" | "covered" | "retired"; @@ -26,6 +34,21 @@ interface LegacyInventoryEntry { notes: string; } +interface LegacyInternalSurface { + id: string; + paths: string[]; + domain: string; + ownerIssue: string; + status: MigrationStatus; + replacementSurface: string; + targetVitestScenarios: string[]; + bridgeProbes: string[]; + retiredReason: string; + deletionReady: boolean; + deletionApprovalIssue?: string; + notes: string; +} + interface LegacyInventory { version: number; statusValues: MigrationStatus[]; @@ -33,6 +56,7 @@ interface LegacyInventory { requires: string[]; }; entries: LegacyInventoryEntry[]; + internalSurfaces: LegacyInternalSurface[]; } function loadInventory(): LegacyInventory { @@ -54,16 +78,86 @@ function listLegacyShellEntrypoints(): string[] { .sort(); } +function listRepoFilesUnder(repoRelativeDir: string): string[] { + const absoluteDir = path.join(REPO_ROOT, repoRelativeDir); + const files: string[] = []; + const visit = (dir: string) => { + for (const dirent of fs.readdirSync(dir, { withFileTypes: true })) { + const absolutePath = path.join(dir, dirent.name); + if (dirent.isDirectory()) { + visit(absolutePath); + } else if (dirent.isFile()) { + files.push(path.relative(REPO_ROOT, absolutePath).split(path.sep).join("/")); + } + } + }; + visit(absoluteDir); + return files.sort(); +} + +function isCoveredByInventoryPath(filePath: string, inventoryPath: string): boolean { + return filePath === inventoryPath || filePath.startsWith(`${inventoryPath}/`); +} + +function expectPathListIsRepoRelative(paths: readonly string[]) { + expect(paths.length).toBeGreaterThan(0); + for (const repoRelativePath of paths) { + expect(repoRelativePath).not.toBe(""); + expect(repoPathExists(repoRelativePath)).toBe(true); + } +} + +function expectMigrationRecordDeletionGate( + record: Pick< + LegacyInventoryEntry | LegacyInternalSurface, + | "status" + | "targetVitestScenarios" + | "bridgeProbes" + | "retiredReason" + | "deletionReady" + | "deletionApprovalIssue" + >, +) { + if (record.status === "covered") { + expect(record.targetVitestScenarios.length).toBeGreaterThan(0); + for (const scenario of record.targetVitestScenarios) { + expect(scenario).toMatch(/^test\/e2e-scenario\/live\/.+\.test\.ts$/); + expect(repoPathExists(scenario)).toBe(true); + } + } + + if (record.status === "bridge-probe") { + expect(record.bridgeProbes.length).toBeGreaterThan(0); + for (const probe of record.bridgeProbes) { + expect(repoPathExists(probe)).toBe(true); + } + } + + if (record.status === "retired") { + expect(record.retiredReason).not.toBe(""); + } + + if (record.deletionReady) { + expect(["covered", "retired"]).toContain(record.status); + expect(record.deletionApprovalIssue).toBe("#4357"); + expect( + record.status === "retired" ? record.retiredReason : record.targetVitestScenarios.length, + ).toBeTruthy(); + } +} + describe("E2E migration inventory deletion gates", () => { it("uses a constrained migration vocabulary with owning issues", () => { const inventory = loadInventory(); const statuses = new Set(inventory.statusValues); const legacyScripts = new Set(); + const internalSurfaceIds = new Set(); expect(inventory.version).toBe(1); expect(inventory.statusValues).toEqual([...EXPECTED_STATUS_VALUES]); expect(inventory.deletionReadiness.requires.length).toBeGreaterThan(0); expect(inventory.entries.length).toBeGreaterThan(0); + expect(inventory.internalSurfaces.length).toBeGreaterThan(0); for (const entry of inventory.entries) { expect(statuses.has(entry.status)).toBe(true); @@ -75,6 +169,18 @@ describe("E2E migration inventory deletion gates", () => { expect(entry.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/); expect(entry.notes).not.toBe(""); } + + for (const surface of inventory.internalSurfaces) { + expect(statuses.has(surface.status)).toBe(true); + expect(surface.id).toMatch(/^[a-z0-9-]+$/); + expect(internalSurfaceIds.has(surface.id)).toBe(false); + internalSurfaceIds.add(surface.id); + expectPathListIsRepoRelative(surface.paths); + expect(surface.domain).not.toBe(""); + expect(surface.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/); + expect(surface.replacementSurface).not.toBe(""); + expect(surface.notes).not.toBe(""); + } }); it("covers every current direct legacy shell entrypoint", () => { @@ -87,36 +193,30 @@ describe("E2E migration inventory deletion gates", () => { expect(inventoriedShellScripts).toEqual(listLegacyShellEntrypoints()); }); - it("requires coverage, retirement evidence, and #4357 approval before deletion", () => { + it("covers legacy scenario runner internal surfaces by path", () => { const inventory = loadInventory(); + const surfacePaths = inventory.internalSurfaces.flatMap((surface) => surface.paths); - for (const entry of inventory.entries) { - if (entry.status === "covered") { - expect(entry.targetVitestScenarios.length).toBeGreaterThan(0); - for (const scenario of entry.targetVitestScenarios) { - expect(scenario).toMatch(/^test\/e2e-scenario\/live\/.+\.test\.ts$/); - expect(repoPathExists(scenario)).toBe(true); - } + for (const root of INTERNAL_SURFACE_ROOTS) { + const files = listRepoFilesUnder(root); + expect(files.length).toBeGreaterThan(0); + for (const file of files) { + expect( + surfacePaths.some((surfacePath) => isCoveredByInventoryPath(file, surfacePath)), + ).toBe(true); } + } + }); - if (entry.status === "bridge-probe") { - expect(entry.bridgeProbes.length).toBeGreaterThan(0); - for (const probe of entry.bridgeProbes) { - expect(repoPathExists(probe)).toBe(true); - } - } + it("requires coverage, retirement evidence, and #4357 approval before deletion", () => { + const inventory = loadInventory(); - if (entry.status === "retired") { - expect(entry.retiredReason).not.toBe(""); - } + for (const entry of inventory.entries) { + expectMigrationRecordDeletionGate(entry); + } - if (entry.deletionReady) { - expect(["covered", "retired"]).toContain(entry.status); - expect(entry.deletionApprovalIssue).toBe("#4357"); - expect( - entry.status === "retired" ? entry.retiredReason : entry.targetVitestScenarios.length, - ).toBeTruthy(); - } + for (const surface of inventory.internalSurfaces) { + expectMigrationRecordDeletionGate(surface); } }); }); diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index 154bdf76c67..e102478d913 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -816,5 +816,75 @@ "deletionReady": false, "notes": "Already uses Vitest, but still dispatches legacy remote shell suites; keep as a bridge until remote execution uses shared fixtures." } + ], + "internalSurfaces": [ + { + "id": "typed-shell-orchestrators", + "paths": ["test/e2e-scenario/scenarios/orchestrators"], + "domain": "scenario-runner", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Retire after the registry-driven Vitest runner owns phase ordering, expected-failure matching, cleanup, redaction, and artifact evidence." + }, + { + "id": "legacy-bash-scenario-workers", + "paths": ["test/e2e-scenario/nemoclaw_scenarios"], + "domain": "scenario-runner", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Install, onboarding, lifecycle, probe, fixture, and context workers are bridge adapters until equivalent Vitest fixtures or CI setup actions own those phases." + }, + { + "id": "legacy-onboarding-assertion-workers", + "paths": ["test/e2e-scenario/onboarding_assertions"], + "domain": "smoke-onboarding", + "ownerIssue": "#4348", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases/onboarding.ts", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Retire after onboarding phase fixtures emit equivalent pass/fail evidence for base install and preflight assertions." + }, + { + "id": "legacy-validation-suites", + "paths": ["test/e2e-scenario/validation_suites"], + "domain": "runtime-suites", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Runtime suite assertions migrate one family at a time into typed Vitest runtime helpers before this shell suite tree can be removed." + }, + { + "id": "legacy-runtime-helper-libraries", + "paths": [ + "test/e2e-scenario/runtime/lib", + "test/e2e-scenario/runtime/reports" + ], + "domain": "scenario-runner", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Runtime helper libraries stay only while bridge shell workers need shared environment, logging, context, teardown, or report rendering behavior." + } ] } From 24b7e890cf50c57a09b65b76a8f7818e7ece37af Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:40:15 -0700 Subject: [PATCH 11/18] chore(e2e): apply static formatting --- src/commands/sandbox/agents/list.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/commands/sandbox/agents/list.ts b/src/commands/sandbox/agents/list.ts index c2f373b6f91..c5d2027cc72 100644 --- a/src/commands/sandbox/agents/list.ts +++ b/src/commands/sandbox/agents/list.ts @@ -24,7 +24,12 @@ export default class SandboxAgentsListCommand extends NemoClawCommand { public async run(): Promise { this.parsed = true; const [sandboxName, ...extraArgs] = this.argv; - if (!sandboxName || sandboxName.trim() === "" || sandboxName === "--help" || sandboxName === "-h") { + if ( + !sandboxName || + sandboxName.trim() === "" || + sandboxName === "--help" || + sandboxName === "-h" + ) { printAgentsPassthroughHelp("list"); return; } From d47a7234b2b968da7345f73279a6ecd182aeba29 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:08:14 -0700 Subject: [PATCH 12/18] chore(e2e): scaffold fan-out draft 05 Signed-off-by: Carlos Villela From 9c828841d9bcc367a1be807528e63894870267d9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 15:46:25 -0700 Subject: [PATCH 13/18] test(e2e): add inference runtime helpers Signed-off-by: Carlos Villela --- .../framework-tests/e2e-phase-runtime.test.ts | 255 +++++++++++++++ test/e2e-scenario/framework/clients/index.ts | 2 + .../framework/clients/provider.ts | 39 ++- test/e2e-scenario/framework/e2e-test.ts | 5 + test/e2e-scenario/framework/phases/index.ts | 11 + test/e2e-scenario/framework/phases/runtime.ts | 291 ++++++++++++++++++ 6 files changed, 597 insertions(+), 6 deletions(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts create mode 100644 test/e2e-scenario/framework/phases/runtime.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts new file mode 100644 index 00000000000..e3c9c71c3bd --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts @@ -0,0 +1,255 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + ProviderClient, + SandboxClient, + trustedProviderEndpoint, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { + inferenceRouteUrl, + RuntimePhaseFixture, + type NemoClawInstance, +} from "../framework/phases/index.ts"; +import type { + ShellProbeResult, + ShellProbeRunOptions, + TrustedShellCommand, +} from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, stdout = "", stderr = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout, + stderr, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run( + command: TrustedShellCommand, + options?: ShellProbeRunOptions, + ): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift(); + if (!response) { + throw new Error( + `FakeRunner response missing for command: ${command.command} ${command.args.join(" ")}`, + ); + } + return response; + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner): RuntimePhaseFixture { + return new RuntimePhaseFixture(new SandboxClient(runner), new ProviderClient(runner)); +} + +describe("runtime phase fixture", () => { + it("is available through the Vitest E2E fixture context", () => { + expectTypeOf().toEqualTypeOf(); + }); + + it("normalizes inference route slugs to the sandbox DNS hostname", () => { + expect(inferenceRouteUrl()).toBe("https://inference.local/v1/models"); + expect(inferenceRouteUrl("inference-local", "v1/chat/completions")).toBe( + "https://inference.local/v1/chat/completions", + ); + expect(inferenceRouteUrl("inference.local", "/v1/models")).toBe( + "https://inference.local/v1/models", + ); + }); + + it("checks inference.local models from inside the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, '{"data":[{"id":"nvidia/model"}]}')); + + const result = await fixture(runner).expectInferenceLocalModels(instance()); + + expect(result.endpoint).toBe("https://inference.local/v1/models"); + expect(runner.calls).toEqual([ + { + command: "openshell", + args: [ + "sandbox", + "exec", + "e2e-ubuntu-repo-cloud-openclaw", + "--", + "curl", + "-fsS", + "--max-time", + "20", + "https://inference.local/v1/models", + ], + options: { + artifactName: "runtime-inference-local-models", + env: expect.objectContaining({ PATH: expect.any(String) }), + redactionValues: [], + timeoutMs: 60_000, + }, + }, + ]); + }); + + it("posts an OpenAI-compatible chat completion to inference.local without shell interpolation", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, JSON.stringify({ choices: [{ message: { content: "ok" } }] }))); + + await fixture(runner).expectInferenceLocalChatCompletion(instance(), { + artifactName: "custom-chat", + maxTokens: 12, + model: "default", + prompt: "Reply with ok", + }); + + const call = runner.calls[0]; + expect(call?.command).toBe("openshell"); + expect(call?.args).toEqual([ + "sandbox", + "exec", + "e2e-ubuntu-repo-cloud-openclaw", + "--", + "curl", + "-fsS", + "--max-time", + "20", + "-H", + "Content-Type: application/json", + "-d", + expect.any(String), + "https://inference.local/v1/chat/completions", + ]); + expect(call?.args).not.toContain("sh"); + const payload = JSON.parse(call?.args[11] ?? "{}"); + expect(payload).toEqual({ + model: "default", + messages: [{ role: "user", content: "Reply with ok" }], + max_tokens: 12, + }); + expect(call?.options?.artifactName).toBe("custom-chat"); + }); + + it("accepts configured status codes for auth-proxy and route-health checks", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "403")); + + await fixture(runner).expectInferenceLocalStatus(instance(), { + allowedStatusCodes: [401, 403], + headers: ["Authorization: Bearer local-proxy-token"], + redactionValues: ["local-proxy-token"], + }); + + expect(runner.calls[0]).toMatchObject({ + command: "openshell", + args: [ + "sandbox", + "exec", + "e2e-ubuntu-repo-cloud-openclaw", + "--", + "curl", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "20", + "-H", + "Authorization: Bearer local-proxy-token", + "https://inference.local/v1/models", + ], + options: { + artifactName: "runtime-inference-local-status", + redactionValues: ["local-proxy-token"], + timeoutMs: 60_000, + }, + }); + }); + + it("calls a trusted compatible provider endpoint with request artifacts and redaction", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, JSON.stringify({ choices: [{ message: { content: "pong" } }] }))); + const endpoint = trustedProviderEndpoint("https://api.example.test/v1/chat/completions", { + allowedHosts: ["api.example.test"], + }); + + const result = await fixture(runner).expectProviderChatCompletion(endpoint, { + apiKey: "provider-secret", + model: "nvidia/model", + prompt: "Reply with pong", + }); + + expect(result.endpoint).toBe("https://api.example.test/v1/chat/completions"); + expect(runner.calls[0]).toEqual({ + command: "curl", + args: [ + "-fsS", + "-H", + "Content-Type: application/json", + "-H", + "Authorization: Bearer provider-secret", + "-d", + JSON.stringify({ + model: "nvidia/model", + messages: [{ role: "user", content: "Reply with pong" }], + max_tokens: 8, + }), + "https://api.example.test/v1/chat/completions", + ], + options: { + artifactName: "curl-https-api.example.test-v1-chat-completions", + redactionValues: ["provider-secret"], + timeoutMs: 60_000, + }, + }); + }); + + it("fails chat probes on malformed compatible responses without echoing the body", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "not json with provider-secret")); + + await expect( + fixture(runner).expectInferenceLocalChatCompletion(instance(), { + redactionValues: ["provider-secret"], + }), + ).rejects.toThrow("inference.local chat completion response was not JSON"); + }); +}); diff --git a/test/e2e-scenario/framework/clients/index.ts b/test/e2e-scenario/framework/clients/index.ts index fcaac7b3918..22144130a9d 100644 --- a/test/e2e-scenario/framework/clients/index.ts +++ b/test/e2e-scenario/framework/clients/index.ts @@ -7,6 +7,8 @@ export { HostCliClient } from "./host.ts"; export { ProviderClient, trustedProviderEndpoint, + type ProviderJsonRequestOptions, + type ProviderJsonResponse, type TrustedProviderEndpoint, } from "./provider.ts"; export { SandboxClient, validateSandboxName } from "./sandbox.ts"; diff --git a/test/e2e-scenario/framework/clients/provider.ts b/test/e2e-scenario/framework/clients/provider.ts index 07767418fb2..4d93cfdd777 100644 --- a/test/e2e-scenario/framework/clients/provider.ts +++ b/test/e2e-scenario/framework/clients/provider.ts @@ -25,6 +25,16 @@ export interface TrustedProviderEndpointOptions { allowedHosts?: readonly string[]; } +export interface ProviderJsonRequestOptions extends ShellProbeRunOptions { + readonly body?: string; + readonly headers?: readonly string[]; +} + +export interface ProviderJsonResponse { + readonly json: T; + readonly result: ShellProbeResult; +} + const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "::1"]); const BLOCKED_HOSTS = new Set(["169.254.169.254", "metadata.google.internal"]); @@ -158,12 +168,13 @@ export class ProviderClient { private curl( endpoint: TrustedProviderEndpoint, + args: readonly string[], options: ShellProbeRunOptions = {}, ): Promise { return this.runner.run( trustedShellCommand({ command: "curl", - args: ["-fsS", endpoint.url], + args: [...args, endpoint.url], reason: "fetch trusted provider endpoint", }), { @@ -174,16 +185,32 @@ export class ProviderClient { ); } - async getJson( + async requestJson( endpoint: TrustedProviderEndpoint, - options: ShellProbeRunOptions = {}, - ): Promise { - const result = await this.curl(endpoint, options); + options: ProviderJsonRequestOptions = {}, + ): Promise> { + const { body, headers, ...runOptions } = options; + const args = ["-fsS"]; + for (const header of headers ?? []) { + args.push("-H", header); + } + if (body !== undefined) { + args.push("-d", body); + } + const result = await this.curl(endpoint, args, runOptions); assertExitZero(result, `curl ${endpoint.logLabel}`); try { - return JSON.parse(result.stdout) as T; + return { json: JSON.parse(result.stdout) as T, result }; } catch { throw new Error("provider response was not JSON"); } } + + async getJson( + endpoint: TrustedProviderEndpoint, + options: ShellProbeRunOptions = {}, + ): Promise { + const response = await this.requestJson(endpoint, options); + return response.json; + } } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index 75891b38fa7..eaa7e4442b3 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -16,6 +16,7 @@ import { EnvironmentPhaseFixture, LifecyclePhaseFixture, OnboardingPhaseFixture, + RuntimePhaseFixture, StateValidationPhaseFixture, } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; @@ -34,6 +35,7 @@ export interface E2EScenarioFixtures { environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; lifecycle: LifecyclePhaseFixture; + runtime: RuntimePhaseFixture; stateValidation: StateValidationPhaseFixture; } @@ -96,6 +98,9 @@ export const test = base.extend({ lifecycle: async ({ cleanup, host, sandbox }, use) => { await use(new LifecyclePhaseFixture(host, sandbox, cleanup)); }, + runtime: async ({ provider, sandbox }, use) => { + await use(new RuntimePhaseFixture(sandbox, provider)); + }, stateValidation: async ({ host, gateway, sandbox }, use) => { await use(new StateValidationPhaseFixture(host, gateway, sandbox)); }, diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index 1905028a0c0..5a16f1e8cfd 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -22,6 +22,17 @@ export { type OnboardingOptions, type OnboardingSecrets, } from "./onboarding.ts"; +export { + inferenceRouteUrl, + RuntimePhaseFixture, + type InferenceRoute, + type InferenceRuntimeChatOptions, + type InferenceRuntimeProbeResult, + type InferenceRuntimeRequestOptions, + type InferenceRuntimeRouteOptions, + type InferenceRuntimeStatusOptions, + type ProviderRuntimeRequestOptions, +} from "./runtime.ts"; export { StateValidationPhaseFixture, type StateValidationProbeResult, diff --git a/test/e2e-scenario/framework/phases/runtime.ts b/test/e2e-scenario/framework/phases/runtime.ts new file mode 100644 index 00000000000..c4bd891ff6e --- /dev/null +++ b/test/e2e-scenario/framework/phases/runtime.ts @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { assertExitZero } from "../clients/command.ts"; +import type { + ProviderClient, + ProviderJsonRequestOptions, + SandboxClient, + TrustedProviderEndpoint, +} from "../clients/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +export type InferenceRoute = "inference-local" | "inference.local"; + +export interface InferenceRuntimeProbeResult { + readonly endpoint: string; + readonly result: ShellProbeResult; +} + +export interface InferenceRuntimeRequestOptions { + readonly artifactName?: string; + readonly curlMaxTimeSeconds?: number; + readonly headers?: readonly string[]; + readonly redactionValues?: readonly string[]; + readonly timeoutMs?: number; +} + +export interface InferenceRuntimeChatOptions extends InferenceRuntimeRequestOptions { + readonly maxTokens?: number; + readonly model?: string; + readonly prompt?: string; +} + +export interface InferenceRuntimeStatusOptions extends InferenceRuntimeRequestOptions { + readonly allowedStatusCodes?: readonly number[]; + readonly path?: string; + readonly route?: InferenceRoute; +} + +export interface InferenceRuntimeRouteOptions extends InferenceRuntimeRequestOptions { + readonly path?: string; + readonly route?: InferenceRoute; +} + +export interface ProviderRuntimeRequestOptions extends InferenceRuntimeRequestOptions { + readonly apiKey?: string; +} + +const DEFAULT_TIMEOUT_MS = 60_000; +const DEFAULT_CURL_MAX_TIME_SECONDS = 20; +const DEFAULT_CHAT_MODEL = "default"; +const DEFAULT_CHAT_PROMPT = "Say ok"; +const DEFAULT_CHAT_MAX_TOKENS = 8; +const MODELS_PATH = "/v1/models"; +const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; + +function inferenceHost(route: InferenceRoute = "inference-local"): string { + switch (route) { + case "inference-local": + case "inference.local": + return "inference.local"; + default: { + const _exhaustive: never = route; + throw new Error(`Unsupported inference route '${_exhaustive}'.`); + } + } +} + +function normalizePath(path: string): string { + if (!path.trim()) { + throw new Error("inference endpoint path is required"); + } + return path.startsWith("/") ? path : `/${path}`; +} + +export function inferenceRouteUrl( + route: InferenceRoute = "inference-local", + path = MODELS_PATH, +): string { + return `https://${inferenceHost(route)}${normalizePath(path)}`; +} + +function curlMaxTime(options: InferenceRuntimeRequestOptions): string { + return String(options.curlMaxTimeSeconds ?? DEFAULT_CURL_MAX_TIME_SECONDS); +} + +function shellOptions( + options: InferenceRuntimeRequestOptions, + artifactName: string, +): ShellProbeRunOptions { + return { + artifactName: options.artifactName ?? artifactName, + env: buildAvailabilityProbeEnv(), + redactionValues: [...(options.redactionValues ?? [])], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }; +} + +function headerArgs(headers: readonly string[] = []): string[] { + return headers.flatMap((header) => ["-H", header]); +} + +function parseHttpStatus(result: ShellProbeResult, label: string): number { + assertExitZero(result, label); + const status = Number(result.stdout.trim()); + if (!Number.isInteger(status) || status < 100 || status > 599) { + throw new Error(`${label} returned invalid HTTP status '${result.stdout.trim() || "empty"}'`); + } + return status; +} + +function openAiChatPayload(options: InferenceRuntimeChatOptions): string { + const model = options.model ?? DEFAULT_CHAT_MODEL; + const prompt = options.prompt ?? DEFAULT_CHAT_PROMPT; + if (!model.trim()) { + throw new Error("inference chat model is required"); + } + if (!prompt.trim()) { + throw new Error("inference chat prompt is required"); + } + return JSON.stringify({ + model, + messages: [{ role: "user", content: prompt }], + max_tokens: options.maxTokens ?? DEFAULT_CHAT_MAX_TOKENS, + }); +} + +function parseJsonBody(body: string, label: string): unknown { + try { + return JSON.parse(body); + } catch { + throw new Error(`${label} response was not JSON`); + } +} + +function hasChoiceContent(choice: unknown): boolean { + if (!choice || typeof choice !== "object") return false; + const message = (choice as { message?: unknown }).message; + if (message && typeof message === "object") { + const content = (message as { content?: unknown }).content; + if (typeof content === "string" && content.length > 0) return true; + } + const text = (choice as { text?: unknown }).text; + return typeof text === "string" && text.length > 0; +} + +function assertChatCompletionShape(json: unknown, label: string): void { + if (!json || typeof json !== "object") { + throw new Error(`${label} response was not an object`); + } + const choices = (json as { choices?: unknown }).choices; + if (!Array.isArray(choices) || choices.length === 0 || !choices.some(hasChoiceContent)) { + throw new Error(`${label} response missing choices/content`); + } +} + +function providerRequestOptions( + options: ProviderRuntimeRequestOptions, + body?: string, +): ProviderJsonRequestOptions { + const headers = [...(options.headers ?? [])]; + const redactionValues = [...(options.redactionValues ?? [])]; + if (body !== undefined) { + headers.unshift("Content-Type: application/json"); + } + if (options.apiKey) { + headers.push(`Authorization: Bearer ${options.apiKey}`); + redactionValues.push(options.apiKey); + } + return { + artifactName: options.artifactName, + body, + headers, + redactionValues, + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }; +} + +export class RuntimePhaseFixture { + constructor( + private readonly sandbox: SandboxClient, + private readonly provider: ProviderClient, + ) {} + + async expectInferenceLocalModels( + instance: NemoClawInstance, + options: InferenceRuntimeRouteOptions = {}, + ): Promise { + const endpoint = inferenceRouteUrl(options.route, options.path ?? MODELS_PATH); + const result = await this.sandbox.exec( + instance.sandboxName, + [ + "curl", + "-fsS", + "--max-time", + curlMaxTime(options), + ...headerArgs(options.headers), + endpoint, + ], + shellOptions(options, "runtime-inference-local-models"), + ); + assertExitZero(result, "inference.local models probe"); + if (result.stdout.trim().length === 0) { + throw new Error("inference.local models probe returned an empty response"); + } + return { endpoint, result }; + } + + async expectInferenceLocalChatCompletion( + instance: NemoClawInstance, + options: InferenceRuntimeChatOptions & { readonly route?: InferenceRoute } = {}, + ): Promise { + const endpoint = inferenceRouteUrl(options.route, CHAT_COMPLETIONS_PATH); + const payload = openAiChatPayload(options); + const result = await this.sandbox.exec( + instance.sandboxName, + [ + "curl", + "-fsS", + "--max-time", + curlMaxTime(options), + "-H", + "Content-Type: application/json", + ...headerArgs(options.headers), + "-d", + payload, + endpoint, + ], + shellOptions(options, "runtime-inference-local-chat-completion"), + ); + assertExitZero(result, "inference.local chat completion probe"); + assertChatCompletionShape( + parseJsonBody(result.stdout, "inference.local chat completion"), + "inference.local chat completion", + ); + return { endpoint, result }; + } + + async expectInferenceLocalStatus( + instance: NemoClawInstance, + options: InferenceRuntimeStatusOptions = {}, + ): Promise { + const allowedStatusCodes = options.allowedStatusCodes ?? [200]; + const endpoint = inferenceRouteUrl(options.route, options.path ?? MODELS_PATH); + const result = await this.sandbox.exec( + instance.sandboxName, + [ + "curl", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + curlMaxTime(options), + ...headerArgs(options.headers), + endpoint, + ], + shellOptions(options, "runtime-inference-local-status"), + ); + const status = parseHttpStatus(result, "inference.local status probe"); + if (!allowedStatusCodes.includes(status)) { + throw new Error( + `inference.local status probe returned HTTP ${status}; expected one of ${allowedStatusCodes.join(", ")}`, + ); + } + return { endpoint, result }; + } + + async expectProviderModels( + endpoint: TrustedProviderEndpoint, + options: ProviderRuntimeRequestOptions = {}, + ): Promise { + const response = await this.provider.requestJson(endpoint, providerRequestOptions(options)); + return { endpoint: endpoint.logLabel, result: response.result }; + } + + async expectProviderChatCompletion( + endpoint: TrustedProviderEndpoint, + options: ProviderRuntimeRequestOptions & InferenceRuntimeChatOptions = {}, + ): Promise { + const response = await this.provider.requestJson( + endpoint, + providerRequestOptions(options, openAiChatPayload(options)), + ); + assertChatCompletionShape(response.json, "provider chat completion"); + return { endpoint: endpoint.logLabel, result: response.result }; + } +} From 1ffe60846b8029059603ab514a579bb1966f07bf Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 15:54:48 -0700 Subject: [PATCH 14/18] test(e2e): validate inference model helper responses Signed-off-by: Carlos Villela --- .../framework-tests/e2e-phase-runtime.test.ts | 28 ++++++++++++++++++ test/e2e-scenario/framework/phases/runtime.ts | 29 +++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts index e3c9c71c3bd..8904233c585 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts @@ -129,6 +129,22 @@ describe("runtime phase fixture", () => { ]); }); + it("rejects inference.local model probes without compatible model data", async () => { + const invalidJson = new FakeRunner(); + invalidJson.enqueue(shellResult(0, "not-json")); + + await expect(fixture(invalidJson).expectInferenceLocalModels(instance())).rejects.toThrow( + "inference.local models response was not JSON", + ); + + const missingModels = new FakeRunner(); + missingModels.enqueue(shellResult(0, '{"error":"unavailable"}')); + + await expect(fixture(missingModels).expectInferenceLocalModels(instance())).rejects.toThrow( + "inference.local models response missing model data", + ); + }); + it("posts an OpenAI-compatible chat completion to inference.local without shell interpolation", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, JSON.stringify({ choices: [{ message: { content: "ok" } }] }))); @@ -242,6 +258,18 @@ describe("runtime phase fixture", () => { }); }); + it("rejects provider model probes without compatible model data", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, JSON.stringify({ error: "unavailable" }))); + const endpoint = trustedProviderEndpoint("https://api.example.test/v1/models", { + allowedHosts: ["api.example.test"], + }); + + await expect(fixture(runner).expectProviderModels(endpoint)).rejects.toThrow( + "provider models response missing model data", + ); + }); + it("fails chat probes on malformed compatible responses without echoing the body", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "not json with provider-secret")); diff --git a/test/e2e-scenario/framework/phases/runtime.ts b/test/e2e-scenario/framework/phases/runtime.ts index c4bd891ff6e..67f80d86e9a 100644 --- a/test/e2e-scenario/framework/phases/runtime.ts +++ b/test/e2e-scenario/framework/phases/runtime.ts @@ -156,6 +156,27 @@ function assertChatCompletionShape(json: unknown, label: string): void { } } +function hasModelIdentifier(entry: unknown): boolean { + if (typeof entry === "string") return entry.trim().length > 0; + if (!entry || typeof entry !== "object") return false; + for (const key of ["id", "model", "name"]) { + const value = (entry as Record)[key]; + if (typeof value === "string" && value.trim().length > 0) return true; + } + return false; +} + +function assertModelListShape(json: unknown, label: string): void { + if (!json || typeof json !== "object") { + throw new Error(`${label} response was not an object`); + } + const body = json as { data?: unknown; models?: unknown }; + const candidates = [body.data, body.models].filter(Array.isArray); + if (!candidates.some((items) => items.some(hasModelIdentifier))) { + throw new Error(`${label} response missing model data`); + } +} + function providerRequestOptions( options: ProviderRuntimeRequestOptions, body?: string, @@ -202,9 +223,10 @@ export class RuntimePhaseFixture { shellOptions(options, "runtime-inference-local-models"), ); assertExitZero(result, "inference.local models probe"); - if (result.stdout.trim().length === 0) { - throw new Error("inference.local models probe returned an empty response"); - } + assertModelListShape( + parseJsonBody(result.stdout, "inference.local models"), + "inference.local models", + ); return { endpoint, result }; } @@ -274,6 +296,7 @@ export class RuntimePhaseFixture { options: ProviderRuntimeRequestOptions = {}, ): Promise { const response = await this.provider.requestJson(endpoint, providerRequestOptions(options)); + assertModelListShape(response.json, "provider models"); return { endpoint: endpoint.logLabel, result: response.result }; } From b23308da49569b38fbdf1b10ca1405d984e1942e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 16:03:58 -0700 Subject: [PATCH 15/18] test(e2e): redact inference helper headers Signed-off-by: Carlos Villela --- .../framework-tests/e2e-phase-runtime.test.ts | 42 ++++++++++++++++++- .../framework/clients/provider.ts | 6 ++- test/e2e-scenario/framework/phases/runtime.ts | 31 +++++++++++++- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts index 8904233c585..31de0ac0104 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts @@ -214,7 +214,11 @@ describe("runtime phase fixture", () => { ], options: { artifactName: "runtime-inference-local-status", - redactionValues: ["local-proxy-token"], + redactionValues: expect.arrayContaining([ + "Authorization: Bearer local-proxy-token", + "Bearer local-proxy-token", + "local-proxy-token", + ]), timeoutMs: 60_000, }, }); @@ -238,6 +242,8 @@ describe("runtime phase fixture", () => { command: "curl", args: [ "-fsS", + "--max-time", + "20", "-H", "Content-Type: application/json", "-H", @@ -258,6 +264,40 @@ describe("runtime phase fixture", () => { }); }); + it("redacts sensitive custom headers and honors provider curl max time", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, JSON.stringify({ data: [{ id: "nvidia/model" }] }))); + const endpoint = trustedProviderEndpoint("https://api.example.test/v1/models", { + allowedHosts: ["api.example.test"], + }); + + await fixture(runner).expectProviderModels(endpoint, { + curlMaxTimeSeconds: 7, + headers: ["Authorization: Bearer custom-provider-token"], + }); + + expect(runner.calls[0]).toMatchObject({ + command: "curl", + args: [ + "-fsS", + "--max-time", + "7", + "-H", + "Authorization: Bearer custom-provider-token", + "https://api.example.test/v1/models", + ], + options: { + artifactName: "curl-https-api.example.test-v1-models", + redactionValues: expect.arrayContaining([ + "Authorization: Bearer custom-provider-token", + "Bearer custom-provider-token", + "custom-provider-token", + ]), + timeoutMs: 60_000, + }, + }); + }); + it("rejects provider model probes without compatible model data", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, JSON.stringify({ error: "unavailable" }))); diff --git a/test/e2e-scenario/framework/clients/provider.ts b/test/e2e-scenario/framework/clients/provider.ts index 4d93cfdd777..143a7ff9808 100644 --- a/test/e2e-scenario/framework/clients/provider.ts +++ b/test/e2e-scenario/framework/clients/provider.ts @@ -27,6 +27,7 @@ export interface TrustedProviderEndpointOptions { export interface ProviderJsonRequestOptions extends ShellProbeRunOptions { readonly body?: string; + readonly curlMaxTimeSeconds?: number; readonly headers?: readonly string[]; } @@ -189,8 +190,11 @@ export class ProviderClient { endpoint: TrustedProviderEndpoint, options: ProviderJsonRequestOptions = {}, ): Promise> { - const { body, headers, ...runOptions } = options; + const { body, curlMaxTimeSeconds, headers, ...runOptions } = options; const args = ["-fsS"]; + if (curlMaxTimeSeconds !== undefined) { + args.push("--max-time", String(curlMaxTimeSeconds)); + } for (const header of headers ?? []) { args.push("-H", header); } diff --git a/test/e2e-scenario/framework/phases/runtime.ts b/test/e2e-scenario/framework/phases/runtime.ts index 67f80d86e9a..67161b4cd78 100644 --- a/test/e2e-scenario/framework/phases/runtime.ts +++ b/test/e2e-scenario/framework/phases/runtime.ts @@ -55,6 +55,7 @@ const DEFAULT_CHAT_PROMPT = "Say ok"; const DEFAULT_CHAT_MAX_TOKENS = 8; const MODELS_PATH = "/v1/models"; const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; +const SENSITIVE_HEADER_NAME = /(authorization|api[-_]?key|token|secret|credential|password)/i; function inferenceHost(route: InferenceRoute = "inference-local"): string { switch (route) { @@ -93,7 +94,10 @@ function shellOptions( return { artifactName: options.artifactName ?? artifactName, env: buildAvailabilityProbeEnv(), - redactionValues: [...(options.redactionValues ?? [])], + redactionValues: uniqueRedactionValues([ + ...(options.redactionValues ?? []), + ...sensitiveHeaderRedactionValues(options.headers), + ]), timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, }; } @@ -102,6 +106,25 @@ function headerArgs(headers: readonly string[] = []): string[] { return headers.flatMap((header) => ["-H", header]); } +function sensitiveHeaderRedactionValues(headers: readonly string[] = []): string[] { + const values = new Set(); + for (const header of headers) { + const separator = header.indexOf(":"); + if (separator === -1) continue; + const name = header.slice(0, separator).trim(); + const value = header.slice(separator + 1).trim(); + if (!value || !SENSITIVE_HEADER_NAME.test(name)) continue; + values.add(header); + values.add(value); + values.add(value.replace(/^Bearer\s+/i, "").trim()); + } + return [...values].filter(Boolean); +} + +function uniqueRedactionValues(values: readonly string[]): string[] { + return [...new Set(values.filter(Boolean))]; +} + function parseHttpStatus(result: ShellProbeResult, label: string): number { assertExitZero(result, label); const status = Number(result.stdout.trim()); @@ -182,7 +205,10 @@ function providerRequestOptions( body?: string, ): ProviderJsonRequestOptions { const headers = [...(options.headers ?? [])]; - const redactionValues = [...(options.redactionValues ?? [])]; + const redactionValues = uniqueRedactionValues([ + ...(options.redactionValues ?? []), + ...sensitiveHeaderRedactionValues(headers), + ]); if (body !== undefined) { headers.unshift("Content-Type: application/json"); } @@ -193,6 +219,7 @@ function providerRequestOptions( return { artifactName: options.artifactName, body, + curlMaxTimeSeconds: options.curlMaxTimeSeconds ?? DEFAULT_CURL_MAX_TIME_SECONDS, headers, redactionValues, timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, From 05b6d1b267da063a128222fdfaa0ff323341499c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 16:30:26 -0700 Subject: [PATCH 16/18] test(e2e): harden provider runtime probes Signed-off-by: Carlos Villela --- .../framework-tests/e2e-clients.test.ts | 52 +++++++++++++++++++ .../framework-tests/e2e-phase-runtime.test.ts | 36 ++++++++++++- .../framework/clients/provider.ts | 30 +++++++++-- test/e2e-scenario/framework/phases/runtime.ts | 20 +++++-- 4 files changed, 130 insertions(+), 8 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index 4a8942a4a4b..77e8824fe22 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -218,6 +218,58 @@ describe("E2E fixture clients", () => { }); }); + it("provider client posts JSON bodies with --data-raw", async () => { + const runner = new FakeRunner(); + runner.stdout = JSON.stringify({ ok: true }); + const provider = new ProviderClient(runner); + const endpoint = trustedProviderEndpoint("https://api.example.test/v1/chat/completions", { + allowedHosts: ["api.example.test"], + }); + + await expect( + provider.requestJson(endpoint, { + body: '{"messages":[]}', + curlMaxTimeSeconds: 5, + headers: ["Content-Type: application/json"], + }), + ).resolves.toMatchObject({ json: { ok: true } }); + + expect(runner.calls[0]?.args).toEqual([ + "-fsS", + "--max-time", + "5", + "-H", + "Content-Type: application/json", + "--data-raw", + '{"messages":[]}', + "https://api.example.test/v1/chat/completions", + ]); + }); + + it("provider client rejects curl-sensitive request options before command construction", async () => { + const endpoint = trustedProviderEndpoint("https://api.example.test/v1/models", { + allowedHosts: ["api.example.test"], + }); + + for (const options of [ + { body: "@/etc/passwd" }, + { headers: ["@/tmp/headers"] }, + { headers: ["Authorization: Bearer token\nX-Leak: value"] }, + { curlMaxTimeSeconds: 0 }, + { curlMaxTimeSeconds: -1 }, + { curlMaxTimeSeconds: Number.NaN }, + { curlMaxTimeSeconds: Number.POSITIVE_INFINITY }, + ]) { + const runner = new FakeRunner(); + const provider = new ProviderClient(runner); + + await expect(provider.requestJson(endpoint, options)).rejects.toThrow( + /@file|CR or LF|finite positive/, + ); + expect(runner.calls).toEqual([]); + } + }); + it("provider client does not follow redirects after endpoint validation", async () => { const runner = new FakeRunner(); runner.stdout = JSON.stringify({ ok: true }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts index 31de0ac0104..926c54b5fe7 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts @@ -129,6 +129,15 @@ describe("runtime phase fixture", () => { ]); }); + it("accepts Ollama-style inference.local model lists", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, '{"models":[{"name":"llama3"}]}')); + + await expect(fixture(runner).expectInferenceLocalModels(instance())).resolves.toMatchObject({ + endpoint: "https://inference.local/v1/models", + }); + }); + it("rejects inference.local model probes without compatible model data", async () => { const invalidJson = new FakeRunner(); invalidJson.enqueue(shellResult(0, "not-json")); @@ -169,7 +178,7 @@ describe("runtime phase fixture", () => { "20", "-H", "Content-Type: application/json", - "-d", + "--data-raw", expect.any(String), "https://inference.local/v1/chat/completions", ]); @@ -248,7 +257,7 @@ describe("runtime phase fixture", () => { "Content-Type: application/json", "-H", "Authorization: Bearer provider-secret", - "-d", + "--data-raw", JSON.stringify({ model: "nvidia/model", messages: [{ role: "user", content: "Reply with pong" }], @@ -264,6 +273,18 @@ describe("runtime phase fixture", () => { }); }); + it("accepts Ollama-style compatible provider model lists", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, JSON.stringify({ models: ["llama3"] }))); + const endpoint = trustedProviderEndpoint("https://api.example.test/v1/models", { + allowedHosts: ["api.example.test"], + }); + + await expect(fixture(runner).expectProviderModels(endpoint)).resolves.toMatchObject({ + endpoint: "https://api.example.test/v1/models", + }); + }); + it("redacts sensitive custom headers and honors provider curl max time", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, JSON.stringify({ data: [{ id: "nvidia/model" }] }))); @@ -298,6 +319,17 @@ describe("runtime phase fixture", () => { }); }); + it("rejects invalid curl max time values before runtime probe execution", async () => { + for (const curlMaxTimeSeconds of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + const runner = new FakeRunner(); + + await expect( + fixture(runner).expectInferenceLocalModels(instance(), { curlMaxTimeSeconds }), + ).rejects.toThrow("inference request curlMaxTimeSeconds must be a finite positive number"); + expect(runner.calls).toEqual([]); + } + }); + it("rejects provider model probes without compatible model data", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, JSON.stringify({ error: "unavailable" }))); diff --git a/test/e2e-scenario/framework/clients/provider.ts b/test/e2e-scenario/framework/clients/provider.ts index 143a7ff9808..257c7c8b6ac 100644 --- a/test/e2e-scenario/framework/clients/provider.ts +++ b/test/e2e-scenario/framework/clients/provider.ts @@ -39,6 +39,30 @@ export interface ProviderJsonResponse { const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "::1"]); const BLOCKED_HOSTS = new Set(["169.254.169.254", "metadata.google.internal"]); +function validateCurlMaxTimeSeconds(value: number): string { + if (!Number.isFinite(value) || value <= 0) { + throw new Error("provider request curlMaxTimeSeconds must be a finite positive number"); + } + return String(value); +} + +function validateCurlHeader(header: string): string { + if (/[\r\n]/.test(header)) { + throw new Error("provider request header must not contain CR or LF"); + } + if (header.trimStart().startsWith("@")) { + throw new Error("provider request header must not use curl @file syntax"); + } + return header; +} + +function validateCurlBody(body: string): string { + if (body.trimStart().startsWith("@")) { + throw new Error("provider request body must not use curl @file syntax"); + } + return body; +} + function queryRedactionValues(url: URL): string[] { const values = new Set(); if (url.search) { @@ -193,13 +217,13 @@ export class ProviderClient { const { body, curlMaxTimeSeconds, headers, ...runOptions } = options; const args = ["-fsS"]; if (curlMaxTimeSeconds !== undefined) { - args.push("--max-time", String(curlMaxTimeSeconds)); + args.push("--max-time", validateCurlMaxTimeSeconds(curlMaxTimeSeconds)); } for (const header of headers ?? []) { - args.push("-H", header); + args.push("-H", validateCurlHeader(header)); } if (body !== undefined) { - args.push("-d", body); + args.push("--data-raw", validateCurlBody(body)); } const result = await this.curl(endpoint, args, runOptions); assertExitZero(result, `curl ${endpoint.logLabel}`); diff --git a/test/e2e-scenario/framework/phases/runtime.ts b/test/e2e-scenario/framework/phases/runtime.ts index 67161b4cd78..e417d30c1ea 100644 --- a/test/e2e-scenario/framework/phases/runtime.ts +++ b/test/e2e-scenario/framework/phases/runtime.ts @@ -84,7 +84,11 @@ export function inferenceRouteUrl( } function curlMaxTime(options: InferenceRuntimeRequestOptions): string { - return String(options.curlMaxTimeSeconds ?? DEFAULT_CURL_MAX_TIME_SECONDS); + const seconds = options.curlMaxTimeSeconds ?? DEFAULT_CURL_MAX_TIME_SECONDS; + if (!Number.isFinite(seconds) || seconds <= 0) { + throw new Error("inference request curlMaxTimeSeconds must be a finite positive number"); + } + return String(seconds); } function shellOptions( @@ -103,7 +107,17 @@ function shellOptions( } function headerArgs(headers: readonly string[] = []): string[] { - return headers.flatMap((header) => ["-H", header]); + return headers.flatMap((header) => ["-H", validatedCurlHeader(header)]); +} + +function validatedCurlHeader(header: string): string { + if (/[\r\n]/.test(header)) { + throw new Error("inference request header must not contain CR or LF"); + } + if (header.trimStart().startsWith("@")) { + throw new Error("inference request header must not use curl @file syntax"); + } + return header; } function sensitiveHeaderRedactionValues(headers: readonly string[] = []): string[] { @@ -273,7 +287,7 @@ export class RuntimePhaseFixture { "-H", "Content-Type: application/json", ...headerArgs(options.headers), - "-d", + "--data-raw", payload, endpoint, ], From 1d3c444c02ff04b56b4c06085e23a8c3984cf0e5 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 16:50:14 -0700 Subject: [PATCH 17/18] test(e2e): run cloud inference suites in Vitest Signed-off-by: Carlos Villela --- .../e2e-live-registry-discovery.test.ts | 6 +- .../framework-tests/e2e-phase-runtime.test.ts | 48 +++++++ .../e2e-scenario-matrix.test.ts | 3 +- .../e2e-scenario-registry.test.ts | 2 +- test/e2e-scenario/framework/phases/index.ts | 5 + test/e2e-scenario/framework/phases/runtime.ts | 122 +++++++++++++++++- .../live/registry-scenarios.test.ts | 11 +- .../migration/legacy-inventory.json | 12 +- .../scenarios/assertions/registry.ts | 20 ++- test/e2e-scenario/scenarios/run.ts | 2 + .../e2e-scenario/scenarios/runtime-support.ts | 9 +- .../scenarios/scenarios/baseline.ts | 2 +- 12 files changed, 226 insertions(+), 16 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 7158039d2c8..f394eaf30a0 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -22,11 +22,11 @@ describe("live Vitest registry discovery support", () => { expect(scenario).toBeTruthy(); expect(liveScenarioSupport(scenario!).supported).toBe(true); - expect(liveScenarioSupport(scenario!).pendingRuntimeSuites).toEqual([ - "smoke", + expect(liveScenarioSupport(scenario!).runtimeSuites).toEqual([ "inference", - "credentials", + "inference-routing", ]); + expect(liveScenarioSupport(scenario!).pendingRuntimeSuites).toEqual(["smoke", "credentials"]); }); it("keeps unsupported onboarding profiles skipped with a concrete reason", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts index 926c54b5fe7..651f0e2d79e 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-runtime.test.ts @@ -233,6 +233,54 @@ describe("runtime phase fixture", () => { }); }); + it("runs the cloud inference suite with stable assertion IDs", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, JSON.stringify({ data: [{ id: "nvidia/model" }] }))); + runner.enqueue(shellResult(0, JSON.stringify({ choices: [{ message: { content: "PONG" } }] }))); + runner.enqueue(shellResult(0, JSON.stringify({ data: [{ id: "nvidia/model" }] }))); + + const result = await fixture(runner).runSuite("inference", instance()); + + expect(result.suiteId).toBe("inference"); + expect(result.assertions.map((assertion) => assertion.id)).toEqual([ + "runtime.inference.models-health", + "runtime.inference.chat-completion", + "runtime.inference.sandbox-local", + ]); + expect(runner.calls.map((call) => call.options?.artifactName)).toEqual([ + "runtime-inference-models-health", + "runtime-inference-chat-completion", + "runtime-inference-sandbox-local", + ]); + }); + + it("runs the inference-routing suite with route health and chat assertions", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "200")); + runner.enqueue(shellResult(0, JSON.stringify({ choices: [{ message: { content: "PONG" } }] }))); + + const result = await fixture(runner).runSuite("inference-routing", instance()); + + expect(result.suiteId).toBe("inference-routing"); + expect(result.assertions.map((assertion) => assertion.id)).toEqual([ + "runtime.inference-routing.provider-route-health", + "runtime.inference-routing.inference-local-chat-completion", + ]); + expect(runner.calls.map((call) => call.options?.artifactName)).toEqual([ + "runtime-inference-routing-provider-route-health", + "runtime-inference-routing-inference-local-chat-completion", + ]); + }); + + it("rejects unwired runtime suites before command execution", async () => { + const runner = new FakeRunner(); + + await expect(fixture(runner).runSuite("credentials", instance())).rejects.toThrow( + "runtime suite 'credentials' is not wired for RuntimePhaseFixture", + ); + expect(runner.calls).toEqual([]); + }); + it("calls a trusted compatible provider endpoint with request artifacts and redaction", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, JSON.stringify({ choices: [{ message: { content: "pong" } }] }))); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index 78eb216d53d..34e684bf83a 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -144,7 +144,8 @@ describe("typed scenario matrix", () => { requiredSecrets: ["NVIDIA_API_KEY"], supported: true, supportReasons: [], - pendingRuntimeSuites: ["smoke", "inference", "credentials"], + runtimeSuites: ["inference", "inference-routing"], + pendingRuntimeSuites: ["smoke", "credentials"], }); // Failing-test-first guard for #4423. Pinned in the matrix to // confirm the lifecycle whitelist + post-reboot-recovery scenario diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts index 81d8f6b3a8e..053938f1d35 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts @@ -88,7 +88,7 @@ describe("deterministic scenario registry", () => { onboarding: "cloud-openclaw", }); expect(plan.expectedStateId).toBe("cloud-openclaw-ready"); - expect(plan.suiteIds).toEqual(["smoke", "inference", "credentials"]); + expect(plan.suiteIds).toEqual(["smoke", "inference", "inference-routing", "credentials"]); expect(plan.onboardingAssertionIds).toEqual(["base-installed", "preflight-passed"]); }); }); diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index 5a16f1e8cfd..e05f4544287 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -24,7 +24,9 @@ export { } from "./onboarding.ts"; export { inferenceRouteUrl, + isRuntimeSuiteSupported, RuntimePhaseFixture, + SUPPORTED_RUNTIME_SUITE_IDS, type InferenceRoute, type InferenceRuntimeChatOptions, type InferenceRuntimeProbeResult, @@ -32,6 +34,9 @@ export { type InferenceRuntimeRouteOptions, type InferenceRuntimeStatusOptions, type ProviderRuntimeRequestOptions, + type RuntimeSuiteAssertionResult, + type RuntimeSuiteId, + type RuntimeSuiteResult, } from "./runtime.ts"; export { StateValidationPhaseFixture, diff --git a/test/e2e-scenario/framework/phases/runtime.ts b/test/e2e-scenario/framework/phases/runtime.ts index e417d30c1ea..ccc0a1add27 100644 --- a/test/e2e-scenario/framework/phases/runtime.ts +++ b/test/e2e-scenario/framework/phases/runtime.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { buildAvailabilityProbeEnv } from "../availability-env.ts"; -import { assertExitZero } from "../clients/command.ts"; +import { artifactLabel, assertExitZero } from "../clients/command.ts"; import type { ProviderClient, ProviderJsonRequestOptions, @@ -19,6 +19,23 @@ export interface InferenceRuntimeProbeResult { readonly result: ShellProbeResult; } +export const SUPPORTED_RUNTIME_SUITE_IDS = [ + "inference", + "cloud-inference", + "inference-routing", +] as const; + +export type RuntimeSuiteId = (typeof SUPPORTED_RUNTIME_SUITE_IDS)[number]; + +export interface RuntimeSuiteAssertionResult extends InferenceRuntimeProbeResult { + readonly id: string; +} + +export interface RuntimeSuiteResult { + readonly suiteId: RuntimeSuiteId; + readonly assertions: RuntimeSuiteAssertionResult[]; +} + export interface InferenceRuntimeRequestOptions { readonly artifactName?: string; readonly curlMaxTimeSeconds?: number; @@ -53,6 +70,9 @@ const DEFAULT_CURL_MAX_TIME_SECONDS = 20; const DEFAULT_CHAT_MODEL = "default"; const DEFAULT_CHAT_PROMPT = "Say ok"; const DEFAULT_CHAT_MAX_TOKENS = 8; +const CLOUD_INFERENCE_CHAT_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const CLOUD_INFERENCE_CHAT_PROMPT = "Reply with exactly one word: PONG"; +const CLOUD_INFERENCE_CHAT_MAX_TOKENS = 100; const MODELS_PATH = "/v1/models"; const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; const SENSITIVE_HEADER_NAME = /(authorization|api[-_]?key|token|secret|credential|password)/i; @@ -240,12 +260,112 @@ function providerRequestOptions( }; } +export function isRuntimeSuiteSupported(suiteId: string): suiteId is RuntimeSuiteId { + return (SUPPORTED_RUNTIME_SUITE_IDS as readonly string[]).includes(suiteId); +} + +function suiteArtifactName(suiteId: string, assertion: string): string { + return `runtime-${artifactLabel(suiteId)}-${artifactLabel(assertion)}`; +} + +function assertionResult( + id: string, + probe: InferenceRuntimeProbeResult, +): RuntimeSuiteAssertionResult { + return { id, endpoint: probe.endpoint, result: probe.result }; +} + export class RuntimePhaseFixture { constructor( private readonly sandbox: SandboxClient, private readonly provider: ProviderClient, ) {} + async runSuite(suiteId: string, instance: NemoClawInstance): Promise { + switch (suiteId) { + case "inference": + case "cloud-inference": + return await this.runCloudInferenceSuite(suiteId, instance); + case "inference-routing": + return await this.runInferenceRoutingSuite(instance); + default: + throw new Error(`runtime suite '${suiteId}' is not wired for RuntimePhaseFixture`); + } + } + + private async runCloudInferenceSuite( + suiteId: "inference" | "cloud-inference", + instance: NemoClawInstance, + ): Promise { + const assertions: RuntimeSuiteAssertionResult[] = []; + assertions.push( + assertionResult( + "runtime.inference.models-health", + await this.expectInferenceLocalModels(instance, { + artifactName: suiteArtifactName(suiteId, "models-health"), + curlMaxTimeSeconds: 20, + timeoutMs: 30_000, + }), + ), + ); + assertions.push( + assertionResult( + "runtime.inference.chat-completion", + await this.expectInferenceLocalChatCompletion(instance, { + artifactName: suiteArtifactName(suiteId, "chat-completion"), + curlMaxTimeSeconds: 40, + maxTokens: CLOUD_INFERENCE_CHAT_MAX_TOKENS, + model: CLOUD_INFERENCE_CHAT_MODEL, + prompt: CLOUD_INFERENCE_CHAT_PROMPT, + timeoutMs: 50_000, + }), + ), + ); + assertions.push( + assertionResult( + "runtime.inference.sandbox-local", + await this.expectInferenceLocalModels(instance, { + artifactName: suiteArtifactName(suiteId, "sandbox-local"), + curlMaxTimeSeconds: 25, + route: "inference-local", + timeoutMs: 35_000, + }), + ), + ); + return { suiteId, assertions }; + } + + private async runInferenceRoutingSuite(instance: NemoClawInstance): Promise { + const suiteId = "inference-routing"; + const assertions: RuntimeSuiteAssertionResult[] = []; + assertions.push( + assertionResult( + "runtime.inference-routing.provider-route-health", + await this.expectInferenceLocalStatus(instance, { + artifactName: suiteArtifactName(suiteId, "provider-route-health"), + curlMaxTimeSeconds: 20, + route: "inference-local", + timeoutMs: 30_000, + }), + ), + ); + assertions.push( + assertionResult( + "runtime.inference-routing.inference-local-chat-completion", + await this.expectInferenceLocalChatCompletion(instance, { + artifactName: suiteArtifactName(suiteId, "inference-local-chat-completion"), + curlMaxTimeSeconds: 40, + maxTokens: CLOUD_INFERENCE_CHAT_MAX_TOKENS, + model: CLOUD_INFERENCE_CHAT_MODEL, + prompt: CLOUD_INFERENCE_CHAT_PROMPT, + route: "inference-local", + timeoutMs: 50_000, + }), + ), + ); + return { suiteId, assertions }; + } + async expectInferenceLocalModels( instance: NemoClawInstance, options: InferenceRuntimeRouteOptions = {}, diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 3395ac553a8..bc5c7ba6a08 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -37,7 +37,7 @@ for (const scenario of listScenarios()) { test( liveScenarioTestName(scenario), - async ({ artifacts, environment, lifecycle, onboard, secrets, stateValidation }) => { + async ({ artifacts, environment, lifecycle, onboard, runtime, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } @@ -57,6 +57,7 @@ for (const scenario of listScenarios()) { id: scenario.id, runner: "vitest", boundary: "typed-registry", + runtimeSuites: support.runtimeSuites, pendingRuntimeSuites: support.pendingRuntimeSuites, }); @@ -84,11 +85,19 @@ for (const scenario of listScenarios()) { } const validation = await stateValidation.from(scenario.expectedStateId, instance); + const runtimeResults = []; + for (const suiteId of support.runtimeSuites) { + runtimeResults.push(await runtime.runSuite(suiteId, instance)); + } await artifacts.writeJson("scenario-result.json", { id: scenario.id, expectedStateId: validation.state.id, probes: validation.probes.map((probe) => probe.id), + runtimeSuites: runtimeResults.map((suite) => ({ + id: suite.suiteId, + assertions: suite.assertions.map((assertion) => assertion.id), + })), pendingRuntimeSuites: support.pendingRuntimeSuites, lifecycle: lifecycleResult ? { profile: lifecycleResult.profile, steps: lifecycleResult.steps.map((s) => s.id) } diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index e102478d913..a8c69134f44 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -39,12 +39,12 @@ "legacyScript": "test/e2e/test-inference-routing.sh", "domain": "inference", "ownerIssue": "#4349", - "status": "not-migrated", - "targetVitestScenarios": [], + "status": "covered", + "targetVitestScenarios": ["test/e2e-scenario/live/registry-scenarios.test.ts"], "bridgeProbes": [], "retiredReason": "", "deletionReady": false, - "notes": "Provider routing and inference.local checks need typed inference/provider fixtures before deletion." + "notes": "Covered by the ubuntu-repo-cloud-openclaw live Vitest scenario via RuntimePhaseFixture suite 'inference-routing'. Do not delete until #4357 records cutover approval and downstream shell entrypoints are retired together." }, { "legacyScript": "test/e2e/test-openclaw-inference-switch.sh", @@ -193,12 +193,12 @@ "legacyScript": "test/e2e/test-cloud-inference-e2e.sh", "domain": "inference", "ownerIssue": "#4349", - "status": "not-migrated", - "targetVitestScenarios": [], + "status": "covered", + "targetVitestScenarios": ["test/e2e-scenario/live/registry-scenarios.test.ts"], "bridgeProbes": [], "retiredReason": "", "deletionReady": false, - "notes": "Initial completeness row; classify detailed coverage and deletion evidence in the owning migration issue before deleting." + "notes": "Covered by the ubuntu-repo-cloud-openclaw live Vitest scenario via RuntimePhaseFixture suite 'inference'. Do not delete until #4357 records cutover approval and downstream shell entrypoints are retired together." }, { "legacyScript": "test/e2e/test-common-egress-agent-e2e.sh", diff --git a/test/e2e-scenario/scenarios/assertions/registry.ts b/test/e2e-scenario/scenarios/assertions/registry.ts index 209dd1f1e0f..e27e5fd2c67 100644 --- a/test/e2e-scenario/scenarios/assertions/registry.ts +++ b/test/e2e-scenario/scenarios/assertions/registry.ts @@ -170,6 +170,24 @@ const cloudInferenceSteps = [ }), ]; +const inferenceRoutingSteps = [ + shellStep({ + id: "runtime.inference-routing.inference-local-chat-completion", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/routing/00-inference-local-chat-completion.sh", + reliability: { + timeoutSeconds: 60, + retry: { attempts: 2, on: ["provider-transient", "model-toolcall-transient"] }, + }, + }), + shellStep({ + id: "runtime.inference-routing.provider-route-health", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/routing/01-provider-route-health.sh", + reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["gateway-transient"] } }, + }), +]; + const credentialsSteps = [ shellStep({ id: "security.credentials.present", @@ -283,7 +301,7 @@ export const validationSuiteGroups: AssertionGroup[] = [ }), ]), suiteGroup("openai-compatible-inference", cloudInferenceSteps), - suiteGroup("inference-routing", cloudInferenceSteps), + suiteGroup("inference-routing", inferenceRoutingSteps), suiteGroup("inference-switch", cloudInferenceSteps), suiteGroup("kimi-compatibility", [ shellStep({ diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts index 7fa79543395..30bcf8a7880 100644 --- a/test/e2e-scenario/scenarios/run.ts +++ b/test/e2e-scenario/scenarios/run.ts @@ -41,6 +41,7 @@ export interface LiveScenarioMatrixEntry extends ScenarioMatrixEntry { requiredSecrets: string[]; supported: boolean; supportReasons: string[]; + runtimeSuites: string[]; pendingRuntimeSuites: string[]; } @@ -145,6 +146,7 @@ function liveMatrixEntry( requiredSecrets: scenario.requiredSecrets ?? [], supported: support.supported, supportReasons: support.reasons, + runtimeSuites: support.runtimeSuites, pendingRuntimeSuites: support.pendingRuntimeSuites, }; } diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 6a9f874264d..e07c166c3f4 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { ScenarioDefinition } from "./types.ts"; +import { isRuntimeSuiteSupported, type RuntimeSuiteId } from "../framework/phases/index.ts"; const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); @@ -17,6 +18,7 @@ const SUPPORTED_LIFECYCLES = new Set(["post-reboot-recovery"]); export interface LiveScenarioSupport { supported: boolean; reasons: string[]; + runtimeSuites: RuntimeSuiteId[]; pendingRuntimeSuites: string[]; } @@ -57,9 +59,14 @@ export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioS reasons.push("missing expectedStateId"); } + const suiteIds = scenario.suiteIds ?? []; + const runtimeSuites = suiteIds.filter(isRuntimeSuiteSupported); + const pendingRuntimeSuites = suiteIds.filter((suiteId) => !isRuntimeSuiteSupported(suiteId)); + return { supported: reasons.length === 0, reasons, - pendingRuntimeSuites: scenario.suiteIds ?? [], + runtimeSuites, + pendingRuntimeSuites, }; } diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index 4e5befad679..9336bedc4d4 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -68,7 +68,7 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ manifestName: "openclaw-nvidia", environment: ubuntuRepoDocker("cloud-openclaw"), expectedStateId: "cloud-openclaw-ready", - suiteIds: ["smoke", "inference", "credentials"], + suiteIds: ["smoke", "inference", "inference-routing", "credentials"], description: "Ubuntu repo checkout with Docker and cloud OpenClaw onboarding.", requiredSecrets: ["NVIDIA_API_KEY"], }, From b467cb89e0bfc0b784bff2e96db1450598bd8a91 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 16:58:05 -0700 Subject: [PATCH 18/18] test(e2e): narrow inference migration inventory status Signed-off-by: Carlos Villela --- .../e2e-scenario/migration/legacy-inventory.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index a8c69134f44..32c8fc501b8 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -39,12 +39,12 @@ "legacyScript": "test/e2e/test-inference-routing.sh", "domain": "inference", "ownerIssue": "#4349", - "status": "covered", - "targetVitestScenarios": ["test/e2e-scenario/live/registry-scenarios.test.ts"], - "bridgeProbes": [], + "status": "bridge-probe", + "targetVitestScenarios": [], + "bridgeProbes": ["test/e2e-scenario/live/registry-scenarios.test.ts"], "retiredReason": "", "deletionReady": false, - "notes": "Covered by the ubuntu-repo-cloud-openclaw live Vitest scenario via RuntimePhaseFixture suite 'inference-routing'. Do not delete until #4357 records cutover approval and downstream shell entrypoints are retired together." + "notes": "Partially represented by the ubuntu-repo-cloud-openclaw live Vitest scenario via RuntimePhaseFixture suite 'inference-routing'. Keep the legacy script until credential isolation, negative classification, cleanup, provider-route, and compatible-endpoint coverage migrate." }, { "legacyScript": "test/e2e/test-openclaw-inference-switch.sh", @@ -193,12 +193,12 @@ "legacyScript": "test/e2e/test-cloud-inference-e2e.sh", "domain": "inference", "ownerIssue": "#4349", - "status": "covered", - "targetVitestScenarios": ["test/e2e-scenario/live/registry-scenarios.test.ts"], - "bridgeProbes": [], + "status": "bridge-probe", + "targetVitestScenarios": [], + "bridgeProbes": ["test/e2e-scenario/live/registry-scenarios.test.ts"], "retiredReason": "", "deletionReady": false, - "notes": "Covered by the ubuntu-repo-cloud-openclaw live Vitest scenario via RuntimePhaseFixture suite 'inference'. Do not delete until #4357 records cutover approval and downstream shell entrypoints are retired together." + "notes": "Partially represented by the ubuntu-repo-cloud-openclaw live Vitest scenario via RuntimePhaseFixture suite 'inference'. Keep the legacy script until repo-skill and sandbox skill filesystem validation migrate." }, { "legacyScript": "test/e2e/test-common-egress-agent-e2e.sh",