From 32ad96a3b52acc4d70d61307b37523a53d4e6816 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:39:02 -0700 Subject: [PATCH 1/7] test(e2e): add environment phase fixture --- .../framework-tests/e2e-clients.test.ts | 2 +- .../e2e-phase-environment.test.ts | 181 ++++++++++++++++++ test/e2e-scenario/framework/clients/host.ts | 6 +- test/e2e-scenario/framework/e2e-test.ts | 5 + .../framework/phases/environment.ts | 107 +++++++++++ test/e2e-scenario/framework/phases/index.ts | 9 + 6 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts create mode 100644 test/e2e-scenario/framework/phases/environment.ts create mode 100644 test/e2e-scenario/framework/phases/index.ts diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index 25c893d5f1a..4d5c3a72dbf 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -61,7 +61,7 @@ describe("E2E fixture clients", () => { { command: "./bin/nemoclaw.js", args: ["--version"], - options: { artifactName: "nemoclaw-version" }, + options: { artifactName: "nemoclaw-version", inheritEnv: true }, }, ]); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts new file mode 100644 index 00000000000..cf41f3983e8 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { EnvironmentPhaseFixture, type DockerRuntimeReady } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; +import type { ScenarioEnvironment } from "../scenarios/types.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: output, + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: Array = []; + + enqueue(response: ShellProbeResult | Error): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift() ?? shellResult(0); + if (response instanceof Error) { + throw response; + } + return response; + } +} + +const cloudOpenClawEnvironment: ScenarioEnvironment = { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", +}; + +describe("environment phase fixture", () => { + it("asserts the current repo CLI and required Docker runtime", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "Docker is available\n")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner, { cliPath: "./bin/nemoclaw.js" })); + + const ready = await environment.assertReady(cloudOpenClawEnvironment); + + expect(ready).toMatchObject({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + cliPath: "./bin/nemoclaw.js", + docker: { + id: "docker-running", + expectation: "required", + available: true, + } satisfies Partial, + }); + expect(runner.calls).toEqual([ + { + command: "./bin/nemoclaw.js", + args: ["--version"], + options: { artifactName: "nemoclaw-version", inheritEnv: true }, + }, + { + command: "docker", + args: ["info"], + options: { + artifactName: "runtime-docker-info-docker-running", + inheritEnv: true, + timeoutMs: 30_000, + }, + }, + ]); + }); + + it("fails when a required Docker runtime is unavailable", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + await expect(environment.assertReady(cloudOpenClawEnvironment)).rejects.toThrow( + /docker runtime docker-running failed: Cannot connect/, + ); + }); + + it("accepts an unavailable Docker runtime for no-Docker negative scenarios", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "docker intentionally unavailable")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + }); + + expect(ready.docker).toMatchObject({ + id: "docker-missing", + expectation: "missing", + available: false, + }); + }); + + it("fails if a no-Docker negative scenario unexpectedly has Docker", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "Docker is available\n")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + await expect( + environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + }), + ).rejects.toThrow(/expected Docker to be unavailable/); + }); + + it("records optional Docker as unavailable without failing", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(new Error("spawn docker ENOENT")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + platform: "macos-local", + runtime: "macos-docker-optional", + }); + + expect(ready.docker).toMatchObject({ + id: "macos-docker-optional", + expectation: "optional", + available: false, + probeError: "spawn docker ENOENT", + }); + }); + + it("rejects unsupported install and runtime IDs", async () => { + const runner = new FakeRunner(); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + await expect(environment.assertReady({ ...cloudOpenClawEnvironment, install: "tarball" })).rejects.toThrow( + /Unsupported scenario install 'tarball'/, + ); + expect(runner.calls).toEqual([]); + + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + await expect(environment.assertReady({ ...cloudOpenClawEnvironment, runtime: "podman-running" })).rejects.toThrow( + /Unsupported scenario runtime 'podman-running'/, + ); + }); + + it("exposes the environment phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/clients/host.ts b/test/e2e-scenario/framework/clients/host.ts index 9e0ee80ebd3..8c5ba1dd9ac 100644 --- a/test/e2e-scenario/framework/clients/host.ts +++ b/test/e2e-scenario/framework/clients/host.ts @@ -21,6 +21,10 @@ export class HostCliClient { this.cwd = options.cwd; } + get commandPath(): string { + return this.cliPath; + } + command(command: string, args: string[] = [], options: ShellProbeRunOptions = {}): Promise { const merged: ShellProbeRunOptions = { ...options }; if (this.cwd && !merged.cwd) { @@ -44,7 +48,7 @@ export class HostCliClient { } async expectNemoclawAvailable(): Promise { - const result = await this.nemoclaw(["--version"], { artifactName: "nemoclaw-version" }); + const result = await this.nemoclaw(["--version"], { artifactName: "nemoclaw-version", inheritEnv: true }); assertExitZero(result, "nemoclaw --version"); return result; } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index e44f50c7f27..6d122b16b1a 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,6 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; +import { EnvironmentPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -25,6 +26,7 @@ export interface E2EScenarioFixtures { sandbox: SandboxClient; provider: ProviderClient; state: StateClient; + environment: EnvironmentPhaseFixture; } export const test = base.extend({ @@ -77,6 +79,9 @@ export const test = base.extend({ state: async ({}, use) => { await use(new StateClient()); }, + environment: async ({ host }, use) => { + await use(new EnvironmentPhaseFixture(host)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/environment.ts b/test/e2e-scenario/framework/phases/environment.ts new file mode 100644 index 00000000000..54cad8d6d60 --- /dev/null +++ b/test/e2e-scenario/framework/phases/environment.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { artifactLabel, assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { ScenarioEnvironment } from "../../scenarios/types.ts"; + +const SUPPORTED_INSTALLS = new Set(["repo-current", "launchable"]); + +const DOCKER_RUNTIME_EXPECTATIONS = { + "docker-running": "required", + "gpu-docker-cdi": "required", + "docker-missing": "missing", + "macos-docker-optional": "optional", +} as const; + +export type DockerRuntimeExpectation = + (typeof DOCKER_RUNTIME_EXPECTATIONS)[keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; + +export interface DockerRuntimeReady { + id: string; + expectation: DockerRuntimeExpectation; + available: boolean; + result?: ShellProbeResult; + probeError?: string; +} + +export interface EnvironmentReady extends ScenarioEnvironment { + cliPath: string; + docker: DockerRuntimeReady; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function supportedRuntime(runtime: string): DockerRuntimeExpectation { + const expectation = DOCKER_RUNTIME_EXPECTATIONS[runtime as keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; + if (!expectation) { + throw new Error(`Unsupported scenario runtime '${runtime}'.`); + } + return expectation; +} + +export class EnvironmentPhaseFixture { + constructor(private readonly host: HostCliClient) {} + + async assertReady(environment: ScenarioEnvironment): Promise { + await this.assertInstallReady(environment.install); + const docker = await this.assertRuntimeReady(environment.runtime); + return { + ...environment, + cliPath: this.host.commandPath, + docker, + }; + } + + private async assertInstallReady(install: string): Promise { + if (!SUPPORTED_INSTALLS.has(install)) { + throw new Error(`Unsupported scenario install '${install}'.`); + } + return this.host.expectNemoclawAvailable(); + } + + private async assertRuntimeReady(runtime: string): Promise { + const expectation = supportedRuntime(runtime); + const result = await this.probeDocker(runtime, expectation); + if (!result.result) { + return result; + } + + if (expectation === "required") { + assertExitZero(result.result, `docker runtime ${runtime}`); + } + if (expectation === "missing" && result.available) { + throw new Error(`docker runtime ${runtime} expected Docker to be unavailable, but 'docker info' succeeded.`); + } + return result; + } + + private async probeDocker(runtime: string, expectation: DockerRuntimeExpectation): Promise { + try { + const result = await this.host.command("docker", ["info"], { + artifactName: `runtime-docker-info-${artifactLabel(runtime)}`, + inheritEnv: true, + timeoutMs: 30_000, + }); + return { + id: runtime, + expectation, + available: result.exitCode === 0, + result, + }; + } catch (error) { + if (expectation === "required") { + throw error; + } + return { + id: runtime, + expectation, + available: false, + probeError: errorMessage(error), + }; + } + } +} diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts new file mode 100644 index 00000000000..b1eba5be54b --- /dev/null +++ b/test/e2e-scenario/framework/phases/index.ts @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + EnvironmentPhaseFixture, + type DockerRuntimeExpectation, + type DockerRuntimeReady, + type EnvironmentReady, +} from "./environment.ts"; From ada6eeec8b1b24de326a20521625f4e96239a910 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:43:29 -0700 Subject: [PATCH 2/7] test(e2e): add onboarding phase fixture --- .../e2e-phase-onboarding.test.ts | 198 ++++++++++++++++++ test/e2e-scenario/framework/e2e-test.ts | 6 +- test/e2e-scenario/framework/phases/index.ts | 7 + .../framework/phases/onboarding.ts | 142 +++++++++++++ 4 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts create mode 100644 test/e2e-scenario/framework/phases/onboarding.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts new file mode 100644 index 00000000000..117d438b1ae --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; +import type { EnvironmentReady } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: output, + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + return this.responses.shift() ?? shellResult(0); + } +} + +class FakeSecrets implements OnboardingSecrets { + readonly requiredCalls: string[] = []; + + constructor(private readonly values: Record = {}) {} + + required(name: string): string { + this.requiredCalls.push(name); + const value = this.values[name]; + if (!value) throw new Error(`skip: missing required E2E secret: ${name}`); + return value; + } +} + +function ready(overrides: Partial = {}): EnvironmentReady { + return { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + cliPath: "nemoclaw", + docker: { + id: "docker-running", + expectation: "required", + available: true, + result: shellResult(0), + }, + ...overrides, + }; +} + +describe("onboarding phase fixture", () => { + it("runs cloud OpenClaw onboarding with explicit non-interactive inputs", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from(ready(), { sandboxName: "e2e-ubuntu-repo-cloud-openclaw" }); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + }); + expect(secrets.requiredCalls).toEqual(["NVIDIA_API_KEY"]); + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw", + env: { + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: "e2e-ubuntu-repo-cloud-openclaw", + }, + inheritEnv: true, + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }, + ]); + }); + + it("fails cloud OpenClaw onboarding on non-zero exit", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(42, "provider rejected credential")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect(onboard.from(ready())).rejects.toThrow(/cloud-openclaw onboarding failed: provider rejected/); + }); + + it("requires Docker for cloud OpenClaw onboarding", async () => { + const onboard = new OnboardingPhaseFixture(new HostCliClient(new FakeRunner()), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect( + onboard.from( + ready({ + docker: { id: "docker-running", expectation: "required", available: false }, + }), + ), + ).rejects.toThrow(/requires an available Docker runtime/); + }); + + it("runs the no-Docker negative path with a failing Docker shim", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(7, "Cannot connect to the Docker daemon")); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + { sandboxName: "e2e-no-docker" }, + ); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw-no-docker", + sandboxName: "e2e-no-docker", + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }); + expect(secrets.requiredCalls).toEqual([]); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-no-docker", + inheritEnv: true, + timeoutMs: 900_000, + }, + }); + expect(runner.calls[0]?.options?.env?.PATH).toContain("e2e-no-docker-"); + }); + + it("fails the no-Docker path when onboarding unexpectedly succeeds", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect( + onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + ), + ).rejects.toThrow(/unexpectedly succeeded/); + }); + + it("rejects unsupported onboarding profiles", async () => { + const onboard = new OnboardingPhaseFixture(new HostCliClient(new FakeRunner()), new FakeSecrets()); + + await expect(onboard.from(ready({ onboarding: "cloud-hermes" }))).rejects.toThrow( + /Unsupported onboarding profile 'cloud-hermes'/, + ); + }); + + it("exposes the onboarding phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index 6d122b16b1a..fc647649fe3 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,7 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; -import { EnvironmentPhaseFixture } from "./phases/index.ts"; +import { EnvironmentPhaseFixture, OnboardingPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -27,6 +27,7 @@ export interface E2EScenarioFixtures { provider: ProviderClient; state: StateClient; environment: EnvironmentPhaseFixture; + onboard: OnboardingPhaseFixture; } export const test = base.extend({ @@ -82,6 +83,9 @@ export const test = base.extend({ environment: async ({ host }, use) => { await use(new EnvironmentPhaseFixture(host)); }, + onboard: async ({ host, secrets }, use) => { + await use(new OnboardingPhaseFixture(host, secrets)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index b1eba5be54b..25b3dba5147 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -7,3 +7,10 @@ export { type DockerRuntimeReady, type EnvironmentReady, } from "./environment.ts"; +export { + OnboardingPhaseFixture, + type NemoClawInstance, + type OnboardingExpectedFailure, + type OnboardingOptions, + type OnboardingSecrets, +} from "./onboarding.ts"; diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts new file mode 100644 index 00000000000..dd22d09fb1a --- /dev/null +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { artifactLabel, assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { EnvironmentReady } from "./environment.ts"; + +const ONBOARD_ARGS = ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"]; +const DEFAULT_TIMEOUT_MS = 15 * 60_000; +const OPENCLAW_GATEWAY_URL = "http://127.0.0.1:18789"; + +export interface OnboardingSecrets { + required(name: string): string; +} + +export interface OnboardingOptions { + sandboxName?: string; + timeoutMs?: number; +} + +export interface OnboardingExpectedFailure { + phase: "preflight"; + errorClass: "docker-missing"; +} + +export interface NemoClawInstance { + onboarding: string; + sandboxName: string; + agent: "openclaw"; + provider: "nvidia"; + providerEnv: "cloud"; + gatewayUrl: string; + result: ShellProbeResult; + expectedFailure?: OnboardingExpectedFailure; +} + +function defaultSandboxName(onboarding: string): string { + return `e2e-${artifactLabel(onboarding)}`; +} + +function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...extra, + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: sandboxName, + }; +} + +function noDockerShim(): string { + return `#!/usr/bin/env bash +printf 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\\n' >&2 +exit 1 +`; +} + +export class OnboardingPhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly secrets: OnboardingSecrets, + ) {} + + async from(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + switch (environment.onboarding) { + case "cloud-openclaw": + return await this.cloudOpenClaw(environment, options); + case "cloud-openclaw-no-docker": + return await this.cloudOpenClawNoDocker(environment, options); + default: + throw new Error(`Unsupported onboarding profile '${environment.onboarding}'.`); + } + } + + async cloudOpenClaw(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + if (!environment.docker.available) { + throw new Error("cloud-openclaw onboarding requires an available Docker runtime."); + } + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + const sandboxName = options.sandboxName ?? defaultSandboxName(environment.onboarding); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw", + env: commandEnv(sandboxName), + inheritEnv: true, + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-openclaw onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + }; + } + + async cloudOpenClawNoDocker(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + if (environment.docker.available) { + throw new Error("cloud-openclaw-no-docker onboarding requires Docker to be unavailable."); + } + const sandboxName = options.sandboxName ?? defaultSandboxName(environment.onboarding); + const shimDir = await mkdtemp(join(tmpdir(), "e2e-no-docker-")); + const shimPath = join(shimDir, "docker"); + try { + await writeFile(shimPath, noDockerShim(), "utf8"); + await chmod(shimPath, 0o700); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw-no-docker", + env: commandEnv(sandboxName, { + PATH: `${shimDir}:${process.env.PATH ?? ""}`, + }), + inheritEnv: true, + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + if (result.exitCode === 0) { + throw new Error("cloud-openclaw-no-docker onboarding unexpectedly succeeded."); + } + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }; + } finally { + await rm(shimDir, { force: true, recursive: true }); + } + } +} From 82c054311c712038a70ab1a1bd87b3745a14b3f9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:47:55 -0700 Subject: [PATCH 3/7] test(e2e): add state validation phase fixture --- .../e2e-phase-state-validation.test.ts | 189 ++++++++++++++++++ .../e2e-scenario/framework/clients/gateway.ts | 10 +- .../e2e-scenario/framework/clients/sandbox.ts | 12 +- test/e2e-scenario/framework/e2e-test.ts | 6 +- test/e2e-scenario/framework/phases/index.ts | 5 + .../framework/phases/state-validation.ts | 128 ++++++++++++ 6 files changed, 338 insertions(+), 12 deletions(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts create mode 100644 test/e2e-scenario/framework/phases/state-validation.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts new file mode 100644 index 00000000000..f6276290327 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + GatewayClient, + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { StateValidationPhaseFixture, type NemoClawInstance } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: exitCode === 0 ? output : "", + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + return this.responses.shift() ?? shellResult(0); + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner): StateValidationPhaseFixture { + const host = new HostCliClient(runner); + return new StateValidationPhaseFixture(host, new GatewayClient(host), new SandboxClient(runner)); +} + +describe("state-validation phase fixture", () => { + it("validates a ready expected state through CLI, gateway, and sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + runner.enqueue(shellResult(0, "running\n")); + + const result = await fixture(runner).from("cloud-openclaw-ready", instance()); + + expect(result.state.id).toBe("cloud-openclaw-ready"); + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-healthy", + "sandbox-running", + ]); + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["--version"], + options: { artifactName: "nemoclaw-version", inheritEnv: true }, + }, + { + command: "nemoclaw", + args: ["gateway", "status"], + options: { artifactName: "gateway-status", inheritEnv: true }, + }, + { + command: "openshell", + args: ["sandbox", "status", "e2e-ubuntu-repo-cloud-openclaw"], + options: { + artifactName: "sandbox-status-e2e-ubuntu-repo-cloud-openclaw", + inheritEnv: true, + }, + }, + ]); + }); + + it("validates an expected preflight failure with absent gateway and sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "other-sandbox\n")); + + const result = await fixture(runner).from( + "preflight-failure-no-sandbox", + instance({ + onboarding: "cloud-openclaw-no-docker", + sandboxName: "e2e-no-docker", + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }), + ); + + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-absent", + "sandbox-absent", + ]); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + ["list"], + ["sandbox", "list"], + ]); + }); + + it("fails a gateway-absent probe if the gateway is running", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /expected gateway to be absent/, + ); + }); + + it("fails a sandbox-absent probe if NemoClaw lists the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /nemoclaw listed it/, + ); + }); + + it("requires an instance for sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + + await expect(fixture(runner).from("cloud-openclaw-ready")).rejects.toThrow( + /probe 'sandbox-running' requires a NemoClaw instance/, + ); + }); + + it("runs only the CLI probe for optional platform state", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + + const result = await fixture(runner).from("macos-cli-ready-docker-optional"); + + expect(result.probes.map((probe) => probe.id)).toEqual(["cli-installed"]); + expect(runner.calls.map((call) => call.args)).toEqual([["--version"]]); + }); + + it("rejects unknown expected-state IDs", async () => { + const runner = new FakeRunner(); + + await expect(fixture(runner).from("missing-state", instance())).rejects.toThrow(/Unknown expected_state/); + }); + + it("exposes the state-validation phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/clients/gateway.ts b/test/e2e-scenario/framework/clients/gateway.ts index 196d57c48a0..aca6e37a7b3 100644 --- a/test/e2e-scenario/framework/clients/gateway.ts +++ b/test/e2e-scenario/framework/clients/gateway.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ShellProbeResult } from "../shell-probe.ts"; +import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { assertExitZero } from "./command.ts"; import type { HostCliClient } from "./host.ts"; @@ -12,12 +12,12 @@ export class GatewayClient { this.host = host; } - status(): Promise { - return this.host.nemoclaw(["gateway", "status"], { artifactName: "gateway-status" }); + status(options: ShellProbeRunOptions = {}): Promise { + return this.host.nemoclaw(["gateway", "status"], { artifactName: "gateway-status", ...options }); } - async expectHealthy(): Promise { - const result = await this.status(); + async expectHealthy(options: ShellProbeRunOptions = {}): Promise { + const result = await this.status(options); assertExitZero(result, "nemoclaw gateway status"); return result; } diff --git a/test/e2e-scenario/framework/clients/sandbox.ts b/test/e2e-scenario/framework/clients/sandbox.ts index af0c0a4154e..4068ee91ac8 100644 --- a/test/e2e-scenario/framework/clients/sandbox.ts +++ b/test/e2e-scenario/framework/clients/sandbox.ts @@ -32,13 +32,13 @@ export class SandboxClient { ); } - list(): Promise { - return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list" }); + list(options: ShellProbeRunOptions = {}): Promise { + return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list", ...options }); } - status(name: string): Promise { + status(name: string, options: ShellProbeRunOptions = {}): Promise { validateSandboxName(name); - return this.openshell(["sandbox", "status", name], { artifactName: `sandbox-status-${name}` }); + return this.openshell(["sandbox", "status", name], { artifactName: `sandbox-status-${name}`, ...options }); } exec(name: string, command: string[], options: ShellProbeRunOptions = {}): Promise { @@ -49,8 +49,8 @@ export class SandboxClient { }); } - async expectRunning(name: string): Promise { - const result = await this.status(name); + async expectRunning(name: string, options: ShellProbeRunOptions = {}): Promise { + const result = await this.status(name, options); assertExitZero(result, `openshell sandbox status ${name}`); return result; } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index fc647649fe3..985930a42fc 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,7 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; -import { EnvironmentPhaseFixture, OnboardingPhaseFixture } from "./phases/index.ts"; +import { EnvironmentPhaseFixture, OnboardingPhaseFixture, StateValidationPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -28,6 +28,7 @@ export interface E2EScenarioFixtures { state: StateClient; environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; + stateValidation: StateValidationPhaseFixture; } export const test = base.extend({ @@ -86,6 +87,9 @@ export const test = base.extend({ onboard: async ({ host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets)); }, + stateValidation: async ({ host, gateway, sandbox }, use) => { + await use(new StateValidationPhaseFixture(host, gateway, sandbox)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index 25b3dba5147..f6345ba476f 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -14,3 +14,8 @@ export { type OnboardingOptions, type OnboardingSecrets, } from "./onboarding.ts"; +export { + StateValidationPhaseFixture, + type StateValidationProbeResult, + type StateValidationResult, +} from "./state-validation.ts"; diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts new file mode 100644 index 00000000000..af5c8fb0e85 --- /dev/null +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayClient, HostCliClient, SandboxClient } from "../clients/index.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import { probesForState, requireExpectedState } from "../../scenarios/expected-states.ts"; +import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +export interface StateValidationProbeResult { + id: StateProbeId; + status: "passed"; + results: ShellProbeResult[]; +} + +export interface StateValidationResult { + state: ExpectedState; + probes: StateValidationProbeResult[]; +} + +function requireInstance(probe: StateProbeId, instance: NemoClawInstance | undefined): NemoClawInstance { + if (!instance) { + throw new Error(`state-validation probe '${probe}' requires a NemoClaw instance.`); + } + return instance; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function outputContainsSandbox(result: ShellProbeResult, sandboxName: string): boolean { + const output = `${result.stdout}\n${result.stderr}`; + return new RegExp(`(^|\\s)${escapeRegExp(sandboxName)}(\\s|$)`, "m").test(output); +} + +export class StateValidationPhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly gateway: GatewayClient, + private readonly sandbox: SandboxClient, + ) {} + + async from(expectedStateId: string, instance?: NemoClawInstance): Promise { + const state = requireExpectedState(expectedStateId); + const probes: StateValidationProbeResult[] = []; + for (const probe of probesForState(state)) { + probes.push(await this.runProbe(probe, instance)); + } + return { state, probes }; + } + + private async runProbe(probe: StateProbeId, instance: NemoClawInstance | undefined): Promise { + switch (probe) { + case "cli-installed": + return await this.expectCliInstalled(); + case "gateway-healthy": + return await this.expectGatewayHealthy(); + case "gateway-absent": + return await this.expectGatewayAbsent(); + case "sandbox-running": + return await this.expectSandboxRunning(requireInstance(probe, instance)); + case "sandbox-absent": + return await this.expectSandboxAbsent(requireInstance(probe, instance)); + default: { + const _exhaustive: never = probe; + throw new Error(`Unsupported state-validation probe '${_exhaustive}'.`); + } + } + } + + private async expectCliInstalled(): Promise { + const result = await this.host.expectNemoclawAvailable(); + return { id: "cli-installed", status: "passed", results: [result] }; + } + + private async expectGatewayHealthy(): Promise { + const result = await this.gateway.expectHealthy({ inheritEnv: true }); + return { id: "gateway-healthy", status: "passed", results: [result] }; + } + + private async expectGatewayAbsent(): Promise { + const result = await this.gateway.status({ + artifactName: "gateway-absent-status", + inheritEnv: true, + }); + if (result.exitCode === 0) { + throw new Error("state-validation expected gateway to be absent, but 'nemoclaw gateway status' succeeded."); + } + return { id: "gateway-absent", status: "passed", results: [result] }; + } + + private async expectSandboxRunning(instance: NemoClawInstance): Promise { + const result = await this.sandbox.expectRunning(instance.sandboxName, { inheritEnv: true }); + return { id: "sandbox-running", status: "passed", results: [result] }; + } + + private async expectSandboxAbsent(instance: NemoClawInstance): Promise { + const results: ShellProbeResult[] = []; + const nemoclawList = await this.host.nemoclaw(["list"], { + artifactName: "sandbox-absent-nemoclaw-list", + inheritEnv: true, + }); + results.push(nemoclawList); + if (nemoclawList.exitCode === 0 && outputContainsSandbox(nemoclawList, instance.sandboxName)) { + throw new Error(`state-validation expected sandbox '${instance.sandboxName}' to be absent, but nemoclaw listed it.`); + } + + let openshellList: ShellProbeResult | undefined; + try { + openshellList = await this.sandbox.list({ + artifactName: "sandbox-absent-openshell-list", + inheritEnv: true, + }); + } catch { + // Missing or unavailable OpenShell is acceptable when asserting absence; + // the user-facing NemoClaw list result above remains the primary signal. + } + if (openshellList) { + results.push(openshellList); + if (openshellList.exitCode === 0 && outputContainsSandbox(openshellList, instance.sandboxName)) { + throw new Error(`state-validation expected sandbox '${instance.sandboxName}' to be absent, but OpenShell listed it.`); + } + } + + return { id: "sandbox-absent", status: "passed", results }; + } +} From 92eac7421637367991265b251592fc9e5e27a678 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 01:06:11 -0700 Subject: [PATCH 4/7] test(e2e): pin state validation absence fallback --- .../e2e-phase-state-validation.test.ts | 59 ++++++++++++++++++- .../framework/phases/state-validation.ts | 6 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index e71928857a0..82c92b2e9ad 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -37,15 +37,23 @@ function shellResult(exitCode: number, output = ""): ShellProbeResult { class FakeRunner implements CommandRunner { readonly calls: RunnerCall[] = []; - private readonly responses: ShellProbeResult[] = []; + private readonly responses: Array = []; enqueue(response: ShellProbeResult): void { this.responses.push(response); } + enqueueError(error: Error): void { + this.responses.push(error); + } + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { this.calls.push({ command: command.command, args: [...command.args], options }); - return this.responses.shift() ?? shellResult(0); + const response = this.responses.shift(); + if (response instanceof Error) { + throw response; + } + return response ?? shellResult(0); } } @@ -162,6 +170,53 @@ describe("state-validation phase fixture", () => { ); }); + it("fails a sandbox-absent probe if OpenShell lists the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /OpenShell listed it/, + ); + }); + + it("tolerates an unavailable OpenShell list after NemoClaw list reports no sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueueError(new Error("spawn openshell ENOENT")); + + const result = await fixture(runner).from("preflight-failure-no-sandbox", instance()); + + const sandboxAbsent = result.probes.find((probe) => probe.id === "sandbox-absent"); + expect(sandboxAbsent?.results).toHaveLength(1); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + ["list"], + ["sandbox", "list"], + ]); + }); + + it("does not treat sandbox name substrings as present", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw-old\n")); + runner.enqueue(shellResult(0, "NAME\nold-e2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from("preflight-failure-no-sandbox", instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-absent", + "sandbox-absent", + ]); + }); + it("requires an instance for sandbox probes", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index af5c8fb0e85..701c83d5601 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -113,8 +113,10 @@ export class StateValidationPhaseFixture { inheritEnv: true, }); } catch { - // Missing or unavailable OpenShell is acceptable when asserting absence; - // the user-facing NemoClaw list result above remains the primary signal. + // Bridge tolerance for negative preflight states: `nemoclaw list` is the + // user-facing registry authority, while OpenShell may be absent before any + // sandbox setup happens. Once the fixture has a typed OpenShell + // availability probe, make this path fail closed. } if (openshellList) { results.push(openshellList); From 0e20a7a59e050e4bcaa8844eb8fcc1d9ec08efc3 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 01:20:26 -0700 Subject: [PATCH 5/7] test(e2e): tighten state validation absence probes --- .../e2e-phase-state-validation.test.ts | 77 ++++++++++++++++++- .../framework/phases/state-validation.ts | 45 ++++++++--- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index 82c92b2e9ad..5a9aa62f7cf 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -104,14 +104,21 @@ describe("state-validation phase fixture", () => { { command: "nemoclaw", args: ["gateway", "status"], - options: { artifactName: "gateway-status", inheritEnv: true }, + options: { + artifactName: "gateway-status", + env: expect.objectContaining({ + PATH: expect.any(String), + }), + }, }, { command: "openshell", args: ["sandbox", "status", "e2e-ubuntu-repo-cloud-openclaw"], options: { artifactName: "sandbox-status-e2e-ubuntu-repo-cloud-openclaw", - inheritEnv: true, + env: expect.objectContaining({ + PATH: expect.any(String), + }), }, }, ]); @@ -121,6 +128,7 @@ describe("state-validation phase fixture", () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); runner.enqueue(shellResult(0, "other-sandbox\n")); @@ -144,9 +152,11 @@ describe("state-validation phase fixture", () => { expect(runner.calls.map((call) => call.args)).toEqual([ ["--version"], ["gateway", "status"], + ["-fsS", "-o", "/dev/null", "--max-time", "3", "http://127.0.0.1:18789/health"], ["list"], ["sandbox", "list"], ]); + expect(result.probes.find((probe) => probe.id === "gateway-absent")?.results).toHaveLength(2); }); it("fails a gateway-absent probe if the gateway is running", async () => { @@ -159,10 +169,42 @@ describe("state-validation phase fixture", () => { ); }); + it("fails a gateway-absent probe if the gateway health endpoint responds", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway status unavailable\n")); + runner.enqueue(shellResult(0, "ok\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /health responded healthy/, + ); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + ["-fsS", "-o", "/dev/null", "--max-time", "3", "http://127.0.0.1:18789/health"], + ]); + }); + + it("requires a loopback gateway URL for gateway-absent health probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + + await expect( + fixture(runner).from( + "preflight-failure-no-sandbox", + instance({ + gatewayUrl: "http://10.0.0.1:18789", + }), + ), + ).rejects.toThrow(/private or link-local/); + }); + it("fails a sandbox-absent probe if NemoClaw lists the sandbox", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( @@ -174,6 +216,7 @@ describe("state-validation phase fixture", () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); @@ -186,6 +229,7 @@ describe("state-validation phase fixture", () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); runner.enqueueError(new Error("spawn openshell ENOENT")); @@ -196,6 +240,7 @@ describe("state-validation phase fixture", () => { expect(runner.calls.map((call) => call.args)).toEqual([ ["--version"], ["gateway", "status"], + ["-fsS", "-o", "/dev/null", "--max-time", "3", "http://127.0.0.1:18789/health"], ["list"], ["sandbox", "list"], ]); @@ -205,6 +250,7 @@ describe("state-validation phase fixture", () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw-old\n")); runner.enqueue(shellResult(0, "NAME\nold-e2e-ubuntu-repo-cloud-openclaw\n")); @@ -217,6 +263,33 @@ describe("state-validation phase fixture", () => { ]); }); + it("does not pass unrelated secret environment values to status probes", async () => { + const original = process.env.NVIDIA_API_KEY; + process.env.NVIDIA_API_KEY = "nvapi-test-secret-value"; + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "other-sandbox\n")); + + await fixture(runner).from("preflight-failure-no-sandbox", instance()); + + for (const call of runner.calls.slice(1)) { + expect(call.options).not.toHaveProperty("inheritEnv"); + expect(call.options?.env).toEqual(expect.objectContaining({ PATH: expect.any(String) })); + expect(call.options?.env).not.toHaveProperty("NVIDIA_API_KEY"); + } + } finally { + if (original === undefined) { + delete process.env.NVIDIA_API_KEY; + } else { + process.env.NVIDIA_API_KEY = original; + } + } + }); + it("requires an instance for sandbox probes", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index 701c83d5601..ea19376facc 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -1,7 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { GatewayClient, HostCliClient, SandboxClient } from "../clients/index.ts"; +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { + trustedProviderEndpoint, + type GatewayClient, + type HostCliClient, + type SandboxClient, +} from "../clients/index.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; import { probesForState, requireExpectedState } from "../../scenarios/expected-states.ts"; import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; @@ -34,6 +40,14 @@ function outputContainsSandbox(result: ShellProbeResult, sandboxName: string): b return new RegExp(`(^|\\s)${escapeRegExp(sandboxName)}(\\s|$)`, "m").test(output); } +function statusProbeEnv(): NodeJS.ProcessEnv { + return buildAvailabilityProbeEnv(); +} + +function gatewayHealthEndpoint(gatewayUrl: string): string { + return trustedProviderEndpoint(`${gatewayUrl.replace(/\/+$/, "")}/health`).url; +} + export class StateValidationPhaseFixture { constructor( private readonly host: HostCliClient, @@ -57,7 +71,7 @@ export class StateValidationPhaseFixture { case "gateway-healthy": return await this.expectGatewayHealthy(); case "gateway-absent": - return await this.expectGatewayAbsent(); + return await this.expectGatewayAbsent(requireInstance(probe, instance)); case "sandbox-running": return await this.expectSandboxRunning(requireInstance(probe, instance)); case "sandbox-absent": @@ -75,23 +89,36 @@ export class StateValidationPhaseFixture { } private async expectGatewayHealthy(): Promise { - const result = await this.gateway.expectHealthy({ inheritEnv: true }); + const result = await this.gateway.expectHealthy({ env: statusProbeEnv() }); return { id: "gateway-healthy", status: "passed", results: [result] }; } - private async expectGatewayAbsent(): Promise { + private async expectGatewayAbsent(instance: NemoClawInstance): Promise { const result = await this.gateway.status({ artifactName: "gateway-absent-status", - inheritEnv: true, + env: statusProbeEnv(), }); if (result.exitCode === 0) { throw new Error("state-validation expected gateway to be absent, but 'nemoclaw gateway status' succeeded."); } - return { id: "gateway-absent", status: "passed", results: [result] }; + const healthUrl = gatewayHealthEndpoint(instance.gatewayUrl); + const health = await this.host.command( + "curl", + ["-fsS", "-o", "/dev/null", "--max-time", "3", healthUrl], + { + artifactName: "gateway-absent-health", + env: statusProbeEnv(), + redactionValues: [healthUrl], + }, + ); + if (health.exitCode === 0) { + throw new Error(`state-validation expected gateway to be absent, but ${healthUrl} responded healthy.`); + } + return { id: "gateway-absent", status: "passed", results: [result, health] }; } private async expectSandboxRunning(instance: NemoClawInstance): Promise { - const result = await this.sandbox.expectRunning(instance.sandboxName, { inheritEnv: true }); + const result = await this.sandbox.expectRunning(instance.sandboxName, { env: statusProbeEnv() }); return { id: "sandbox-running", status: "passed", results: [result] }; } @@ -99,7 +126,7 @@ export class StateValidationPhaseFixture { const results: ShellProbeResult[] = []; const nemoclawList = await this.host.nemoclaw(["list"], { artifactName: "sandbox-absent-nemoclaw-list", - inheritEnv: true, + env: statusProbeEnv(), }); results.push(nemoclawList); if (nemoclawList.exitCode === 0 && outputContainsSandbox(nemoclawList, instance.sandboxName)) { @@ -110,7 +137,7 @@ export class StateValidationPhaseFixture { try { openshellList = await this.sandbox.list({ artifactName: "sandbox-absent-openshell-list", - inheritEnv: true, + env: statusProbeEnv(), }); } catch { // Bridge tolerance for negative preflight states: `nemoclaw list` is the From aa5ec2f4f5db4a9c70e60ad7cee6eeb645e2746d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 01:32:53 -0700 Subject: [PATCH 6/7] test(e2e): align state validation probe contracts --- .../framework-tests/e2e-clients.test.ts | 47 ++++++ .../e2e-phase-state-validation.test.ts | 156 ++++++++++++++++-- .../framework/phases/onboarding.ts | 7 +- .../framework/phases/state-validation.ts | 111 +++++++++++-- 4 files changed, 288 insertions(+), 33 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index b652c6e102e..429ef2c88db 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -106,6 +106,30 @@ describe("E2E fixture clients", () => { }); }); + it("gateway client preserves caller-provided probe options", async () => { + const runner = new FakeRunner(); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + const gateway = new GatewayClient(host); + + await gateway.status({ + artifactName: "custom-gateway-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }); + + expect(runner.calls[0]).toEqual({ + command: "nemoclaw", + args: ["gateway", "status"], + options: { + artifactName: "custom-gateway-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }, + }); + }); + it("sandbox client builds OpenShell sandbox commands", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); @@ -121,6 +145,29 @@ describe("E2E fixture clients", () => { }); }); + it("sandbox client preserves caller-provided probe options", async () => { + const runner = new FakeRunner(); + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + await sandbox.status("assistant", { + artifactName: "custom-sandbox-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }); + + expect(runner.calls[0]).toEqual({ + command: "openshell", + args: ["sandbox", "status", "assistant"], + options: { + artifactName: "custom-sandbox-status", + env: { NEMOCLAW_TEST_VALUE: "1" }, + inheritEnv: true, + timeoutMs: 123, + }, + }); + }); + it("sandbox client rejects flag-shaped sandbox names before command construction", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index 5a9aa62f7cf..45938c253c1 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -19,6 +19,37 @@ interface RunnerCall { options?: ShellProbeRunOptions; } +const GATEWAY_HEALTH_CURL_ARGS = [ + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://127.0.0.1:18789/health", +]; +const GATEWAY_BASE_CURL_ARGS = [ + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://127.0.0.1:18789/", +]; +const GATEWAY_ABSENT_HEALTH_CURL_ARGS = [ + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "3", + "http://127.0.0.1:18789/health", +]; + function shellResult(exitCode: number, output = ""): ShellProbeResult { return { command: [], @@ -76,11 +107,11 @@ function fixture(runner: FakeRunner): StateValidationPhaseFixture { } describe("state-validation phase fixture", () => { - it("validates a ready expected state through CLI, gateway, and sandbox probes", async () => { + it("validates a ready expected state through CLI, gateway health, and sandbox registry probes", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); - runner.enqueue(shellResult(0, "gateway healthy\n")); - runner.enqueue(shellResult(0, "running\n")); + runner.enqueue(shellResult(0, "200")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); const result = await fixture(runner).from("cloud-openclaw-ready", instance()); @@ -102,20 +133,21 @@ describe("state-validation phase fixture", () => { }, }, { - command: "nemoclaw", - args: ["gateway", "status"], + command: "curl", + args: GATEWAY_HEALTH_CURL_ARGS, options: { - artifactName: "gateway-status", + artifactName: "gateway-health", env: expect.objectContaining({ PATH: expect.any(String), }), + redactionValues: ["http://127.0.0.1:18789/health"], }, }, { - command: "openshell", - args: ["sandbox", "status", "e2e-ubuntu-repo-cloud-openclaw"], + command: "nemoclaw", + args: ["list"], options: { - artifactName: "sandbox-status-e2e-ubuntu-repo-cloud-openclaw", + artifactName: "sandbox-running-nemoclaw-list", env: expect.objectContaining({ PATH: expect.any(String), }), @@ -124,6 +156,88 @@ describe("state-validation phase fixture", () => { ]); }); + it("accepts a healthy gateway base URL fallback when the health endpoint is unavailable", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "204")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from("cloud-openclaw-ready", instance()); + + expect(result.probes.find((probe) => probe.id === "gateway-healthy")?.results).toHaveLength(2); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + GATEWAY_HEALTH_CURL_ARGS, + GATEWAY_BASE_CURL_ARGS, + ["list"], + ]); + }); + + it("accepts the sandbox-local Ollama gateway fallback", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "401")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + const result = await fixture(runner).from( + "local-ollama-openclaw-ready", + instance({ + provider: "ollama", + providerEnv: "local", + }), + ); + + expect(result.probes.find((probe) => probe.id === "gateway-healthy")?.results).toHaveLength(3); + expect(runner.calls[3]).toMatchObject({ + command: "openshell", + args: [ + "sandbox", + "exec", + "e2e-ubuntu-repo-cloud-openclaw", + "--", + "curl", + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://localhost:18789/health", + ], + }); + }); + + it("fails a gateway-healthy probe if the gateway HTTP probes are unhealthy", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(7, "connection refused")); + + await expect(fixture(runner).from("cloud-openclaw-ready", instance())).rejects.toThrow( + /expected gateway .* to be healthy/, + ); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + GATEWAY_HEALTH_CURL_ARGS, + GATEWAY_BASE_CURL_ARGS, + ]); + }); + + it("fails a sandbox-running probe if NemoClaw does not list the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "200")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + + await expect(fixture(runner).from("cloud-openclaw-ready", instance())).rejects.toThrow( + /nemoclaw did not list it/, + ); + }); + it("validates an expected preflight failure with absent gateway and sandbox probes", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); @@ -152,7 +266,7 @@ describe("state-validation phase fixture", () => { expect(runner.calls.map((call) => call.args)).toEqual([ ["--version"], ["gateway", "status"], - ["-fsS", "-o", "/dev/null", "--max-time", "3", "http://127.0.0.1:18789/health"], + GATEWAY_ABSENT_HEALTH_CURL_ARGS, ["list"], ["sandbox", "list"], ]); @@ -181,7 +295,7 @@ describe("state-validation phase fixture", () => { expect(runner.calls.map((call) => call.args)).toEqual([ ["--version"], ["gateway", "status"], - ["-fsS", "-o", "/dev/null", "--max-time", "3", "http://127.0.0.1:18789/health"], + GATEWAY_ABSENT_HEALTH_CURL_ARGS, ]); }); @@ -240,12 +354,25 @@ describe("state-validation phase fixture", () => { expect(runner.calls.map((call) => call.args)).toEqual([ ["--version"], ["gateway", "status"], - ["-fsS", "-o", "/dev/null", "--max-time", "3", "http://127.0.0.1:18789/health"], + GATEWAY_ABSENT_HEALTH_CURL_ARGS, ["list"], ["sandbox", "list"], ]); }); + it("fails a sandbox-absent probe if OpenShell list errors unexpectedly", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(7, "connection refused")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueueError(new Error("openshell permission denied")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /could not verify OpenShell sandbox absence/, + ); + }); + it("does not treat sandbox name substrings as present", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); @@ -290,13 +417,12 @@ describe("state-validation phase fixture", () => { } }); - it("requires an instance for sandbox probes", async () => { + it("requires an instance for probes that use instance context", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); - runner.enqueue(shellResult(0, "gateway healthy\n")); await expect(fixture(runner).from("cloud-openclaw-ready")).rejects.toThrow( - /probe 'sandbox-running' requires a NemoClaw instance/, + /probe 'gateway-healthy' requires a NemoClaw instance/, ); }); diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index 458813958c0..717aad6ed19 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -58,9 +58,10 @@ export interface OnboardingExpectedFailure { export interface NemoClawInstance { onboarding: string; sandboxName: string; - agent: "openclaw"; - provider: "nvidia"; - providerEnv: "cloud"; + agent: "openclaw" | "hermes"; + provider: "nvidia" | "ollama"; + providerEnv: "cloud" | "local"; + platformOs?: "ubuntu" | "macos" | "windows"; gatewayUrl: string; result: ShellProbeResult; expectedFailure?: OnboardingExpectedFailure; diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index ea19376facc..68160c8cc09 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -48,6 +48,27 @@ function gatewayHealthEndpoint(gatewayUrl: string): string { return trustedProviderEndpoint(`${gatewayUrl.replace(/\/+$/, "")}/health`).url; } +function gatewayBaseEndpoint(gatewayUrl: string): string { + return trustedProviderEndpoint(gatewayUrl).url; +} + +function resultHttpCode(result: ShellProbeResult): string { + return result.stdout.trim(); +} + +function resultHasHttpCode(result: ShellProbeResult, allowedCodes: readonly string[]): boolean { + return result.exitCode === 0 && allowedCodes.includes(resultHttpCode(result)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isMissingOpenShellError(error: unknown): boolean { + const code = typeof error === "object" && error !== null && "code" in error ? error.code : undefined; + return code === "ENOENT" || /\bENOENT\b/.test(errorMessage(error)); +} + export class StateValidationPhaseFixture { constructor( private readonly host: HostCliClient, @@ -69,7 +90,7 @@ export class StateValidationPhaseFixture { case "cli-installed": return await this.expectCliInstalled(); case "gateway-healthy": - return await this.expectGatewayHealthy(); + return await this.expectGatewayHealthy(requireInstance(probe, instance)); case "gateway-absent": return await this.expectGatewayAbsent(requireInstance(probe, instance)); case "sandbox-running": @@ -88,9 +109,63 @@ export class StateValidationPhaseFixture { return { id: "cli-installed", status: "passed", results: [result] }; } - private async expectGatewayHealthy(): Promise { - const result = await this.gateway.expectHealthy({ env: statusProbeEnv() }); - return { id: "gateway-healthy", status: "passed", results: [result] }; + private curlHttpStatus(url: string, artifactName: string, maxTimeSeconds: string): Promise { + return this.host.command( + "curl", + ["-fsS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", maxTimeSeconds, url], + { + artifactName, + env: statusProbeEnv(), + redactionValues: [url], + }, + ); + } + + private async expectGatewayHealthy(instance: NemoClawInstance): Promise { + const results: ShellProbeResult[] = []; + const health = await this.curlHttpStatus(gatewayHealthEndpoint(instance.gatewayUrl), "gateway-health", "5"); + results.push(health); + if (resultHasHttpCode(health, ["200"])) { + return { id: "gateway-healthy", status: "passed", results }; + } + + const base = await this.curlHttpStatus(gatewayBaseEndpoint(instance.gatewayUrl), "gateway-base", "5"); + results.push(base); + if (resultHasHttpCode(base, ["200", "204"])) { + return { id: "gateway-healthy", status: "passed", results }; + } + + if ((instance.platformOs ?? "ubuntu") === "ubuntu" && instance.provider === "ollama") { + const sandboxLocal = await this.sandbox.exec( + instance.sandboxName, + [ + "curl", + "-fsS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + "http://localhost:18789/health", + ], + { + artifactName: "gateway-sandbox-local-health", + env: statusProbeEnv(), + timeoutMs: 15_000, + }, + ); + results.push(sandboxLocal); + if (resultHasHttpCode(sandboxLocal, ["200", "401"])) { + return { id: "gateway-healthy", status: "passed", results }; + } + } + + const last = results.at(-1) ?? base; + throw new Error( + `state-validation expected gateway '${instance.gatewayUrl}' to be healthy, ` + + `but HTTP probes failed (last http_code=${resultHttpCode(last) || "000"}).`, + ); } private async expectGatewayAbsent(instance: NemoClawInstance): Promise { @@ -102,15 +177,7 @@ export class StateValidationPhaseFixture { throw new Error("state-validation expected gateway to be absent, but 'nemoclaw gateway status' succeeded."); } const healthUrl = gatewayHealthEndpoint(instance.gatewayUrl); - const health = await this.host.command( - "curl", - ["-fsS", "-o", "/dev/null", "--max-time", "3", healthUrl], - { - artifactName: "gateway-absent-health", - env: statusProbeEnv(), - redactionValues: [healthUrl], - }, - ); + const health = await this.curlHttpStatus(healthUrl, "gateway-absent-health", "3"); if (health.exitCode === 0) { throw new Error(`state-validation expected gateway to be absent, but ${healthUrl} responded healthy.`); } @@ -118,7 +185,18 @@ export class StateValidationPhaseFixture { } private async expectSandboxRunning(instance: NemoClawInstance): Promise { - const result = await this.sandbox.expectRunning(instance.sandboxName, { env: statusProbeEnv() }); + const result = await this.host.nemoclaw(["list"], { + artifactName: "sandbox-running-nemoclaw-list", + env: statusProbeEnv(), + }); + if (result.exitCode !== 0) { + throw new Error("state-validation expected sandbox to be running, but 'nemoclaw list' failed."); + } + if (!outputContainsSandbox(result, instance.sandboxName)) { + throw new Error( + `state-validation expected sandbox '${instance.sandboxName}' to be running, but nemoclaw did not list it.`, + ); + } return { id: "sandbox-running", status: "passed", results: [result] }; } @@ -139,7 +217,10 @@ export class StateValidationPhaseFixture { artifactName: "sandbox-absent-openshell-list", env: statusProbeEnv(), }); - } catch { + } catch (error) { + if (!isMissingOpenShellError(error)) { + throw new Error(`state-validation could not verify OpenShell sandbox absence: ${errorMessage(error)}`); + } // Bridge tolerance for negative preflight states: `nemoclaw list` is the // user-facing registry authority, while OpenShell may be absent before any // sandbox setup happens. Once the fixture has a typed OpenShell From 9e3f224c8c69d6da74d0abb2963c87525c85aa08 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 01:41:43 -0700 Subject: [PATCH 7/7] test(e2e): fix ollama gateway fallback curl flags --- .../framework-tests/e2e-phase-state-validation.test.ts | 2 +- test/e2e-scenario/framework/phases/state-validation.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index 45938c253c1..ecf66f18d12 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -199,7 +199,7 @@ describe("state-validation phase fixture", () => { "e2e-ubuntu-repo-cloud-openclaw", "--", "curl", - "-fsS", + "-sS", "-o", "/dev/null", "-w", diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index 68160c8cc09..b665e4ee233 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -140,7 +140,7 @@ export class StateValidationPhaseFixture { instance.sandboxName, [ "curl", - "-fsS", + "-sS", "-o", "/dev/null", "-w",