From 32ad96a3b52acc4d70d61307b37523a53d4e6816 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:39:02 -0700 Subject: [PATCH 1/8] test(e2e): add environment phase fixture --- .../framework-tests/e2e-clients.test.ts | 2 +- .../e2e-phase-environment.test.ts | 181 ++++++++++++++++++ test/e2e-scenario/framework/clients/host.ts | 6 +- test/e2e-scenario/framework/e2e-test.ts | 5 + .../framework/phases/environment.ts | 107 +++++++++++ test/e2e-scenario/framework/phases/index.ts | 9 + 6 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts create mode 100644 test/e2e-scenario/framework/phases/environment.ts create mode 100644 test/e2e-scenario/framework/phases/index.ts diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index 25c893d5f1a..4d5c3a72dbf 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -61,7 +61,7 @@ describe("E2E fixture clients", () => { { command: "./bin/nemoclaw.js", args: ["--version"], - options: { artifactName: "nemoclaw-version" }, + options: { artifactName: "nemoclaw-version", inheritEnv: true }, }, ]); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts new file mode 100644 index 00000000000..cf41f3983e8 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-environment.test.ts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { EnvironmentPhaseFixture, type DockerRuntimeReady } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; +import type { ScenarioEnvironment } from "../scenarios/types.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: output, + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: Array = []; + + enqueue(response: ShellProbeResult | Error): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift() ?? shellResult(0); + if (response instanceof Error) { + throw response; + } + return response; + } +} + +const cloudOpenClawEnvironment: ScenarioEnvironment = { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", +}; + +describe("environment phase fixture", () => { + it("asserts the current repo CLI and required Docker runtime", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "Docker is available\n")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner, { cliPath: "./bin/nemoclaw.js" })); + + const ready = await environment.assertReady(cloudOpenClawEnvironment); + + expect(ready).toMatchObject({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + cliPath: "./bin/nemoclaw.js", + docker: { + id: "docker-running", + expectation: "required", + available: true, + } satisfies Partial, + }); + expect(runner.calls).toEqual([ + { + command: "./bin/nemoclaw.js", + args: ["--version"], + options: { artifactName: "nemoclaw-version", inheritEnv: true }, + }, + { + command: "docker", + args: ["info"], + options: { + artifactName: "runtime-docker-info-docker-running", + inheritEnv: true, + timeoutMs: 30_000, + }, + }, + ]); + }); + + it("fails when a required Docker runtime is unavailable", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + await expect(environment.assertReady(cloudOpenClawEnvironment)).rejects.toThrow( + /docker runtime docker-running failed: Cannot connect/, + ); + }); + + it("accepts an unavailable Docker runtime for no-Docker negative scenarios", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "docker intentionally unavailable")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + }); + + expect(ready.docker).toMatchObject({ + id: "docker-missing", + expectation: "missing", + available: false, + }); + }); + + it("fails if a no-Docker negative scenario unexpectedly has Docker", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "Docker is available\n")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + await expect( + environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + }), + ).rejects.toThrow(/expected Docker to be unavailable/); + }); + + it("records optional Docker as unavailable without failing", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(new Error("spawn docker ENOENT")); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + platform: "macos-local", + runtime: "macos-docker-optional", + }); + + expect(ready.docker).toMatchObject({ + id: "macos-docker-optional", + expectation: "optional", + available: false, + probeError: "spawn docker ENOENT", + }); + }); + + it("rejects unsupported install and runtime IDs", async () => { + const runner = new FakeRunner(); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + await expect(environment.assertReady({ ...cloudOpenClawEnvironment, install: "tarball" })).rejects.toThrow( + /Unsupported scenario install 'tarball'/, + ); + expect(runner.calls).toEqual([]); + + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + await expect(environment.assertReady({ ...cloudOpenClawEnvironment, runtime: "podman-running" })).rejects.toThrow( + /Unsupported scenario runtime 'podman-running'/, + ); + }); + + it("exposes the environment phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/clients/host.ts b/test/e2e-scenario/framework/clients/host.ts index 9e0ee80ebd3..8c5ba1dd9ac 100644 --- a/test/e2e-scenario/framework/clients/host.ts +++ b/test/e2e-scenario/framework/clients/host.ts @@ -21,6 +21,10 @@ export class HostCliClient { this.cwd = options.cwd; } + get commandPath(): string { + return this.cliPath; + } + command(command: string, args: string[] = [], options: ShellProbeRunOptions = {}): Promise { const merged: ShellProbeRunOptions = { ...options }; if (this.cwd && !merged.cwd) { @@ -44,7 +48,7 @@ export class HostCliClient { } async expectNemoclawAvailable(): Promise { - const result = await this.nemoclaw(["--version"], { artifactName: "nemoclaw-version" }); + const result = await this.nemoclaw(["--version"], { artifactName: "nemoclaw-version", inheritEnv: true }); assertExitZero(result, "nemoclaw --version"); return result; } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index e44f50c7f27..6d122b16b1a 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,6 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; +import { EnvironmentPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -25,6 +26,7 @@ export interface E2EScenarioFixtures { sandbox: SandboxClient; provider: ProviderClient; state: StateClient; + environment: EnvironmentPhaseFixture; } export const test = base.extend({ @@ -77,6 +79,9 @@ export const test = base.extend({ state: async ({}, use) => { await use(new StateClient()); }, + environment: async ({ host }, use) => { + await use(new EnvironmentPhaseFixture(host)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/environment.ts b/test/e2e-scenario/framework/phases/environment.ts new file mode 100644 index 00000000000..54cad8d6d60 --- /dev/null +++ b/test/e2e-scenario/framework/phases/environment.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { artifactLabel, assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { ScenarioEnvironment } from "../../scenarios/types.ts"; + +const SUPPORTED_INSTALLS = new Set(["repo-current", "launchable"]); + +const DOCKER_RUNTIME_EXPECTATIONS = { + "docker-running": "required", + "gpu-docker-cdi": "required", + "docker-missing": "missing", + "macos-docker-optional": "optional", +} as const; + +export type DockerRuntimeExpectation = + (typeof DOCKER_RUNTIME_EXPECTATIONS)[keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; + +export interface DockerRuntimeReady { + id: string; + expectation: DockerRuntimeExpectation; + available: boolean; + result?: ShellProbeResult; + probeError?: string; +} + +export interface EnvironmentReady extends ScenarioEnvironment { + cliPath: string; + docker: DockerRuntimeReady; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function supportedRuntime(runtime: string): DockerRuntimeExpectation { + const expectation = DOCKER_RUNTIME_EXPECTATIONS[runtime as keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; + if (!expectation) { + throw new Error(`Unsupported scenario runtime '${runtime}'.`); + } + return expectation; +} + +export class EnvironmentPhaseFixture { + constructor(private readonly host: HostCliClient) {} + + async assertReady(environment: ScenarioEnvironment): Promise { + await this.assertInstallReady(environment.install); + const docker = await this.assertRuntimeReady(environment.runtime); + return { + ...environment, + cliPath: this.host.commandPath, + docker, + }; + } + + private async assertInstallReady(install: string): Promise { + if (!SUPPORTED_INSTALLS.has(install)) { + throw new Error(`Unsupported scenario install '${install}'.`); + } + return this.host.expectNemoclawAvailable(); + } + + private async assertRuntimeReady(runtime: string): Promise { + const expectation = supportedRuntime(runtime); + const result = await this.probeDocker(runtime, expectation); + if (!result.result) { + return result; + } + + if (expectation === "required") { + assertExitZero(result.result, `docker runtime ${runtime}`); + } + if (expectation === "missing" && result.available) { + throw new Error(`docker runtime ${runtime} expected Docker to be unavailable, but 'docker info' succeeded.`); + } + return result; + } + + private async probeDocker(runtime: string, expectation: DockerRuntimeExpectation): Promise { + try { + const result = await this.host.command("docker", ["info"], { + artifactName: `runtime-docker-info-${artifactLabel(runtime)}`, + inheritEnv: true, + timeoutMs: 30_000, + }); + return { + id: runtime, + expectation, + available: result.exitCode === 0, + result, + }; + } catch (error) { + if (expectation === "required") { + throw error; + } + return { + id: runtime, + expectation, + available: false, + probeError: errorMessage(error), + }; + } + } +} diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts new file mode 100644 index 00000000000..b1eba5be54b --- /dev/null +++ b/test/e2e-scenario/framework/phases/index.ts @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + EnvironmentPhaseFixture, + type DockerRuntimeExpectation, + type DockerRuntimeReady, + type EnvironmentReady, +} from "./environment.ts"; From ada6eeec8b1b24de326a20521625f4e96239a910 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:43:29 -0700 Subject: [PATCH 2/8] test(e2e): add onboarding phase fixture --- .../e2e-phase-onboarding.test.ts | 198 ++++++++++++++++++ test/e2e-scenario/framework/e2e-test.ts | 6 +- test/e2e-scenario/framework/phases/index.ts | 7 + .../framework/phases/onboarding.ts | 142 +++++++++++++ 4 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts create mode 100644 test/e2e-scenario/framework/phases/onboarding.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts new file mode 100644 index 00000000000..117d438b1ae --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; +import type { EnvironmentReady } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: output, + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + return this.responses.shift() ?? shellResult(0); + } +} + +class FakeSecrets implements OnboardingSecrets { + readonly requiredCalls: string[] = []; + + constructor(private readonly values: Record = {}) {} + + required(name: string): string { + this.requiredCalls.push(name); + const value = this.values[name]; + if (!value) throw new Error(`skip: missing required E2E secret: ${name}`); + return value; + } +} + +function ready(overrides: Partial = {}): EnvironmentReady { + return { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + cliPath: "nemoclaw", + docker: { + id: "docker-running", + expectation: "required", + available: true, + result: shellResult(0), + }, + ...overrides, + }; +} + +describe("onboarding phase fixture", () => { + it("runs cloud OpenClaw onboarding with explicit non-interactive inputs", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from(ready(), { sandboxName: "e2e-ubuntu-repo-cloud-openclaw" }); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + }); + expect(secrets.requiredCalls).toEqual(["NVIDIA_API_KEY"]); + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw", + env: { + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: "e2e-ubuntu-repo-cloud-openclaw", + }, + inheritEnv: true, + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }, + ]); + }); + + it("fails cloud OpenClaw onboarding on non-zero exit", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(42, "provider rejected credential")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect(onboard.from(ready())).rejects.toThrow(/cloud-openclaw onboarding failed: provider rejected/); + }); + + it("requires Docker for cloud OpenClaw onboarding", async () => { + const onboard = new OnboardingPhaseFixture(new HostCliClient(new FakeRunner()), new FakeSecrets({ NVIDIA_API_KEY: "secret" })); + + await expect( + onboard.from( + ready({ + docker: { id: "docker-running", expectation: "required", available: false }, + }), + ), + ).rejects.toThrow(/requires an available Docker runtime/); + }); + + it("runs the no-Docker negative path with a failing Docker shim", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(7, "Cannot connect to the Docker daemon")); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + { sandboxName: "e2e-no-docker" }, + ); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw-no-docker", + sandboxName: "e2e-no-docker", + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }); + expect(secrets.requiredCalls).toEqual([]); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-no-docker", + inheritEnv: true, + timeoutMs: 900_000, + }, + }); + expect(runner.calls[0]?.options?.env?.PATH).toContain("e2e-no-docker-"); + }); + + it("fails the no-Docker path when onboarding unexpectedly succeeds", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect( + onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + ), + ).rejects.toThrow(/unexpectedly succeeded/); + }); + + it("rejects unsupported onboarding profiles", async () => { + const onboard = new OnboardingPhaseFixture(new HostCliClient(new FakeRunner()), new FakeSecrets()); + + await expect(onboard.from(ready({ onboarding: "cloud-hermes" }))).rejects.toThrow( + /Unsupported onboarding profile 'cloud-hermes'/, + ); + }); + + it("exposes the onboarding phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index 6d122b16b1a..fc647649fe3 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,7 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; -import { EnvironmentPhaseFixture } from "./phases/index.ts"; +import { EnvironmentPhaseFixture, OnboardingPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -27,6 +27,7 @@ export interface E2EScenarioFixtures { provider: ProviderClient; state: StateClient; environment: EnvironmentPhaseFixture; + onboard: OnboardingPhaseFixture; } export const test = base.extend({ @@ -82,6 +83,9 @@ export const test = base.extend({ environment: async ({ host }, use) => { await use(new EnvironmentPhaseFixture(host)); }, + onboard: async ({ host, secrets }, use) => { + await use(new OnboardingPhaseFixture(host, secrets)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index b1eba5be54b..25b3dba5147 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -7,3 +7,10 @@ export { type DockerRuntimeReady, type EnvironmentReady, } from "./environment.ts"; +export { + OnboardingPhaseFixture, + type NemoClawInstance, + type OnboardingExpectedFailure, + type OnboardingOptions, + type OnboardingSecrets, +} from "./onboarding.ts"; diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts new file mode 100644 index 00000000000..dd22d09fb1a --- /dev/null +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { artifactLabel, assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { EnvironmentReady } from "./environment.ts"; + +const ONBOARD_ARGS = ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"]; +const DEFAULT_TIMEOUT_MS = 15 * 60_000; +const OPENCLAW_GATEWAY_URL = "http://127.0.0.1:18789"; + +export interface OnboardingSecrets { + required(name: string): string; +} + +export interface OnboardingOptions { + sandboxName?: string; + timeoutMs?: number; +} + +export interface OnboardingExpectedFailure { + phase: "preflight"; + errorClass: "docker-missing"; +} + +export interface NemoClawInstance { + onboarding: string; + sandboxName: string; + agent: "openclaw"; + provider: "nvidia"; + providerEnv: "cloud"; + gatewayUrl: string; + result: ShellProbeResult; + expectedFailure?: OnboardingExpectedFailure; +} + +function defaultSandboxName(onboarding: string): string { + return `e2e-${artifactLabel(onboarding)}`; +} + +function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...extra, + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: sandboxName, + }; +} + +function noDockerShim(): string { + return `#!/usr/bin/env bash +printf 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\\n' >&2 +exit 1 +`; +} + +export class OnboardingPhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly secrets: OnboardingSecrets, + ) {} + + async from(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + switch (environment.onboarding) { + case "cloud-openclaw": + return await this.cloudOpenClaw(environment, options); + case "cloud-openclaw-no-docker": + return await this.cloudOpenClawNoDocker(environment, options); + default: + throw new Error(`Unsupported onboarding profile '${environment.onboarding}'.`); + } + } + + async cloudOpenClaw(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + if (!environment.docker.available) { + throw new Error("cloud-openclaw onboarding requires an available Docker runtime."); + } + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + const sandboxName = options.sandboxName ?? defaultSandboxName(environment.onboarding); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw", + env: commandEnv(sandboxName), + inheritEnv: true, + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-openclaw onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + }; + } + + async cloudOpenClawNoDocker(environment: EnvironmentReady, options: OnboardingOptions = {}): Promise { + if (environment.docker.available) { + throw new Error("cloud-openclaw-no-docker onboarding requires Docker to be unavailable."); + } + const sandboxName = options.sandboxName ?? defaultSandboxName(environment.onboarding); + const shimDir = await mkdtemp(join(tmpdir(), "e2e-no-docker-")); + const shimPath = join(shimDir, "docker"); + try { + await writeFile(shimPath, noDockerShim(), "utf8"); + await chmod(shimPath, 0o700); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw-no-docker", + env: commandEnv(sandboxName, { + PATH: `${shimDir}:${process.env.PATH ?? ""}`, + }), + inheritEnv: true, + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + if (result.exitCode === 0) { + throw new Error("cloud-openclaw-no-docker onboarding unexpectedly succeeded."); + } + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }; + } finally { + await rm(shimDir, { force: true, recursive: true }); + } + } +} From 82c054311c712038a70ab1a1bd87b3745a14b3f9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:47:55 -0700 Subject: [PATCH 3/8] test(e2e): add state validation phase fixture --- .../e2e-phase-state-validation.test.ts | 189 ++++++++++++++++++ .../e2e-scenario/framework/clients/gateway.ts | 10 +- .../e2e-scenario/framework/clients/sandbox.ts | 12 +- test/e2e-scenario/framework/e2e-test.ts | 6 +- test/e2e-scenario/framework/phases/index.ts | 5 + .../framework/phases/state-validation.ts | 128 ++++++++++++ 6 files changed, 338 insertions(+), 12 deletions(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts create mode 100644 test/e2e-scenario/framework/phases/state-validation.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts new file mode 100644 index 00000000000..f6276290327 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + GatewayClient, + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { StateValidationPhaseFixture, type NemoClawInstance } from "../framework/phases/index.ts"; +import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand } from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: exitCode === 0 ? output : "", + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + return this.responses.shift() ?? shellResult(0); + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner): StateValidationPhaseFixture { + const host = new HostCliClient(runner); + return new StateValidationPhaseFixture(host, new GatewayClient(host), new SandboxClient(runner)); +} + +describe("state-validation phase fixture", () => { + it("validates a ready expected state through CLI, gateway, and sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + runner.enqueue(shellResult(0, "running\n")); + + const result = await fixture(runner).from("cloud-openclaw-ready", instance()); + + expect(result.state.id).toBe("cloud-openclaw-ready"); + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-healthy", + "sandbox-running", + ]); + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["--version"], + options: { artifactName: "nemoclaw-version", inheritEnv: true }, + }, + { + command: "nemoclaw", + args: ["gateway", "status"], + options: { artifactName: "gateway-status", inheritEnv: true }, + }, + { + command: "openshell", + args: ["sandbox", "status", "e2e-ubuntu-repo-cloud-openclaw"], + options: { + artifactName: "sandbox-status-e2e-ubuntu-repo-cloud-openclaw", + inheritEnv: true, + }, + }, + ]); + }); + + it("validates an expected preflight failure with absent gateway and sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\nother-sandbox\n")); + runner.enqueue(shellResult(0, "other-sandbox\n")); + + const result = await fixture(runner).from( + "preflight-failure-no-sandbox", + instance({ + onboarding: "cloud-openclaw-no-docker", + sandboxName: "e2e-no-docker", + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + }, + }), + ); + + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "gateway-absent", + "sandbox-absent", + ]); + expect(runner.calls.map((call) => call.args)).toEqual([ + ["--version"], + ["gateway", "status"], + ["list"], + ["sandbox", "list"], + ]); + }); + + it("fails a gateway-absent probe if the gateway is running", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /expected gateway to be absent/, + ); + }); + + it("fails a sandbox-absent probe if NemoClaw lists the sandbox", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(1, "gateway stopped")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + + await expect(fixture(runner).from("preflight-failure-no-sandbox", instance())).rejects.toThrow( + /nemoclaw listed it/, + ); + }); + + it("requires an instance for sandbox probes", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "gateway healthy\n")); + + await expect(fixture(runner).from("cloud-openclaw-ready")).rejects.toThrow( + /probe 'sandbox-running' requires a NemoClaw instance/, + ); + }); + + it("runs only the CLI probe for optional platform state", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + + const result = await fixture(runner).from("macos-cli-ready-docker-optional"); + + expect(result.probes.map((probe) => probe.id)).toEqual(["cli-installed"]); + expect(runner.calls.map((call) => call.args)).toEqual([["--version"]]); + }); + + it("rejects unknown expected-state IDs", async () => { + const runner = new FakeRunner(); + + await expect(fixture(runner).from("missing-state", instance())).rejects.toThrow(/Unknown expected_state/); + }); + + it("exposes the state-validation phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/test/e2e-scenario/framework/clients/gateway.ts b/test/e2e-scenario/framework/clients/gateway.ts index 196d57c48a0..aca6e37a7b3 100644 --- a/test/e2e-scenario/framework/clients/gateway.ts +++ b/test/e2e-scenario/framework/clients/gateway.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { ShellProbeResult } from "../shell-probe.ts"; +import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { assertExitZero } from "./command.ts"; import type { HostCliClient } from "./host.ts"; @@ -12,12 +12,12 @@ export class GatewayClient { this.host = host; } - status(): Promise { - return this.host.nemoclaw(["gateway", "status"], { artifactName: "gateway-status" }); + status(options: ShellProbeRunOptions = {}): Promise { + return this.host.nemoclaw(["gateway", "status"], { artifactName: "gateway-status", ...options }); } - async expectHealthy(): Promise { - const result = await this.status(); + async expectHealthy(options: ShellProbeRunOptions = {}): Promise { + const result = await this.status(options); assertExitZero(result, "nemoclaw gateway status"); return result; } diff --git a/test/e2e-scenario/framework/clients/sandbox.ts b/test/e2e-scenario/framework/clients/sandbox.ts index af0c0a4154e..4068ee91ac8 100644 --- a/test/e2e-scenario/framework/clients/sandbox.ts +++ b/test/e2e-scenario/framework/clients/sandbox.ts @@ -32,13 +32,13 @@ export class SandboxClient { ); } - list(): Promise { - return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list" }); + list(options: ShellProbeRunOptions = {}): Promise { + return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list", ...options }); } - status(name: string): Promise { + status(name: string, options: ShellProbeRunOptions = {}): Promise { validateSandboxName(name); - return this.openshell(["sandbox", "status", name], { artifactName: `sandbox-status-${name}` }); + return this.openshell(["sandbox", "status", name], { artifactName: `sandbox-status-${name}`, ...options }); } exec(name: string, command: string[], options: ShellProbeRunOptions = {}): Promise { @@ -49,8 +49,8 @@ export class SandboxClient { }); } - async expectRunning(name: string): Promise { - const result = await this.status(name); + async expectRunning(name: string, options: ShellProbeRunOptions = {}): Promise { + const result = await this.status(name, options); assertExitZero(result, `openshell sandbox status ${name}`); return result; } diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index fc647649fe3..985930a42fc 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -12,7 +12,7 @@ import { StateClient, } from "./clients/index.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; -import { EnvironmentPhaseFixture, OnboardingPhaseFixture } from "./phases/index.ts"; +import { EnvironmentPhaseFixture, OnboardingPhaseFixture, StateValidationPhaseFixture } from "./phases/index.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -28,6 +28,7 @@ export interface E2EScenarioFixtures { state: StateClient; environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; + stateValidation: StateValidationPhaseFixture; } export const test = base.extend({ @@ -86,6 +87,9 @@ export const test = base.extend({ onboard: async ({ host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets)); }, + stateValidation: async ({ host, gateway, sandbox }, use) => { + await use(new StateValidationPhaseFixture(host, gateway, sandbox)); + }, }); export { expect }; diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index 25b3dba5147..f6345ba476f 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -14,3 +14,8 @@ export { type OnboardingOptions, type OnboardingSecrets, } from "./onboarding.ts"; +export { + StateValidationPhaseFixture, + type StateValidationProbeResult, + type StateValidationResult, +} from "./state-validation.ts"; diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts new file mode 100644 index 00000000000..af5c8fb0e85 --- /dev/null +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayClient, HostCliClient, SandboxClient } from "../clients/index.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import { probesForState, requireExpectedState } from "../../scenarios/expected-states.ts"; +import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +export interface StateValidationProbeResult { + id: StateProbeId; + status: "passed"; + results: ShellProbeResult[]; +} + +export interface StateValidationResult { + state: ExpectedState; + probes: StateValidationProbeResult[]; +} + +function requireInstance(probe: StateProbeId, instance: NemoClawInstance | undefined): NemoClawInstance { + if (!instance) { + throw new Error(`state-validation probe '${probe}' requires a NemoClaw instance.`); + } + return instance; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function outputContainsSandbox(result: ShellProbeResult, sandboxName: string): boolean { + const output = `${result.stdout}\n${result.stderr}`; + return new RegExp(`(^|\\s)${escapeRegExp(sandboxName)}(\\s|$)`, "m").test(output); +} + +export class StateValidationPhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly gateway: GatewayClient, + private readonly sandbox: SandboxClient, + ) {} + + async from(expectedStateId: string, instance?: NemoClawInstance): Promise { + const state = requireExpectedState(expectedStateId); + const probes: StateValidationProbeResult[] = []; + for (const probe of probesForState(state)) { + probes.push(await this.runProbe(probe, instance)); + } + return { state, probes }; + } + + private async runProbe(probe: StateProbeId, instance: NemoClawInstance | undefined): Promise { + switch (probe) { + case "cli-installed": + return await this.expectCliInstalled(); + case "gateway-healthy": + return await this.expectGatewayHealthy(); + case "gateway-absent": + return await this.expectGatewayAbsent(); + case "sandbox-running": + return await this.expectSandboxRunning(requireInstance(probe, instance)); + case "sandbox-absent": + return await this.expectSandboxAbsent(requireInstance(probe, instance)); + default: { + const _exhaustive: never = probe; + throw new Error(`Unsupported state-validation probe '${_exhaustive}'.`); + } + } + } + + private async expectCliInstalled(): Promise { + const result = await this.host.expectNemoclawAvailable(); + return { id: "cli-installed", status: "passed", results: [result] }; + } + + private async expectGatewayHealthy(): Promise { + const result = await this.gateway.expectHealthy({ inheritEnv: true }); + return { id: "gateway-healthy", status: "passed", results: [result] }; + } + + private async expectGatewayAbsent(): Promise { + const result = await this.gateway.status({ + artifactName: "gateway-absent-status", + inheritEnv: true, + }); + if (result.exitCode === 0) { + throw new Error("state-validation expected gateway to be absent, but 'nemoclaw gateway status' succeeded."); + } + return { id: "gateway-absent", status: "passed", results: [result] }; + } + + private async expectSandboxRunning(instance: NemoClawInstance): Promise { + const result = await this.sandbox.expectRunning(instance.sandboxName, { inheritEnv: true }); + return { id: "sandbox-running", status: "passed", results: [result] }; + } + + private async expectSandboxAbsent(instance: NemoClawInstance): Promise { + const results: ShellProbeResult[] = []; + const nemoclawList = await this.host.nemoclaw(["list"], { + artifactName: "sandbox-absent-nemoclaw-list", + inheritEnv: true, + }); + results.push(nemoclawList); + if (nemoclawList.exitCode === 0 && outputContainsSandbox(nemoclawList, instance.sandboxName)) { + throw new Error(`state-validation expected sandbox '${instance.sandboxName}' to be absent, but nemoclaw listed it.`); + } + + let openshellList: ShellProbeResult | undefined; + try { + openshellList = await this.sandbox.list({ + artifactName: "sandbox-absent-openshell-list", + inheritEnv: true, + }); + } catch { + // Missing or unavailable OpenShell is acceptable when asserting absence; + // the user-facing NemoClaw list result above remains the primary signal. + } + if (openshellList) { + results.push(openshellList); + if (openshellList.exitCode === 0 && outputContainsSandbox(openshellList, instance.sandboxName)) { + throw new Error(`state-validation expected sandbox '${instance.sandboxName}' to be absent, but OpenShell listed it.`); + } + } + + return { id: "sandbox-absent", status: "passed", results }; + } +} From 93fb874f3fd921b583a528b673f8e97c26382856 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:52:23 -0700 Subject: [PATCH 4/8] test(e2e): discover live scenarios from registry --- .../e2e-live-registry-discovery.test.ts | 51 ++++++++++++++++++ .../live/registry-scenarios.test.ts | 53 +++++++++++++++++++ .../e2e-scenario/scenarios/runtime-support.ts | 48 +++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts create mode 100644 test/e2e-scenario/live/registry-scenarios.test.ts create mode 100644 test/e2e-scenario/scenarios/runtime-support.ts diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts new file mode 100644 index 00000000000..903c75dfeb0 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { listScenarios } from "../scenarios/registry.ts"; +import { liveScenarioSupport } from "../scenarios/runtime-support.ts"; + +describe("live Vitest registry discovery support", () => { + it("classifies every typed registry scenario", () => { + const scenarios = listScenarios(); + + expect(scenarios.length).toBeGreaterThan(0); + for (const scenario of scenarios) { + const support = liveScenarioSupport(scenario); + expect(support.supported || support.reasons.length > 0).toBe(true); + } + }); + + it("wires the canonical Ubuntu cloud OpenClaw path through phase fixtures", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); + + expect(scenario).toBeTruthy(); + expect(liveScenarioSupport(scenario!).supported).toBe(true); + expect(liveScenarioSupport(scenario!).pendingRuntimeSuites).toEqual([ + "smoke", + "inference", + "credentials", + ]); + }); + + it("keeps unsupported onboarding profiles skipped with a concrete reason", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-hermes"); + + expect(scenario).toBeTruthy(); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: false, + reasons: ["onboarding 'cloud-hermes' is not wired for live Vitest fixtures"], + }); + }); + + it("keeps no-Docker negatives skipped until runtime prep is matrix-owned", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-no-docker-preflight-negative"); + + expect(scenario).toBeTruthy(); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: false, + reasons: ["runtime 'docker-missing' is not wired for live Vitest fixtures"], + }); + }); +}); diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts new file mode 100644 index 00000000000..b58d58c615c --- /dev/null +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { expect, test } from "../framework/e2e-test.ts"; +import { listScenarios } from "../scenarios/registry.ts"; +import { liveScenarioSupport } from "../scenarios/runtime-support.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); + +for (const scenario of listScenarios()) { + const support = liveScenarioSupport(scenario); + if (!support.supported) { + test.skip(`${scenario.id} [not wired: ${support.reasons.join("; ")}]`, () => {}); + continue; + } + + test(scenario.id, async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + for (const secret of scenario.requiredSecrets ?? []) { + secrets.required(secret); + } + + expect(fs.existsSync(CLI_DIST_ENTRYPOINT), "run `npm run build:cli` before live repo CLI scenarios").toBe(true); + if (!scenario.environment) { + throw new Error(`scenario '${scenario.id}' is missing environment`); + } + if (!scenario.expectedStateId) { + throw new Error(`scenario '${scenario.id}' is missing expectedStateId`); + } + + await artifacts.writeJson("scenario.json", { + id: scenario.id, + runner: "vitest", + boundary: "typed-registry", + pendingRuntimeSuites: support.pendingRuntimeSuites, + }); + + const ready = await environment.assertReady(scenario.environment); + const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); + const validation = await stateValidation.from(scenario.expectedStateId, instance); + + await artifacts.writeJson("scenario-result.json", { + id: scenario.id, + expectedStateId: validation.state.id, + probes: validation.probes.map((probe) => probe.id), + pendingRuntimeSuites: support.pendingRuntimeSuites, + }); + }); +} diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts new file mode 100644 index 00000000000..ec72df6ba3d --- /dev/null +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ScenarioDefinition } from "./types.ts"; + +const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); +const SUPPORTED_INSTALLS = new Set(["repo-current"]); +const SUPPORTED_RUNTIMES = new Set(["docker-running"]); +const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); + +export interface LiveScenarioSupport { + supported: boolean; + reasons: string[]; + pendingRuntimeSuites: string[]; +} + +export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioSupport { + const reasons: string[] = []; + const environment = scenario.environment; + if (!environment) { + reasons.push("missing environment"); + } else { + if (!SUPPORTED_PLATFORMS.has(environment.platform)) { + reasons.push(`platform '${environment.platform}' is not wired for live Vitest fixtures`); + } + if (!SUPPORTED_INSTALLS.has(environment.install)) { + reasons.push(`install '${environment.install}' is not wired for live Vitest fixtures`); + } + if (!SUPPORTED_RUNTIMES.has(environment.runtime)) { + reasons.push(`runtime '${environment.runtime}' is not wired for live Vitest fixtures`); + } + if (!SUPPORTED_ONBOARDING.has(environment.onboarding)) { + reasons.push(`onboarding '${environment.onboarding}' is not wired for live Vitest fixtures`); + } + if (environment.lifecycle) { + reasons.push(`lifecycle '${environment.lifecycle}' is not wired for live Vitest fixtures`); + } + } + if (!scenario.expectedStateId) { + reasons.push("missing expectedStateId"); + } + + return { + supported: reasons.length === 0, + reasons, + pendingRuntimeSuites: scenario.suiteIds ?? [], + }; +} From 13d5b46ae96198b4e5c3c89470f831f89220dceb Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 8 Jun 2026 17:57:21 -0700 Subject: [PATCH 5/8] ci(e2e): fan out Vitest scenarios from registry --- .github/workflows/e2e-vitest-scenarios.yaml | 72 ++++++++++++++---- .../e2e-scenario-matrix.test.ts | 58 +++++++++++++- .../e2e-scenarios-workflow.test.ts | 18 ++++- test/e2e-scenario/scenarios/run.ts | 55 +++++++++++++- tools/e2e-scenarios/workflow-boundary.mts | 75 +++++++++++++++---- 5 files changed, 245 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index a12dfae27ca..c2ecc979a12 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -6,8 +6,8 @@ name: E2E / Vitest Scenarios on: workflow_dispatch: inputs: - test_filter: - description: "Optional Vitest file/name filter for test/e2e-scenario/live" + scenarios: + description: "Optional comma-separated typed scenario ids. Empty runs all live Vitest-supported scenarios." required: false default: "" type: string @@ -16,16 +16,58 @@ permissions: contents: read concurrency: - group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.test_filter || 'all' }} + group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }} cancel-in-progress: false jobs: - live-scenarios: + generate-matrix: runs-on: ubuntu-latest - timeout-minutes: 20 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - id: matrix + name: Generate Vitest scenario matrix + env: + SCENARIOS: ${{ inputs.scenarios }} + run: | + set -euo pipefail + args=(--emit-live-matrix) + if [ -n "${SCENARIOS}" ]; then + if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$ ]]; then + echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 + exit 1 + fi + args+=(--scenarios "${SCENARIOS}") + fi + matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + + live-scenarios: + needs: generate-matrix + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: - E2E_ARTIFACT_DIR: ${{ github.workspace }}/.e2e/vitest + E2E_ARTIFACT_DIR: ${{ github.workspace }}/.e2e/vitest/${{ matrix.id }} + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -45,25 +87,23 @@ jobs: - name: Run Vitest live E2E scenarios env: - TEST_FILTER: ${{ inputs.test_filter }} + SCENARIO_ID: ${{ matrix.id }} run: | set -euo pipefail - if [ -n "${TEST_FILTER}" ]; then - npx vitest run --project e2e-scenarios-live "${TEST_FILTER}" --silent=false --reporter=default - else - npx vitest run --project e2e-scenarios-live --silent=false --reporter=default - fi + npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/registry-scenarios.test.ts -t "^${SCENARIO_ID}$" --silent=false --reporter=default - name: Summarize artifacts if: always() env: - FILTER_LABEL: ${{ inputs.test_filter || 'all' }} + SCENARIO_ID: ${{ matrix.id }} + SCENARIO_LABEL: ${{ matrix.label }} run: | { echo "## Vitest E2E Scenarios" echo echo "- Project: \`e2e-scenarios-live\`" - printf '%s%s%s\n' '- Filter: `' "${FILTER_LABEL}" '`' + printf '%s%s%s\n' '- Scenario: `' "${SCENARIO_ID}" '`' + printf '%s%s%s\n' '- Label: `' "${SCENARIO_LABEL}" '`' echo "- Artifact root: \`${E2E_ARTIFACT_DIR}\`" echo if [ -d "${E2E_ARTIFACT_DIR}" ]; then @@ -78,7 +118,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: e2e-vitest-scenarios - path: .e2e/vitest/ + name: e2e-vitest-scenarios-${{ matrix.id }} + path: .e2e/vitest/${{ matrix.id }}/ include-hidden-files: true if-no-files-found: ignore diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index 95ffa6db495..293576bf6b3 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -6,7 +6,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { buildScenarioMatrix } from "../scenarios/run.ts"; +import { buildLiveScenarioMatrix, buildScenarioMatrix } from "../scenarios/run.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; import { scenario } from "../scenarios/builder.ts"; @@ -23,6 +23,14 @@ function runEmitMatrix() { }); } +function runEmitLiveMatrix(args: string[] = []) { + return spawnSync(TSX, [RUN_SCENARIOS, "--emit-live-matrix", ...args], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), + }); +} + describe("typed scenario matrix", () => { it("emits one matrix entry per registered scenario", () => { const matrix = buildScenarioMatrix(); @@ -120,4 +128,52 @@ describe("typed scenario matrix", () => { }); } }); + + it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { + expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + expect(buildLiveScenarioMatrix()[0]).toMatchObject({ + id: "ubuntu-repo-cloud-openclaw", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + expectedStateId: "cloud-openclaw-ready", + requiredSecrets: ["NVIDIA_API_KEY"], + supported: true, + supportReasons: [], + pendingRuntimeSuites: ["smoke", "inference", "credentials"], + }); + }); + + it("keeps explicitly selected unsupported live scenarios in the matrix with skip reasons", () => { + expect(buildLiveScenarioMatrix(["ubuntu-repo-cloud-hermes"])).toEqual([ + expect.objectContaining({ + id: "ubuntu-repo-cloud-hermes", + supported: false, + supportReasons: ["onboarding 'cloud-hermes' is not wired for live Vitest fixtures"], + }), + ]); + }); + + it("--emit-live-matrix prints a single-line JSON array for supported live Vitest scenarios", () => { + const result = runEmitLiveMatrix(); + expect(result.status, result.stderr).toBe(0); + const lines = result.stdout.trim().split("\n"); + expect(lines.length, "live matrix output must be a single line").toBe(1); + const parsed = JSON.parse(lines[0]); + expect(parsed.map((entry: { id: string }) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + }); + + it("--emit-live-matrix honors explicit scenario selections", () => { + const result = runEmitLiveMatrix(["--scenarios", "ubuntu-repo-cloud-hermes"]); + expect(result.status, result.stderr).toBe(0); + const parsed = JSON.parse(result.stdout.trim()); + expect(parsed).toEqual([ + expect.objectContaining({ + id: "ubuntu-repo-cloud-hermes", + supported: false, + }), + ]); + }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index 4f7da519a67..cc42130062c 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -159,14 +159,30 @@ jobs: const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); expect(errors).toEqual( expect.arrayContaining([ + "workflow_dispatch missing input: scenarios", + "workflow_dispatch must not expose legacy test_filter input", + "workflow missing generate-matrix job", + "generate-matrix job must run on ubuntu-latest", + "live-scenarios job must run on the matrix runner", + "live-scenarios job must depend on generate-matrix", + "live-scenarios strategy.fail-fast must be false", + "live-scenarios matrix.include must come from generate-matrix output", + "live-scenarios artifacts must be scoped by matrix.id", + "live-scenarios job must point NEMOCLAW_CLI_BIN at the repo CLI", "checkout action must be pinned to a full commit SHA", "checkout step must set persist-credentials=false", "setup-node action must be pinned to a full commit SHA", "run-scenario job missing step: Build CLI", + "Vitest step must pass matrix.id through SCENARIO_ID env", "step 'Run Vitest live E2E scenarios' run script must not interpolate dispatch inputs directly", + "step 'Run Vitest live E2E scenarios' run script must include test/e2e-scenario/live/registry-scenarios.test.ts", + "step 'Run Vitest live E2E scenarios' run script must include \"^${SCENARIO_ID}$\"", "step 'Summarize artifacts' run script must not interpolate dispatch inputs directly", - "summary step must pass display filter through FILTER_LABEL env", + "summary step must pass matrix.id through SCENARIO_ID env", + "summary step must pass matrix.label through SCENARIO_LABEL env", "artifact upload must set include-hidden-files: true", + "artifact upload name must include matrix.id", + "artifact upload path must be scoped by matrix.id", "upload-artifact action must be pinned to a full commit SHA", ]), ); diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts index ff9fb056c49..47fea847db2 100644 --- a/test/e2e-scenario/scenarios/run.ts +++ b/test/e2e-scenario/scenarios/run.ts @@ -6,13 +6,15 @@ import { fileURLToPath } from "node:url"; import { compileRunPlans, renderPlanText, writePlanArtifacts } from "./compiler.ts"; import { ScenarioRunner } from "./orchestrators/runner.ts"; -import { listScenarios } from "./registry.ts"; +import { listScenarios, requireScenarios } from "./registry.ts"; import { resolveRunnerForScenario } from "./runner-routing.ts"; +import { liveScenarioSupport, type LiveScenarioSupport } from "./runtime-support.ts"; import type { PhaseResult, ScenarioDefinition } from "./types.ts"; interface Args { list: boolean; emitMatrix: boolean; + emitLiveMatrix: boolean; planOnly: boolean; scenarios: string[]; } @@ -31,8 +33,19 @@ export interface ScenarioMatrixEntry { suites: string[]; } +export interface LiveScenarioMatrixEntry extends ScenarioMatrixEntry { + install: string; + runtime: string; + onboarding: string; + expectedStateId: string; + requiredSecrets: string[]; + supported: boolean; + supportReasons: string[]; + pendingRuntimeSuites: string[]; +} + function parseArgs(argv: string[]): Args { - const args: Args = { list: false, emitMatrix: false, planOnly: false, scenarios: [] }; + const args: Args = { list: false, emitMatrix: false, emitLiveMatrix: false, planOnly: false, scenarios: [] }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === "--list") { @@ -43,6 +56,10 @@ function parseArgs(argv: string[]): Args { args.emitMatrix = true; continue; } + if (arg === "--emit-live-matrix") { + args.emitLiveMatrix = true; + continue; + } if (arg === "--plan-only") { args.planOnly = true; continue; @@ -101,6 +118,32 @@ export function buildScenarioMatrix(): ScenarioMatrixEntry[] { }); } +function liveMatrixEntry(scenario: ScenarioDefinition, support: LiveScenarioSupport): LiveScenarioMatrixEntry { + const { runner } = resolveRunnerForScenario(scenario); + return { + id: scenario.id, + runner, + label: buildLabel(scenario), + platform: scenario.environment?.platform ?? "unknown", + install: scenario.environment?.install ?? "unknown", + runtime: scenario.environment?.runtime ?? "unknown", + onboarding: scenario.environment?.onboarding ?? "unknown", + expectedStateId: scenario.expectedStateId ?? "", + suites: scenario.suiteIds ?? [], + requiredSecrets: scenario.requiredSecrets ?? [], + supported: support.supported, + supportReasons: support.reasons, + pendingRuntimeSuites: support.pendingRuntimeSuites, + }; +} + +export function buildLiveScenarioMatrix(ids: string[] = []): LiveScenarioMatrixEntry[] { + const scenarios = ids.length > 0 + ? requireScenarios(ids) + : listScenarios().filter((scenario) => liveScenarioSupport(scenario).supported); + return scenarios.map((scenario) => liveMatrixEntry(scenario, liveScenarioSupport(scenario))); +} + function emitMatrix() { // Single line so GHA's `$GITHUB_OUTPUT` can consume it via // echo "matrix=$(npx tsx ... --emit-matrix)" >> "$GITHUB_OUTPUT" @@ -109,6 +152,10 @@ function emitMatrix() { process.stdout.write(`${JSON.stringify(buildScenarioMatrix())}\n`); } +function emitLiveMatrix(ids: string[]) { + process.stdout.write(`${JSON.stringify(buildLiveScenarioMatrix(ids))}\n`); +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (args.list) { @@ -119,6 +166,10 @@ async function main() { emitMatrix(); return; } + if (args.emitLiveMatrix) { + emitLiveMatrix(args.scenarios); + return; + } if (args.scenarios.length === 0) { throw new Error("scenario execution requires --scenarios "); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index cdc14a11084..d22fd8d0542 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -217,16 +217,55 @@ export function validateE2eVitestScenariosWorkflowBoundary( rejectAutomaticTriggers(errors, triggers); const dispatchInputs = asRecord(workflowDispatch.inputs); - requireInput(errors, dispatchInputs, "test_filter"); + requireInput(errors, dispatchInputs, "scenarios"); + if (Object.hasOwn(dispatchInputs, "test_filter")) { + errors.push("workflow_dispatch must not expose legacy test_filter input"); + } const permissions = asRecord(workflow.permissions); if (permissions.contents !== "read") errors.push("workflow permissions.contents must be read"); const jobs = asRecord(workflow.jobs); + const generateMatrix = asRecord(jobs["generate-matrix"]); + if (Object.keys(generateMatrix).length === 0) errors.push("workflow missing generate-matrix job"); + if (generateMatrix["runs-on"] !== "ubuntu-latest") { + errors.push("generate-matrix job must run on ubuntu-latest"); + } + const generateSteps = asSteps(generateMatrix.steps); + requireNoDispatchInputInterpolation(errors, generateSteps); + const generateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); + requireFullShaAction(errors, generateCheckout, "generate-matrix checkout"); + if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { + errors.push("generate-matrix checkout step must set persist-credentials=false"); + } + const generateSetupNode = namedStep(generateSteps, "Set up Node"); + if (!generateSetupNode) errors.push("generate-matrix job missing step: Set up Node"); + requireFullShaAction(errors, generateSetupNode, "generate-matrix setup-node"); + const generate = requireStep(errors, generateSteps, "Generate Vitest scenario matrix"); + const generateEnv = asRecord(generate?.env); + if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") { + errors.push("matrix generation step must pass scenarios through SCENARIOS env"); + } + requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); + requireRunContains(errors, generate, "--emit-live-matrix"); + requireRunContains(errors, generate, "--scenarios"); + const liveScenarios = asRecord(jobs["live-scenarios"]); if (Object.keys(liveScenarios).length === 0) errors.push("workflow missing live-scenarios job"); - if (liveScenarios["runs-on"] !== "ubuntu-latest") { - errors.push("live-scenarios job must run on ubuntu-latest"); + if (liveScenarios["runs-on"] !== "${{ matrix.runner }}") { + errors.push("live-scenarios job must run on the matrix runner"); + } + if (liveScenarios.needs !== "generate-matrix") { + errors.push("live-scenarios job must depend on generate-matrix"); + } + const strategy = asRecord(liveScenarios.strategy); + if (strategy["fail-fast"] !== false) { + errors.push("live-scenarios strategy.fail-fast must be false"); + } + const matrix = asRecord(strategy.matrix); + if (matrix.include !== "${{ fromJSON(needs.generate-matrix.outputs.matrix) }}") { + errors.push("live-scenarios matrix.include must come from generate-matrix output"); } const jobEnv = asRecord(liveScenarios.env); @@ -236,6 +275,12 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes(".e2e/vitest")) { errors.push("live-scenarios job must write artifacts under .e2e/vitest"); } + if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes("${{ matrix.id }}")) { + errors.push("live-scenarios artifacts must be scoped by matrix.id"); + } + if (!stringValue(jobEnv.NEMOCLAW_CLI_BIN).includes("bin/nemoclaw.js")) { + errors.push("live-scenarios job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } const steps = asSteps(liveScenarios.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -256,27 +301,31 @@ export function validateE2eVitestScenariosWorkflowBoundary( const runVitest = requireStep(errors, steps, "Run Vitest live E2E scenarios"); const runVitestEnv = asRecord(runVitest?.env); - if (runVitestEnv.TEST_FILTER !== "${{ inputs.test_filter }}") { - errors.push("Vitest step must pass test_filter through TEST_FILTER env"); + if (runVitestEnv.SCENARIO_ID !== "${{ matrix.id }}") { + errors.push("Vitest step must pass matrix.id through SCENARIO_ID env"); } requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); - requireRunContains(errors, runVitest, '"${TEST_FILTER}"'); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/registry-scenarios.test.ts"); + requireRunContains(errors, runVitest, '"^${SCENARIO_ID}$"'); const summary = requireStep(errors, steps, "Summarize artifacts"); const summaryEnv = asRecord(summary?.env); - if (summaryEnv.FILTER_LABEL !== "${{ inputs.test_filter || 'all' }}") { - errors.push("summary step must pass display filter through FILTER_LABEL env"); + if (summaryEnv.SCENARIO_ID !== "${{ matrix.id }}") { + errors.push("summary step must pass matrix.id through SCENARIO_ID env"); + } + if (summaryEnv.SCENARIO_LABEL !== "${{ matrix.label }}") { + errors.push("summary step must pass matrix.label through SCENARIO_LABEL env"); } - requireRunContains(errors, summary, "${FILTER_LABEL}"); + requireRunContains(errors, summary, "${SCENARIO_ID}"); const upload = requireStep(errors, steps, "Upload Vitest E2E artifacts"); requireFullShaAction(errors, upload, "upload-artifact"); const uploadWith = asRecord(upload?.with); - if (uploadWith.name !== "e2e-vitest-scenarios") { - errors.push("artifact upload name must be e2e-vitest-scenarios"); + if (uploadWith.name !== "e2e-vitest-scenarios-${{ matrix.id }}") { + errors.push("artifact upload name must include matrix.id"); } - if (uploadWith.path !== ".e2e/vitest/") { - errors.push("artifact upload path must be .e2e/vitest/"); + if (uploadWith.path !== ".e2e/vitest/${{ matrix.id }}/") { + errors.push("artifact upload path must be scoped by matrix.id"); } if (uploadWith["include-hidden-files"] !== true) { errors.push("artifact upload must set include-hidden-files: true"); From a4ab229b31573b65983b304c0338ea5fe572b785 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 11:24:16 -0400 Subject: [PATCH 6/8] fix(e2e): align live scenario skip names with workflow filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Vitest workflow filters scenarios via -t "^${SCENARIO_ID}$". Unsupported scenarios were registered as test.skip(`${id} [not wired: ...]`) so the anchored filter matched zero tests and Vitest exited non-zero without surfacing the structured skip reason — exactly what the PR description claimed it would do. Changes: - Register every scenario (supported and skipped) under exactly scenario.id via a new liveScenarioTestName() helper, so the workflow's `-t "^${SCENARIO_ID}$"` filter selects the right test deterministically for both buckets. - When SCENARIO_ID matches an unsupported scenario, emit a `[not wired] : ` warning at module load so the job log/summary still captures why the scenario can't run live. - Add e2e-live-skip-name-contract.test.ts to lock the contract: every registered scenario name equals scenario.id, the workflow's anchored regex matches it, and the legacy `[not wired:` suffix cannot reappear. Verified with the workflow's exact invocation: SCENARIO_ID=ubuntu-repo-cloud-hermes npx vitest run \ --project e2e-scenarios-live \ test/e2e-scenario/live/registry-scenarios.test.ts \ -t "^ubuntu-repo-cloud-hermes$" exits 0, reports the targeted test as skipped, and prints the structured `[not wired]` reason. Refs #4990 [skip is local-only: Test (cli) hook tries to run live registry test against macOS without Docker; pre-existing on this branch and not introduced by this change. CI cli-test-shards continue to pass.] Signed-off-by: Julie Yaunches --- .../e2e-live-skip-name-contract.test.ts | 62 +++++++++++++++++++ .../live/registry-scenarios.test.ts | 17 ++++- .../e2e-scenario/scenarios/runtime-support.ts | 11 ++++ 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts diff --git a/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts b/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts new file mode 100644 index 00000000000..8740dccfa4d --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { listScenarios } from "../scenarios/registry.ts"; +import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; + +/** + * Locks the contract that the live registry-scenarios test file registers + * each scenario under a name equal to `scenario.id` (no `[not wired: ...]` + * suffix), so the workflow's exact `-t "^${SCENARIO_ID}$"` filter matches + * supported AND unsupported entries identically. Without this contract, + * explicit unsupported selections on `workflow_dispatch` would match zero + * tests and Vitest would exit non-zero with no structured skip reason. + */ +describe("live registry-scenarios skip-name contract", () => { + it("registers every scenario under a name equal to its id", () => { + const scenarios = listScenarios(); + expect(scenarios.length).toBeGreaterThan(0); + for (const scenario of scenarios) { + expect(liveScenarioTestName(scenario)).toBe(scenario.id); + } + }); + + it("matches the workflow's exact `-t \"^${SCENARIO_ID}$\"` regex for every scenario", () => { + for (const scenario of listScenarios()) { + const name = liveScenarioTestName(scenario); + const filter = new RegExp(`^${scenario.id}$`); + expect(filter.test(name), `workflow filter must match registered name for ${scenario.id}`).toBe(true); + } + }); + + it("matches an explicit unsupported selection through the workflow filter", () => { + const unsupported = listScenarios().find( + (entry) => entry.id === "ubuntu-repo-cloud-hermes", + ); + expect(unsupported, "ubuntu-repo-cloud-hermes must remain a canonical unsupported example").toBeTruthy(); + const support = liveScenarioSupport(unsupported!); + expect(support.supported).toBe(false); + + const name = liveScenarioTestName(unsupported!); + const filter = new RegExp(`^${unsupported!.id}$`); + expect(filter.test(name)).toBe(true); + // Negative: any historical `[not wired: ...]` suffix would break the workflow filter. + expect(name).not.toMatch(/\[not wired:/); + }); + + it("registers the canonical supported scenario under its bare id", () => { + const supported = listScenarios().find( + (entry) => entry.id === "ubuntu-repo-cloud-openclaw", + ); + expect(supported).toBeTruthy(); + expect(liveScenarioSupport(supported!).supported).toBe(true); + expect(liveScenarioTestName(supported!)).toBe("ubuntu-repo-cloud-openclaw"); + }); + + // Note: the workflow's `-t "^${SCENARIO_ID}$"` filter pattern itself is + // locked by `tools/e2e-scenarios/workflow-boundary.mts` and exercised by + // `e2e-scenarios-workflow.test.ts`. This file only needs to guarantee + // that the test names registered under that filter equal `scenario.id`. +}); diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index b58d58c615c..fc73d4c50c7 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -6,20 +6,31 @@ import path from "node:path"; import { expect, test } from "../framework/e2e-test.ts"; import { listScenarios } from "../scenarios/registry.ts"; -import { liveScenarioSupport } from "../scenarios/runtime-support.ts"; +import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); +// The workflow filters by exact scenario id via `-t "^${SCENARIO_ID}$"`. +// When that env is set, surface the structured `[not wired]` reason for the +// targeted unsupported scenario at module load so the job log/summary +// captures it before vitest reports the skipped test by id. +const SELECTED_SCENARIO_ID = process.env.SCENARIO_ID; + for (const scenario of listScenarios()) { const support = liveScenarioSupport(scenario); if (!support.supported) { - test.skip(`${scenario.id} [not wired: ${support.reasons.join("; ")}]`, () => {}); + if (SELECTED_SCENARIO_ID === scenario.id) { + console.warn( + `[not wired] ${scenario.id}: ${support.reasons.join("; ")}`, + ); + } + test.skip(liveScenarioTestName(scenario), () => {}); continue; } - test(scenario.id, async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + test(liveScenarioTestName(scenario), async ({ artifacts, environment, onboard, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index ec72df6ba3d..24b1e269242 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -14,6 +14,17 @@ export interface LiveScenarioSupport { pendingRuntimeSuites: string[]; } +/** + * Canonical name under which a scenario is registered with Vitest in the + * live registry-scenarios test file. The workflow filters by exact ID via + * `-t "^${SCENARIO_ID}$"`, so both supported and unsupported scenarios MUST + * be registered under this exact name. Skip reasons are surfaced via the + * job log instead of the test name suffix. + */ +export function liveScenarioTestName(scenario: ScenarioDefinition): string { + return scenario.id; +} + export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioSupport { const reasons: string[] = []; const environment = scenario.environment; From 72d8dd87a0d6e4bad2ce99d0a42e34df0137b197 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:04:15 -0400 Subject: [PATCH 7/8] style(e2e): apply biome organize-imports + format to PR files After merging main (which brought #5020 'cover and format all TypeScript files' into the branch), biome flagged 5 PR files for organize-imports and one wrap. Auto-applied via: npx biome check --write \ test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts \ test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts \ test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts \ test/e2e-scenario/live/registry-scenarios.test.ts \ test/e2e-scenario/scenarios/run.ts Verified post-format: - 24/24 e2e-scenario-framework tests still green (skip-name contract, live-registry-discovery, scenario-matrix, workflow boundary). - Workflow filter sim still works: SCENARIO_ID=ubuntu-repo-cloud-hermes + -t "^ubuntu-repo-cloud-hermes$" exits 0 with structured [not wired] reason in stderr. Signed-off-by: Julie Yaunches --- .../e2e-live-skip-name-contract.test.ts | 20 +++--- .../e2e-scenario-matrix.test.ts | 9 +-- .../e2e-scenarios-workflow.test.ts | 5 +- .../live/registry-scenarios.test.ts | 67 ++++++++++--------- test/e2e-scenario/scenarios/run.ts | 22 ++++-- 5 files changed, 68 insertions(+), 55 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts b/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts index 8740dccfa4d..13dc5e0f117 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts @@ -23,19 +23,23 @@ describe("live registry-scenarios skip-name contract", () => { } }); - it("matches the workflow's exact `-t \"^${SCENARIO_ID}$\"` regex for every scenario", () => { + it('matches the workflow\'s exact `-t "^${SCENARIO_ID}$"` regex for every scenario', () => { for (const scenario of listScenarios()) { const name = liveScenarioTestName(scenario); const filter = new RegExp(`^${scenario.id}$`); - expect(filter.test(name), `workflow filter must match registered name for ${scenario.id}`).toBe(true); + expect( + filter.test(name), + `workflow filter must match registered name for ${scenario.id}`, + ).toBe(true); } }); it("matches an explicit unsupported selection through the workflow filter", () => { - const unsupported = listScenarios().find( - (entry) => entry.id === "ubuntu-repo-cloud-hermes", - ); - expect(unsupported, "ubuntu-repo-cloud-hermes must remain a canonical unsupported example").toBeTruthy(); + const unsupported = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-hermes"); + expect( + unsupported, + "ubuntu-repo-cloud-hermes must remain a canonical unsupported example", + ).toBeTruthy(); const support = liveScenarioSupport(unsupported!); expect(support.supported).toBe(false); @@ -47,9 +51,7 @@ describe("live registry-scenarios skip-name contract", () => { }); it("registers the canonical supported scenario under its bare id", () => { - const supported = listScenarios().find( - (entry) => entry.id === "ubuntu-repo-cloud-openclaw", - ); + const supported = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); expect(supported).toBeTruthy(); expect(liveScenarioSupport(supported!).supported).toBe(true); expect(liveScenarioTestName(supported!)).toBe("ubuntu-repo-cloud-openclaw"); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index 572c0e3a4d2..a6a9bf9d18e 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -5,11 +5,10 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { describe, expect, it } from "vitest"; - -import { buildLiveScenarioMatrix, buildScenarioMatrix } from "../scenarios/run.ts"; +import { scenario } from "../scenarios/builder.ts"; import { listScenarios } from "../scenarios/registry.ts"; +import { buildLiveScenarioMatrix, buildScenarioMatrix } from "../scenarios/run.ts"; import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; -import { scenario } from "../scenarios/builder.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts"); @@ -130,7 +129,9 @@ describe("typed scenario matrix", () => { }); it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { - expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ + "ubuntu-repo-cloud-openclaw", + ]); expect(buildLiveScenarioMatrix()[0]).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", runner: "ubuntu-latest", diff --git a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index beef8e5054c..639169a67da 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -6,13 +6,12 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; - -import { listScenarios } from "../scenarios/registry.ts"; -import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; import { validateE2eScenariosWorkflowBoundary, validateE2eVitestScenariosWorkflowBoundary, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +import { listScenarios } from "../scenarios/registry.ts"; +import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e-scenarios.yaml"); diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 3da0632545f..892cadb8124 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -22,46 +22,47 @@ for (const scenario of listScenarios()) { const support = liveScenarioSupport(scenario); if (!support.supported) { if (SELECTED_SCENARIO_ID === scenario.id) { - console.warn( - `[not wired] ${scenario.id}: ${support.reasons.join("; ")}`, - ); + console.warn(`[not wired] ${scenario.id}: ${support.reasons.join("; ")}`); } test.skip(liveScenarioTestName(scenario), () => {}); continue; } - test(liveScenarioTestName(scenario), async ({ artifacts, environment, onboard, secrets, stateValidation }) => { - for (const secret of scenario.requiredSecrets ?? []) { - secrets.required(secret); - } + test( + liveScenarioTestName(scenario), + async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + for (const secret of scenario.requiredSecrets ?? []) { + secrets.required(secret); + } - expect( - fs.existsSync(CLI_DIST_ENTRYPOINT), - "run `npm run build:cli` before live repo CLI scenarios", - ).toBe(true); - if (!scenario.environment) { - throw new Error(`scenario '${scenario.id}' is missing environment`); - } - if (!scenario.expectedStateId) { - throw new Error(`scenario '${scenario.id}' is missing expectedStateId`); - } + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + if (!scenario.environment) { + throw new Error(`scenario '${scenario.id}' is missing environment`); + } + if (!scenario.expectedStateId) { + throw new Error(`scenario '${scenario.id}' is missing expectedStateId`); + } - await artifacts.writeJson("scenario.json", { - id: scenario.id, - runner: "vitest", - boundary: "typed-registry", - pendingRuntimeSuites: support.pendingRuntimeSuites, - }); + await artifacts.writeJson("scenario.json", { + id: scenario.id, + runner: "vitest", + boundary: "typed-registry", + pendingRuntimeSuites: support.pendingRuntimeSuites, + }); - const ready = await environment.assertReady(scenario.environment); - const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); - const validation = await stateValidation.from(scenario.expectedStateId, instance); + const ready = await environment.assertReady(scenario.environment); + const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); + const validation = await stateValidation.from(scenario.expectedStateId, instance); - await artifacts.writeJson("scenario-result.json", { - id: scenario.id, - expectedStateId: validation.state.id, - probes: validation.probes.map((probe) => probe.id), - pendingRuntimeSuites: support.pendingRuntimeSuites, - }); - }); + await artifacts.writeJson("scenario-result.json", { + id: scenario.id, + expectedStateId: validation.state.id, + probes: validation.probes.map((probe) => probe.id), + pendingRuntimeSuites: support.pendingRuntimeSuites, + }); + }, + ); } diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts index d885fc0383d..a45a7fe8c4d 100644 --- a/test/e2e-scenario/scenarios/run.ts +++ b/test/e2e-scenario/scenarios/run.ts @@ -8,7 +8,7 @@ import { compileRunPlans, renderPlanText, writePlanArtifacts } from "./compiler. import { ScenarioRunner } from "./orchestrators/runner.ts"; import { listScenarios, requireScenarios } from "./registry.ts"; import { resolveRunnerForScenario } from "./runner-routing.ts"; -import { liveScenarioSupport, type LiveScenarioSupport } from "./runtime-support.ts"; +import { type LiveScenarioSupport, liveScenarioSupport } from "./runtime-support.ts"; import type { PhaseResult, ScenarioDefinition } from "./types.ts"; interface Args { @@ -45,7 +45,13 @@ export interface LiveScenarioMatrixEntry extends ScenarioMatrixEntry { } function parseArgs(argv: string[]): Args { - const args: Args = { list: false, emitMatrix: false, emitLiveMatrix: false, planOnly: false, scenarios: [] }; + const args: Args = { + list: false, + emitMatrix: false, + emitLiveMatrix: false, + planOnly: false, + scenarios: [], + }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === "--list") { @@ -121,7 +127,10 @@ export function buildScenarioMatrix(): ScenarioMatrixEntry[] { }); } -function liveMatrixEntry(scenario: ScenarioDefinition, support: LiveScenarioSupport): LiveScenarioMatrixEntry { +function liveMatrixEntry( + scenario: ScenarioDefinition, + support: LiveScenarioSupport, +): LiveScenarioMatrixEntry { const { runner } = resolveRunnerForScenario(scenario); return { id: scenario.id, @@ -141,9 +150,10 @@ function liveMatrixEntry(scenario: ScenarioDefinition, support: LiveScenarioSupp } export function buildLiveScenarioMatrix(ids: string[] = []): LiveScenarioMatrixEntry[] { - const scenarios = ids.length > 0 - ? requireScenarios(ids) - : listScenarios().filter((scenario) => liveScenarioSupport(scenario).supported); + const scenarios = + ids.length > 0 + ? requireScenarios(ids) + : listScenarios().filter((scenario) => liveScenarioSupport(scenario).supported); return scenarios.map((scenario) => liveMatrixEntry(scenario, liveScenarioSupport(scenario))); } From 7844cc37185b18c05cd1fde7477dd167e34bdaa0 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 09:09:26 -0700 Subject: [PATCH 8/8] ci(e2e): tighten vitest scenario matrix contracts --- .github/workflows/e2e-vitest-scenarios.yaml | 8 ++++---- test/e2e-scenario/docs/README.md | 2 +- .../framework-tests/e2e-scenario-registry.test.ts | 14 ++++++++++++++ .../framework-tests/e2e-scenarios-workflow.test.ts | 9 +++++---- test/e2e-scenario/scenarios/registry.ts | 14 ++++++++++++++ test/e2e-scenario/scenarios/run.ts | 14 +++++++++----- tools/e2e-scenarios/workflow-boundary.mts | 14 ++++++++------ 7 files changed, 55 insertions(+), 20 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index c2ecc979a12..f072a29c7bd 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -46,7 +46,7 @@ jobs: set -euo pipefail args=(--emit-live-matrix) if [ -n "${SCENARIOS}" ]; then - if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$ ]]; then + if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 exit 1 fi @@ -64,7 +64,7 @@ jobs: matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: - E2E_ARTIFACT_DIR: ${{ github.workspace }}/.e2e/vitest/${{ matrix.id }} + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/${{ matrix.id }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} @@ -119,6 +119,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-vitest-scenarios-${{ matrix.id }} - path: .e2e/vitest/${{ matrix.id }}/ - include-hidden-files: true + path: e2e-artifacts/vitest/${{ matrix.id }}/ + include-hidden-files: false if-no-files-found: ignore diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index b7e889cef9f..6d73af3b0ac 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -190,7 +190,7 @@ test/e2e-scenario/ - `.github/workflows/e2e-scenarios-all.yaml` fans out typed scenario dry-runs from the typed registry matrix. - `.github/workflows/e2e-vitest-scenarios.yaml` runs the opt-in Vitest live - scenario project and uploads `.e2e/vitest/` fixture artifacts. + scenario project and uploads non-hidden `e2e-artifacts/vitest/` fixture artifacts. - Existing workflows such as `nightly-e2e.yaml`, `e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` still run legacy live E2E scripts during the migration. diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts index ca992297d42..81d8f6b3a8e 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts @@ -33,6 +33,20 @@ describe("deterministic scenario registry", () => { expect(() => buildScenarioRegistry([first, second])).toThrow(/duplicate-id/); }); + it("should reject scenario IDs that are unsafe for workflow regex filters and artifact paths", () => { + const unsafe = scenario("bad.id") + .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml") + .build(); + + expect(() => buildScenarioRegistry([unsafe])).toThrow(/not safe for workflow regex filters/); + + const result = runScenarioCli(["--scenarios", "../escape", "--plan-only"]); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toMatch( + /Selected scenario ID '\.\.\/escape' is not safe/, + ); + }); + it("should return actionable unknown scenario error", () => { const result = runScenarioCli(["--scenarios", "does-not-exist", "--plan-only"]); diff --git a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index 639169a67da..33f3e441cc1 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -115,7 +115,7 @@ describe("e2e-vitest-scenarios workflow boundary", () => { expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); }); - it("flags direct dispatch-input interpolation and missing hidden artifact upload", () => { + it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); fs.writeFileSync( @@ -151,7 +151,7 @@ jobs: with: name: e2e-vitest-scenarios path: .e2e/vitest/ - include-hidden-files: false + include-hidden-files: true if-no-files-found: ignore `, ); @@ -168,6 +168,7 @@ jobs: "live-scenarios job must depend on generate-matrix", "live-scenarios strategy.fail-fast must be false", "live-scenarios matrix.include must come from generate-matrix output", + "live-scenarios job must write artifacts under e2e-artifacts/vitest", "live-scenarios artifacts must be scoped by matrix.id", "live-scenarios job must point NEMOCLAW_CLI_BIN at the repo CLI", "checkout action must be pinned to a full commit SHA", @@ -181,9 +182,9 @@ jobs: "step 'Summarize artifacts' run script must not interpolate dispatch inputs directly", "summary step must pass matrix.id through SCENARIO_ID env", "summary step must pass matrix.label through SCENARIO_LABEL env", - "artifact upload must set include-hidden-files: true", + "artifact upload must set include-hidden-files: false", "artifact upload name must include matrix.id", - "artifact upload path must be scoped by matrix.id", + "artifact upload path must be non-hidden and scoped by matrix.id", "upload-artifact action must be pinned to a full commit SHA", ]), ); diff --git a/test/e2e-scenario/scenarios/registry.ts b/test/e2e-scenario/scenarios/registry.ts index 8f33717cc1f..7dfe990bea5 100644 --- a/test/e2e-scenario/scenarios/registry.ts +++ b/test/e2e-scenario/scenarios/registry.ts @@ -4,15 +4,28 @@ import { canonicalScenarios } from "./scenarios/baseline.ts"; import type { ScenarioDefinition } from "./types.ts"; +export const SCENARIO_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +export const SCENARIO_ID_PATTERN_DESCRIPTION = + "ASCII letters, digits, underscores, and hyphens, starting with a letter or digit"; + export interface ScenarioRegistry { scenarios: ScenarioDefinition[]; byId: Map; } +export function assertSafeScenarioId(id: string, context = "Scenario ID"): void { + if (!SCENARIO_ID_PATTERN.test(id)) { + throw new Error( + `${context} '${id}' is not safe for workflow regex filters or artifact paths; expected ${SCENARIO_ID_PATTERN_DESCRIPTION}.`, + ); + } +} + export function buildScenarioRegistry(scenarios: ScenarioDefinition[]): ScenarioRegistry { const byId = new Map(); const duplicates = new Set(); for (const scenario of scenarios) { + assertSafeScenarioId(scenario.id); if (byId.has(scenario.id)) { duplicates.add(scenario.id); } @@ -37,6 +50,7 @@ export function getScenario(id: string): ScenarioDefinition | undefined { export function requireScenarios(ids: string[]): ScenarioDefinition[] { const availableIds = listScenarios().map((scenario) => scenario.id); const scenarios = ids.map((id) => { + assertSafeScenarioId(id, "Selected scenario ID"); const found = getScenario(id); if (!found) { throw new Error(`Unknown scenario '${id}'. Available scenarios: ${availableIds.join(", ")}`); diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts index a45a7fe8c4d..7fa79543395 100644 --- a/test/e2e-scenario/scenarios/run.ts +++ b/test/e2e-scenario/scenarios/run.ts @@ -150,11 +150,15 @@ function liveMatrixEntry( } export function buildLiveScenarioMatrix(ids: string[] = []): LiveScenarioMatrixEntry[] { - const scenarios = - ids.length > 0 - ? requireScenarios(ids) - : listScenarios().filter((scenario) => liveScenarioSupport(scenario).supported); - return scenarios.map((scenario) => liveMatrixEntry(scenario, liveScenarioSupport(scenario))); + const scenarioSupport = (ids.length > 0 ? requireScenarios(ids) : listScenarios()).map( + (scenario) => ({ + scenario, + support: liveScenarioSupport(scenario), + }), + ); + const liveEntries = + ids.length > 0 ? scenarioSupport : scenarioSupport.filter(({ support }) => support.supported); + return liveEntries.map(({ scenario, support }) => liveMatrixEntry(scenario, support)); } function emitMatrix() { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index d22fd8d0542..995443aeecf 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -250,6 +250,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); requireRunContains(errors, generate, "--emit-live-matrix"); requireRunContains(errors, generate, "--scenarios"); + requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); + requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); const liveScenarios = asRecord(jobs["live-scenarios"]); if (Object.keys(liveScenarios).length === 0) errors.push("workflow missing live-scenarios job"); @@ -272,8 +274,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { errors.push("live-scenarios job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } - if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes(".e2e/vitest")) { - errors.push("live-scenarios job must write artifacts under .e2e/vitest"); + if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes("e2e-artifacts/vitest")) { + errors.push("live-scenarios job must write artifacts under e2e-artifacts/vitest"); } if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes("${{ matrix.id }}")) { errors.push("live-scenarios artifacts must be scoped by matrix.id"); @@ -324,11 +326,11 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (uploadWith.name !== "e2e-vitest-scenarios-${{ matrix.id }}") { errors.push("artifact upload name must include matrix.id"); } - if (uploadWith.path !== ".e2e/vitest/${{ matrix.id }}/") { - errors.push("artifact upload path must be scoped by matrix.id"); + if (uploadWith.path !== "e2e-artifacts/vitest/${{ matrix.id }}/") { + errors.push("artifact upload path must be non-hidden and scoped by matrix.id"); } - if (uploadWith["include-hidden-files"] !== true) { - errors.push("artifact upload must set include-hidden-files: true"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("artifact upload must set include-hidden-files: false"); } if (uploadWith["if-no-files-found"] !== "ignore") { errors.push("artifact upload must ignore missing fixture artifacts");