diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7ee99364fb1..a93a5cde40d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -369,13 +369,19 @@ jobs: shell: bash run: bash scripts/check-dcode-profile-import-gate.sh + # The common live-Vitest invocation (fixed e2e-live project + standard + # reporters) is built by the trusted helper, which validates the test path + # and selector against traversal and shell metacharacters before running. + # Covered by test/e2e/support/live-vitest-invocation.test.ts. - name: Run live E2E tests env: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} TARGET_ID: ${{ matrix.id }} run: | set -euo pipefail - npx vitest run --project e2e-live test/e2e/live/registry-targets.test.ts -t "^${TARGET_ID}$" --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts + node --experimental-strip-types tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/registry-targets.test.ts \ + --selector "^${TARGET_ID}$" # The sanitizer reads raw traces only after checking the workflow-owned # runner-temp path, then writes the timing-only file into upload roots. diff --git a/test/e2e/support/live-vitest-invocation.test.ts b/test/e2e/support/live-vitest-invocation.test.ts new file mode 100644 index 00000000000..ef0bb2782e2 --- /dev/null +++ b/test/e2e/support/live-vitest-invocation.test.ts @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { + buildLiveVitestArgs, + LIVE_VITEST_PROJECT, + RISK_SIGNAL_REPORTER, + resolveChildExitCode, + validateLiveProject, + validateLiveSelector, + validateLiveTestPath, +} from "../../../tools/e2e/live-vitest-invocation.mts"; + +const HELPER = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../tools/e2e/live-vitest-invocation.mts", +); + +function runHelper(args: string[]) { + return spawnSync(process.execPath, ["--experimental-strip-types", HELPER, ...args], { + encoding: "utf-8", + }); +} + +describe("validateLiveProject (#6961)", () => { + it("accepts the live project and defaults to it", () => { + expect(validateLiveProject("e2e-live")).toBe(LIVE_VITEST_PROJECT); + expect(validateLiveProject(undefined)).toBe(LIVE_VITEST_PROJECT); + }); + + it("rejects any other project", () => { + for (const project of ["cli", "e2e-support", "e2e-live-extra", "integration"]) { + expect(() => validateLiveProject(project)).toThrow(/unsupported vitest project/); + } + }); +}); + +describe("validateLiveTestPath (#6961)", () => { + it("accepts a real live test path", () => { + expect(validateLiveTestPath("test/e2e/live/registry-targets.test.ts")).toBe( + "test/e2e/live/registry-targets.test.ts", + ); + }); + + it("rejects paths outside the live test root", () => { + expect(() => validateLiveTestPath("test/e2e/support/thing.test.ts")).toThrow( + /must be under test\/e2e\/live/, + ); + expect(() => validateLiveTestPath("src/lib/onboard.ts")).toThrow(/must be under/); + }); + + it("rejects '..' traversal", () => { + expect(() => validateLiveTestPath("test/e2e/live/../../../etc/passwd")).toThrow( + /has an unsupported character|traverse/, + ); + expect(() => validateLiveTestPath("test/e2e/live/../support/x.test.ts")).toThrow(/traverse/); + }); + + it("rejects absolute paths", () => { + expect(() => validateLiveTestPath("/etc/passwd")).toThrow(/unsupported character|absolute/); + }); + + it("rejects shell metacharacters", () => { + for (const bad of [ + "test/e2e/live/x.test.ts; rm -rf /", + "test/e2e/live/$(whoami).test.ts", + "test/e2e/live/x.test.ts && curl evil", + "test/e2e/live/`id`.test.ts", + "test/e2e/live/x.test.ts|cat", + ]) { + expect(() => validateLiveTestPath(bad)).toThrow(/unsupported character/); + } + }); + + it("requires a .test.ts file", () => { + expect(() => validateLiveTestPath("test/e2e/live/fixtures")).toThrow(/\.test\.ts/); + }); + + it("requires a non-empty path", () => { + expect(() => validateLiveTestPath("")).toThrow(/required/); + expect(() => validateLiveTestPath(undefined)).toThrow(/required/); + }); +}); + +describe("validateLiveSelector (#6961)", () => { + it("accepts anchored title patterns", () => { + expect(validateLiveSelector("^ubuntu-repo-cloud-openclaw$")).toBe( + "^ubuntu-repo-cloud-openclaw$", + ); + expect(validateLiveSelector("^skill-agent$")).toBe("^skill-agent$"); + }); + + it("rejects shell metacharacters in the expanded selector", () => { + for (const bad of [ + "^$(touch pwned)$", + "^x$; rm -rf /", + "^x$ && evil", + "^`id`$", + "^x|y$", + "^x>out$", + ]) { + expect(() => validateLiveSelector(bad)).toThrow(/unsupported character/); + } + }); + + it("requires a non-empty selector", () => { + expect(() => validateLiveSelector("")).toThrow(/required/); + expect(() => validateLiveSelector(undefined)).toThrow(/required/); + }); +}); + +describe("buildLiveVitestArgs (#6961)", () => { + it("builds the standard invocation from validated inputs", () => { + expect( + buildLiveVitestArgs({ + testPath: "test/e2e/live/registry-targets.test.ts", + selector: "^ubuntu-repo-cloud-openclaw$", + }), + ).toEqual([ + "vitest", + "run", + "--project", + "e2e-live", + "test/e2e/live/registry-targets.test.ts", + "-t", + "^ubuntu-repo-cloud-openclaw$", + "--silent=false", + "--reporter=default", + `--reporter=${RISK_SIGNAL_REPORTER}`, + ]); + }); + + it("fails closed on an invalid input before producing any argv", () => { + expect(() => + buildLiveVitestArgs({ + testPath: "test/e2e/live/x.test.ts", + selector: "^x$; rm -rf /", + }), + ).toThrow(/unsupported character/); + expect(() => + buildLiveVitestArgs({ + testPath: "test/e2e/support/x.test.ts", + selector: "^x$", + project: "e2e-live", + }), + ).toThrow(/must be under/); + }); +}); + +describe("CLI subcommand guard (#6961)", () => { + // A typo must never look like a passing E2E run: the previous guard only ran + // on `run` and fell through to a silent exit 0 for anything else. + it.each([ + "runx", + "ru", + "RUN", + "--test-path", + ])("fails on the unsupported subcommand %j instead of exiting 0", (subcommand) => { + const result = runHelper([subcommand]); + + expect(result.status).not.toBe(0); + expect(result.stderr).toMatch(/unsupported subcommand/); + expect(result.stderr).toMatch(/usage: live-vitest-invocation\.mts run/); + }); + + it("fails when no subcommand is given", () => { + const result = runHelper([]); + + expect(result.status).not.toBe(0); + expect(result.stderr).toMatch(/missing subcommand/); + }); + + it("rejects an invalid selector through the CLI rather than running vitest", () => { + const result = runHelper([ + "run", + "--test-path", + "test/e2e/live/registry-targets.test.ts", + "--selector", + "^x$; rm -rf /", + ]); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toMatch(/unsupported character/); + }); +}); + +describe("resolveChildExitCode (termination behavior, #6961)", () => { + it("passes a normal exit status straight through", () => { + expect(resolveChildExitCode({ status: 0 })).toBe(0); + expect(resolveChildExitCode({ status: 1 })).toBe(1); + expect(resolveChildExitCode({ status: 137 })).toBe(137); + }); + + it("maps a signal death to 128+signo like the shell it replaced", () => { + expect(resolveChildExitCode({ status: null, signal: "SIGKILL" })).toBe( + 128 + (os.constants.signals.SIGKILL as number), + ); + expect(resolveChildExitCode({ status: null, signal: "SIGTERM" })).toBe( + 128 + (os.constants.signals.SIGTERM as number), + ); + }); + + it("reports a spawn failure as a generic failure", () => { + expect(resolveChildExitCode({ status: null, error: new Error("ENOENT") })).toBe(1); + expect(resolveChildExitCode({ status: null })).toBe(1); + }); +}); diff --git a/tools/e2e/live-vitest-invocation.mts b/tools/e2e/live-vitest-invocation.mts new file mode 100644 index 00000000000..296bfa2aa5c --- /dev/null +++ b/tools/e2e/live-vitest-invocation.mts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Validated construction of the common live-Vitest invocation (#6961). + * + * `.github/workflows/e2e.yaml` repeats the same `npx vitest run --project + * e2e-live … --reporter=default --reporter=test/e2e/risk-signal-reporter.ts` + * shape across many jobs. This helper owns that shape so a job supplies only a + * validated test path and `-t` selector; the project, reporters, and silence + * flag are fixed here. + * + * The inputs cross a trust boundary — a job passes matrix-derived values — so + * they are validated before any command is built: the project must be the + * expected live project, the test path must resolve under `test/e2e/live/` + * without traversal, and neither the path nor the selector may contain shell + * metacharacters. Rejection is fail-closed; nothing is quoted-away and run. + */ + +import { spawnSync } from "node:child_process"; +import os from "node:os"; +import { pathToFileURL } from "node:url"; + +import { parseArgs } from "../advisors/io.mts"; + +export const LIVE_VITEST_PROJECT = "e2e-live"; +export const LIVE_TEST_ROOT = "test/e2e/live/"; +export const RISK_SIGNAL_REPORTER = "test/e2e/risk-signal-reporter.ts"; + +// Anything outside this set can begin a shell word-split, redirection, +// substitution, or quote breakout. The selector intentionally allows `^`, `$`, +// and `-` (anchored Vitest title patterns) but nothing that reaches the shell. +const SHELL_METACHARACTER = /[^A-Za-z0-9_./^$=:@+-]/u; +const TEST_PATH_PATTERN = /^[A-Za-z0-9_./-]+$/u; + +export interface LiveVitestInvocation { + testPath: string | undefined; + selector: string | undefined; + project?: string; +} + +function assertNoShellMetacharacters(value: string, field: string): void { + const match = SHELL_METACHARACTER.exec(value); + if (match) { + throw new Error(`${field} contains an unsupported character ${JSON.stringify(match[0])}`); + } +} + +/** Validate the `--project` value is the one live project this helper serves. */ +export function validateLiveProject(project: string | undefined): string { + const resolved = (project ?? LIVE_VITEST_PROJECT).trim(); + if (resolved !== LIVE_VITEST_PROJECT) { + throw new Error( + `unsupported vitest project ${JSON.stringify(resolved)}; this helper only runs ${LIVE_VITEST_PROJECT}`, + ); + } + return resolved; +} + +/** + * Validate a live test path: under `test/e2e/live/`, a real `.test.ts` file + * name, no `..` traversal, no shell metacharacters, no absolute path. + */ +export function validateLiveTestPath(testPath: string | undefined): string { + const value = (testPath ?? "").trim(); + if (!value) throw new Error("test path is required"); + if (!TEST_PATH_PATTERN.test(value)) { + assertNoShellMetacharacters(value, "test path"); + throw new Error(`test path ${JSON.stringify(value)} has an unsupported character`); + } + if (value.startsWith("/")) { + throw new Error("test path must be repository-relative, not absolute"); + } + if (value.split("/").includes("..")) { + throw new Error("test path must not traverse with '..'"); + } + if (!value.startsWith(LIVE_TEST_ROOT)) { + throw new Error(`test path must be under ${LIVE_TEST_ROOT}, got ${JSON.stringify(value)}`); + } + if (!value.endsWith(".test.ts")) { + throw new Error("test path must name a .test.ts file"); + } + return value; +} + +/** + * Validate a Vitest `-t` selector. Anchored title patterns like `^${TARGET_ID}$` + * are expected (the shell expands `TARGET_ID` before this sees it), so the + * expanded value must still be free of shell metacharacters. + */ +export function validateLiveSelector(selector: string | undefined): string { + const value = (selector ?? "").trim(); + if (!value) throw new Error("selector is required"); + assertNoShellMetacharacters(value, "selector"); + return value; +} + +/** + * Build the argv for the common live-Vitest invocation from validated inputs. + * Returned as an argv array (never a shell string) so the caller can spawn it + * without a shell. + */ +export function buildLiveVitestArgs(invocation: LiveVitestInvocation): string[] { + const project = validateLiveProject(invocation.project); + const testPath = validateLiveTestPath(invocation.testPath); + const selector = validateLiveSelector(invocation.selector); + return [ + "vitest", + "run", + "--project", + project, + testPath, + "-t", + selector, + "--silent=false", + "--reporter=default", + `--reporter=${RISK_SIGNAL_REPORTER}`, + ]; +} + +// ── CLI ────────────────────────────────────────────────────────────────────── + +export const LIVE_VITEST_USAGE = + "usage: live-vitest-invocation.mts run --test-path --selector [--project e2e-live]"; + +/** + * Translate a finished child process into this process's exit code, preserving + * the shell's termination semantics. + * + * A bare `npx vitest …` under `set -euo pipefail` surfaces a signal death as + * 128+signo, not as a generic failure. Collapsing that to 1 would make a killed + * or OOM-reaped test run indistinguishable from an ordinary test failure. + */ +export function resolveChildExitCode(result: { + status: number | null; + signal?: NodeJS.Signals | null; + error?: Error; +}): number { + if (result.error) return 1; + if (typeof result.status === "number") return result.status; + if (result.signal) { + const signo = os.constants.signals[result.signal]; + return typeof signo === "number" ? 128 + signo : 1; + } + return 1; +} + +function runCli(): void { + const args = parseArgs(process.argv.slice(3)); + const argv = buildLiveVitestArgs({ + testPath: args.testPath, + selector: args.selector, + project: args.project, + }); + // Run through the repository's pinned vitest binary, without a shell, so the + // validated argv is passed verbatim. + const result = spawnSync("npx", argv, { stdio: "inherit" }); + if (result.error) { + console.error(`failed to spawn vitest: ${result.error.message}`); + } + process.exit(resolveChildExitCode(result)); +} + +function main(): void { + const subcommand = process.argv[2]; + // Fail closed on a missing or unknown subcommand. Exiting 0 here would let a + // typo silently skip the live E2E run while the job still reported success. + if (subcommand !== "run") { + console.error( + subcommand + ? `unsupported subcommand ${JSON.stringify(subcommand)}\n${LIVE_VITEST_USAGE}` + : `missing subcommand\n${LIVE_VITEST_USAGE}`, + ); + process.exit(2); + } + runCli(); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 8681ac60d7a..91a2e252dd0 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -4005,9 +4005,19 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { errors.push("live E2E step must receive NVIDIA_INFERENCE_API_KEY from secrets"); } - requireRunContains(errors, runVitest, "npx vitest run --project e2e-live"); - requireRunContains(errors, runVitest, "test/e2e/live/registry-targets.test.ts"); - requireRunContains(errors, runVitest, '"^${TARGET_ID}$"'); + // The common live-Vitest invocation is built by the trusted helper (#6961), + // which fixes the e2e-live project and reporters and validates the path and + // selector before running. Match the subcommand as a whole word: a substring + // check would also accept a typo like `... .mts runx`, which the helper + // rejects but which must never reach the workflow in the first place. + if ( + runVitest && + !/tools\/e2e\/live-vitest-invocation\.mts run(?![\w-])/u.test(stringValue(runVitest.run)) + ) { + errors.push("live E2E step must invoke tools/e2e/live-vitest-invocation.mts run"); + } + requireRunContains(errors, runVitest, "--test-path test/e2e/live/registry-targets.test.ts"); + requireRunContains(errors, runVitest, '--selector "^${TARGET_ID}$"'); const sanitizeTrace = requireStep(errors, steps, "Build trusted live E2E timing summary"); const sanitizeTraceEnv = asRecord(sanitizeTrace?.env);