diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index a12dfae27c..f072a29c7b 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -6,8 +6,8 @@ name: E2E / Vitest Scenarios on: workflow_dispatch: inputs: - test_filter: - description: "Optional Vitest file/name filter for test/e2e-scenario/live" + scenarios: + description: "Optional comma-separated typed scenario ids. Empty runs all live Vitest-supported scenarios." required: false default: "" type: string @@ -16,16 +16,58 @@ permissions: contents: read concurrency: - group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.test_filter || 'all' }} + group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }} cancel-in-progress: false jobs: - live-scenarios: + generate-matrix: runs-on: ubuntu-latest - timeout-minutes: 20 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - id: matrix + name: Generate Vitest scenario matrix + env: + SCENARIOS: ${{ inputs.scenarios }} + run: | + set -euo pipefail + args=(--emit-live-matrix) + if [ -n "${SCENARIOS}" ]; then + if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then + echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 + exit 1 + fi + args+=(--scenarios "${SCENARIOS}") + fi + matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + + live-scenarios: + needs: generate-matrix + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: - E2E_ARTIFACT_DIR: ${{ github.workspace }}/.e2e/vitest + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/${{ matrix.id }} + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -45,25 +87,23 @@ jobs: - name: Run Vitest live E2E scenarios env: - TEST_FILTER: ${{ inputs.test_filter }} + SCENARIO_ID: ${{ matrix.id }} run: | set -euo pipefail - if [ -n "${TEST_FILTER}" ]; then - npx vitest run --project e2e-scenarios-live "${TEST_FILTER}" --silent=false --reporter=default - else - npx vitest run --project e2e-scenarios-live --silent=false --reporter=default - fi + npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/registry-scenarios.test.ts -t "^${SCENARIO_ID}$" --silent=false --reporter=default - name: Summarize artifacts if: always() env: - FILTER_LABEL: ${{ inputs.test_filter || 'all' }} + SCENARIO_ID: ${{ matrix.id }} + SCENARIO_LABEL: ${{ matrix.label }} run: | { echo "## Vitest E2E Scenarios" echo echo "- Project: \`e2e-scenarios-live\`" - printf '%s%s%s\n' '- Filter: `' "${FILTER_LABEL}" '`' + printf '%s%s%s\n' '- Scenario: `' "${SCENARIO_ID}" '`' + printf '%s%s%s\n' '- Label: `' "${SCENARIO_LABEL}" '`' echo "- Artifact root: \`${E2E_ARTIFACT_DIR}\`" echo if [ -d "${E2E_ARTIFACT_DIR}" ]; then @@ -78,7 +118,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: e2e-vitest-scenarios - path: .e2e/vitest/ - include-hidden-files: true + name: e2e-vitest-scenarios-${{ matrix.id }} + path: e2e-artifacts/vitest/${{ matrix.id }}/ + include-hidden-files: false if-no-files-found: ignore diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index b7e889cef9..6d73af3b0a 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -190,7 +190,7 @@ test/e2e-scenario/ - `.github/workflows/e2e-scenarios-all.yaml` fans out typed scenario dry-runs from the typed registry matrix. - `.github/workflows/e2e-vitest-scenarios.yaml` runs the opt-in Vitest live - scenario project and uploads `.e2e/vitest/` fixture artifacts. + scenario project and uploads non-hidden `e2e-artifacts/vitest/` fixture artifacts. - Existing workflows such as `nightly-e2e.yaml`, `e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` still run legacy live E2E scripts during the migration. diff --git a/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts b/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts new file mode 100644 index 0000000000..13dc5e0f11 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { listScenarios } from "../scenarios/registry.ts"; +import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; + +/** + * Locks the contract that the live registry-scenarios test file registers + * each scenario under a name equal to `scenario.id` (no `[not wired: ...]` + * suffix), so the workflow's exact `-t "^${SCENARIO_ID}$"` filter matches + * supported AND unsupported entries identically. Without this contract, + * explicit unsupported selections on `workflow_dispatch` would match zero + * tests and Vitest would exit non-zero with no structured skip reason. + */ +describe("live registry-scenarios skip-name contract", () => { + it("registers every scenario under a name equal to its id", () => { + const scenarios = listScenarios(); + expect(scenarios.length).toBeGreaterThan(0); + for (const scenario of scenarios) { + expect(liveScenarioTestName(scenario)).toBe(scenario.id); + } + }); + + it('matches the workflow\'s exact `-t "^${SCENARIO_ID}$"` regex for every scenario', () => { + for (const scenario of listScenarios()) { + const name = liveScenarioTestName(scenario); + const filter = new RegExp(`^${scenario.id}$`); + expect( + filter.test(name), + `workflow filter must match registered name for ${scenario.id}`, + ).toBe(true); + } + }); + + it("matches an explicit unsupported selection through the workflow filter", () => { + const unsupported = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-hermes"); + expect( + unsupported, + "ubuntu-repo-cloud-hermes must remain a canonical unsupported example", + ).toBeTruthy(); + const support = liveScenarioSupport(unsupported!); + expect(support.supported).toBe(false); + + const name = liveScenarioTestName(unsupported!); + const filter = new RegExp(`^${unsupported!.id}$`); + expect(filter.test(name)).toBe(true); + // Negative: any historical `[not wired: ...]` suffix would break the workflow filter. + expect(name).not.toMatch(/\[not wired:/); + }); + + it("registers the canonical supported scenario under its bare id", () => { + const supported = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); + expect(supported).toBeTruthy(); + expect(liveScenarioSupport(supported!).supported).toBe(true); + expect(liveScenarioTestName(supported!)).toBe("ubuntu-repo-cloud-openclaw"); + }); + + // Note: the workflow's `-t "^${SCENARIO_ID}$"` filter pattern itself is + // locked by `tools/e2e-scenarios/workflow-boundary.mts` and exercised by + // `e2e-scenarios-workflow.test.ts`. This file only needs to guarantee + // that the test names registered under that filter equal `scenario.id`. +}); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index 0f47c371af..a6a9bf9d18 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -5,11 +5,10 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { describe, expect, it } from "vitest"; - -import { buildScenarioMatrix } from "../scenarios/run.ts"; +import { scenario } from "../scenarios/builder.ts"; import { listScenarios } from "../scenarios/registry.ts"; +import { buildLiveScenarioMatrix, buildScenarioMatrix } from "../scenarios/run.ts"; import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; -import { scenario } from "../scenarios/builder.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts"); @@ -23,6 +22,14 @@ function runEmitMatrix() { }); } +function runEmitLiveMatrix(args: string[] = []) { + return spawnSync(TSX, [RUN_SCENARIOS, "--emit-live-matrix", ...args], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), + }); +} + describe("typed scenario matrix", () => { it("emits one matrix entry per registered scenario", () => { const matrix = buildScenarioMatrix(); @@ -120,4 +127,54 @@ describe("typed scenario matrix", () => { }); } }); + + it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { + expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ + "ubuntu-repo-cloud-openclaw", + ]); + expect(buildLiveScenarioMatrix()[0]).toMatchObject({ + id: "ubuntu-repo-cloud-openclaw", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + expectedStateId: "cloud-openclaw-ready", + requiredSecrets: ["NVIDIA_API_KEY"], + supported: true, + supportReasons: [], + pendingRuntimeSuites: ["smoke", "inference", "credentials"], + }); + }); + + it("keeps explicitly selected unsupported live scenarios in the matrix with skip reasons", () => { + expect(buildLiveScenarioMatrix(["ubuntu-repo-cloud-hermes"])).toEqual([ + expect.objectContaining({ + id: "ubuntu-repo-cloud-hermes", + supported: false, + supportReasons: ["onboarding 'cloud-hermes' is not wired for live Vitest fixtures"], + }), + ]); + }); + + it("--emit-live-matrix prints a single-line JSON array for supported live Vitest scenarios", () => { + const result = runEmitLiveMatrix(); + expect(result.status, result.stderr).toBe(0); + const lines = result.stdout.trim().split("\n"); + expect(lines.length, "live matrix output must be a single line").toBe(1); + const parsed = JSON.parse(lines[0]); + expect(parsed.map((entry: { id: string }) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + }); + + it("--emit-live-matrix honors explicit scenario selections", () => { + const result = runEmitLiveMatrix(["--scenarios", "ubuntu-repo-cloud-hermes"]); + expect(result.status, result.stderr).toBe(0); + const parsed = JSON.parse(result.stdout.trim()); + expect(parsed).toEqual([ + expect.objectContaining({ + id: "ubuntu-repo-cloud-hermes", + supported: false, + }), + ]); + }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts index ca992297d4..81d8f6b3a8 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts @@ -33,6 +33,20 @@ describe("deterministic scenario registry", () => { expect(() => buildScenarioRegistry([first, second])).toThrow(/duplicate-id/); }); + it("should reject scenario IDs that are unsafe for workflow regex filters and artifact paths", () => { + const unsafe = scenario("bad.id") + .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml") + .build(); + + expect(() => buildScenarioRegistry([unsafe])).toThrow(/not safe for workflow regex filters/); + + const result = runScenarioCli(["--scenarios", "../escape", "--plan-only"]); + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toMatch( + /Selected scenario ID '\.\.\/escape' is not safe/, + ); + }); + it("should return actionable unknown scenario error", () => { const result = runScenarioCli(["--scenarios", "does-not-exist", "--plan-only"]); diff --git a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index 260ee767b6..33f3e441cc 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -6,13 +6,12 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; - -import { listScenarios } from "../scenarios/registry.ts"; -import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; import { validateE2eScenariosWorkflowBoundary, validateE2eVitestScenariosWorkflowBoundary, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +import { listScenarios } from "../scenarios/registry.ts"; +import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e-scenarios.yaml"); @@ -116,7 +115,7 @@ describe("e2e-vitest-scenarios workflow boundary", () => { expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); }); - it("flags direct dispatch-input interpolation and missing hidden artifact upload", () => { + it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); fs.writeFileSync( @@ -152,7 +151,7 @@ jobs: with: name: e2e-vitest-scenarios path: .e2e/vitest/ - include-hidden-files: false + include-hidden-files: true if-no-files-found: ignore `, ); @@ -161,14 +160,31 @@ jobs: const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); expect(errors).toEqual( expect.arrayContaining([ + "workflow_dispatch missing input: scenarios", + "workflow_dispatch must not expose legacy test_filter input", + "workflow missing generate-matrix job", + "generate-matrix job must run on ubuntu-latest", + "live-scenarios job must run on the matrix runner", + "live-scenarios job must depend on generate-matrix", + "live-scenarios strategy.fail-fast must be false", + "live-scenarios matrix.include must come from generate-matrix output", + "live-scenarios job must write artifacts under e2e-artifacts/vitest", + "live-scenarios artifacts must be scoped by matrix.id", + "live-scenarios job must point NEMOCLAW_CLI_BIN at the repo CLI", "checkout action must be pinned to a full commit SHA", "checkout step must set persist-credentials=false", "setup-node action must be pinned to a full commit SHA", "run-scenario job missing step: Build CLI", + "Vitest step must pass matrix.id through SCENARIO_ID env", "step 'Run Vitest live E2E scenarios' run script must not interpolate dispatch inputs directly", + "step 'Run Vitest live E2E scenarios' run script must include test/e2e-scenario/live/registry-scenarios.test.ts", + "step 'Run Vitest live E2E scenarios' run script must include \"^${SCENARIO_ID}$\"", "step 'Summarize artifacts' run script must not interpolate dispatch inputs directly", - "summary step must pass display filter through FILTER_LABEL env", - "artifact upload must set include-hidden-files: true", + "summary step must pass matrix.id through SCENARIO_ID env", + "summary step must pass matrix.label through SCENARIO_LABEL env", + "artifact upload must set include-hidden-files: false", + "artifact upload name must include matrix.id", + "artifact upload path must be non-hidden and scoped by matrix.id", "upload-artifact action must be pinned to a full commit SHA", ]), ); diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 0740764845..892cadb812 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -6,51 +6,63 @@ import path from "node:path"; import { expect, test } from "../framework/e2e-test.ts"; import { listScenarios } from "../scenarios/registry.ts"; -import { liveScenarioSupport } from "../scenarios/runtime-support.ts"; +import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); +// The workflow filters by exact scenario id via `-t "^${SCENARIO_ID}$"`. +// When that env is set, surface the structured `[not wired]` reason for the +// targeted unsupported scenario at module load so the job log/summary +// captures it before vitest reports the skipped test by id. +const SELECTED_SCENARIO_ID = process.env.SCENARIO_ID; + for (const scenario of listScenarios()) { const support = liveScenarioSupport(scenario); if (!support.supported) { - test.skip(`${scenario.id} [not wired: ${support.reasons.join("; ")}]`, () => {}); + if (SELECTED_SCENARIO_ID === scenario.id) { + console.warn(`[not wired] ${scenario.id}: ${support.reasons.join("; ")}`); + } + test.skip(liveScenarioTestName(scenario), () => {}); continue; } - test(scenario.id, async ({ artifacts, environment, onboard, secrets, stateValidation }) => { - for (const secret of scenario.requiredSecrets ?? []) { - secrets.required(secret); - } + test( + liveScenarioTestName(scenario), + async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + for (const secret of scenario.requiredSecrets ?? []) { + secrets.required(secret); + } - expect( - fs.existsSync(CLI_DIST_ENTRYPOINT), - "run `npm run build:cli` before live repo CLI scenarios", - ).toBe(true); - if (!scenario.environment) { - throw new Error(`scenario '${scenario.id}' is missing environment`); - } - if (!scenario.expectedStateId) { - throw new Error(`scenario '${scenario.id}' is missing expectedStateId`); - } + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + if (!scenario.environment) { + throw new Error(`scenario '${scenario.id}' is missing environment`); + } + if (!scenario.expectedStateId) { + throw new Error(`scenario '${scenario.id}' is missing expectedStateId`); + } + + await artifacts.writeJson("scenario.json", { + id: scenario.id, + runner: "vitest", + boundary: "typed-registry", + pendingRuntimeSuites: support.pendingRuntimeSuites, + }); + + const ready = await environment.assertReady(scenario.environment); + const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); + const validation = await stateValidation.from(scenario.expectedStateId, instance); - await artifacts.writeJson("scenario.json", { - id: scenario.id, - runner: "vitest", - boundary: "typed-registry", - pendingRuntimeSuites: support.pendingRuntimeSuites, - }); - - const ready = await environment.assertReady(scenario.environment); - const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); - const validation = await stateValidation.from(scenario.expectedStateId, instance); - - await artifacts.writeJson("scenario-result.json", { - id: scenario.id, - expectedStateId: validation.state.id, - probes: validation.probes.map((probe) => probe.id), - pendingRuntimeSuites: support.pendingRuntimeSuites, - }); - }); + await artifacts.writeJson("scenario-result.json", { + id: scenario.id, + expectedStateId: validation.state.id, + probes: validation.probes.map((probe) => probe.id), + pendingRuntimeSuites: support.pendingRuntimeSuites, + }); + }, + ); } diff --git a/test/e2e-scenario/scenarios/registry.ts b/test/e2e-scenario/scenarios/registry.ts index 8f33717cc1..7dfe990bea 100644 --- a/test/e2e-scenario/scenarios/registry.ts +++ b/test/e2e-scenario/scenarios/registry.ts @@ -4,15 +4,28 @@ import { canonicalScenarios } from "./scenarios/baseline.ts"; import type { ScenarioDefinition } from "./types.ts"; +export const SCENARIO_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; +export const SCENARIO_ID_PATTERN_DESCRIPTION = + "ASCII letters, digits, underscores, and hyphens, starting with a letter or digit"; + export interface ScenarioRegistry { scenarios: ScenarioDefinition[]; byId: Map; } +export function assertSafeScenarioId(id: string, context = "Scenario ID"): void { + if (!SCENARIO_ID_PATTERN.test(id)) { + throw new Error( + `${context} '${id}' is not safe for workflow regex filters or artifact paths; expected ${SCENARIO_ID_PATTERN_DESCRIPTION}.`, + ); + } +} + export function buildScenarioRegistry(scenarios: ScenarioDefinition[]): ScenarioRegistry { const byId = new Map(); const duplicates = new Set(); for (const scenario of scenarios) { + assertSafeScenarioId(scenario.id); if (byId.has(scenario.id)) { duplicates.add(scenario.id); } @@ -37,6 +50,7 @@ export function getScenario(id: string): ScenarioDefinition | undefined { export function requireScenarios(ids: string[]): ScenarioDefinition[] { const availableIds = listScenarios().map((scenario) => scenario.id); const scenarios = ids.map((id) => { + assertSafeScenarioId(id, "Selected scenario ID"); const found = getScenario(id); if (!found) { throw new Error(`Unknown scenario '${id}'. Available scenarios: ${availableIds.join(", ")}`); diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts index 29431c1497..7fa7954339 100644 --- a/test/e2e-scenario/scenarios/run.ts +++ b/test/e2e-scenario/scenarios/run.ts @@ -6,13 +6,15 @@ import { fileURLToPath } from "node:url"; import { compileRunPlans, renderPlanText, writePlanArtifacts } from "./compiler.ts"; import { ScenarioRunner } from "./orchestrators/runner.ts"; -import { listScenarios } from "./registry.ts"; +import { listScenarios, requireScenarios } from "./registry.ts"; import { resolveRunnerForScenario } from "./runner-routing.ts"; +import { type LiveScenarioSupport, liveScenarioSupport } from "./runtime-support.ts"; import type { PhaseResult, ScenarioDefinition } from "./types.ts"; interface Args { list: boolean; emitMatrix: boolean; + emitLiveMatrix: boolean; planOnly: boolean; scenarios: string[]; } @@ -31,8 +33,25 @@ export interface ScenarioMatrixEntry { suites: string[]; } +export interface LiveScenarioMatrixEntry extends ScenarioMatrixEntry { + install: string; + runtime: string; + onboarding: string; + expectedStateId: string; + requiredSecrets: string[]; + supported: boolean; + supportReasons: string[]; + pendingRuntimeSuites: string[]; +} + function parseArgs(argv: string[]): Args { - const args: Args = { list: false, emitMatrix: false, planOnly: false, scenarios: [] }; + const args: Args = { + list: false, + emitMatrix: false, + emitLiveMatrix: false, + planOnly: false, + scenarios: [], + }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === "--list") { @@ -43,6 +62,10 @@ function parseArgs(argv: string[]): Args { args.emitMatrix = true; continue; } + if (arg === "--emit-live-matrix") { + args.emitLiveMatrix = true; + continue; + } if (arg === "--plan-only") { args.planOnly = true; continue; @@ -104,6 +127,40 @@ export function buildScenarioMatrix(): ScenarioMatrixEntry[] { }); } +function liveMatrixEntry( + scenario: ScenarioDefinition, + support: LiveScenarioSupport, +): LiveScenarioMatrixEntry { + const { runner } = resolveRunnerForScenario(scenario); + return { + id: scenario.id, + runner, + label: buildLabel(scenario), + platform: scenario.environment?.platform ?? "unknown", + install: scenario.environment?.install ?? "unknown", + runtime: scenario.environment?.runtime ?? "unknown", + onboarding: scenario.environment?.onboarding ?? "unknown", + expectedStateId: scenario.expectedStateId ?? "", + suites: scenario.suiteIds ?? [], + requiredSecrets: scenario.requiredSecrets ?? [], + supported: support.supported, + supportReasons: support.reasons, + pendingRuntimeSuites: support.pendingRuntimeSuites, + }; +} + +export function buildLiveScenarioMatrix(ids: string[] = []): LiveScenarioMatrixEntry[] { + const scenarioSupport = (ids.length > 0 ? requireScenarios(ids) : listScenarios()).map( + (scenario) => ({ + scenario, + support: liveScenarioSupport(scenario), + }), + ); + const liveEntries = + ids.length > 0 ? scenarioSupport : scenarioSupport.filter(({ support }) => support.supported); + return liveEntries.map(({ scenario, support }) => liveMatrixEntry(scenario, support)); +} + function emitMatrix() { // Single line so GHA's `$GITHUB_OUTPUT` can consume it via // echo "matrix=$(npx tsx ... --emit-matrix)" >> "$GITHUB_OUTPUT" @@ -112,6 +169,10 @@ function emitMatrix() { process.stdout.write(`${JSON.stringify(buildScenarioMatrix())}\n`); } +function emitLiveMatrix(ids: string[]) { + process.stdout.write(`${JSON.stringify(buildLiveScenarioMatrix(ids))}\n`); +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (args.list) { @@ -122,6 +183,10 @@ async function main() { emitMatrix(); return; } + if (args.emitLiveMatrix) { + emitLiveMatrix(args.scenarios); + return; + } if (args.scenarios.length === 0) { throw new Error("scenario execution requires --scenarios "); diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index ec72df6ba3..24b1e26924 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -14,6 +14,17 @@ export interface LiveScenarioSupport { pendingRuntimeSuites: string[]; } +/** + * Canonical name under which a scenario is registered with Vitest in the + * live registry-scenarios test file. The workflow filters by exact ID via + * `-t "^${SCENARIO_ID}$"`, so both supported and unsupported scenarios MUST + * be registered under this exact name. Skip reasons are surfaced via the + * job log instead of the test name suffix. + */ +export function liveScenarioTestName(scenario: ScenarioDefinition): string { + return scenario.id; +} + export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioSupport { const reasons: string[] = []; const environment = scenario.environment; diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index cdc14a1108..995443aeec 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -217,24 +217,71 @@ export function validateE2eVitestScenariosWorkflowBoundary( rejectAutomaticTriggers(errors, triggers); const dispatchInputs = asRecord(workflowDispatch.inputs); - requireInput(errors, dispatchInputs, "test_filter"); + requireInput(errors, dispatchInputs, "scenarios"); + if (Object.hasOwn(dispatchInputs, "test_filter")) { + errors.push("workflow_dispatch must not expose legacy test_filter input"); + } const permissions = asRecord(workflow.permissions); if (permissions.contents !== "read") errors.push("workflow permissions.contents must be read"); const jobs = asRecord(workflow.jobs); + const generateMatrix = asRecord(jobs["generate-matrix"]); + if (Object.keys(generateMatrix).length === 0) errors.push("workflow missing generate-matrix job"); + if (generateMatrix["runs-on"] !== "ubuntu-latest") { + errors.push("generate-matrix job must run on ubuntu-latest"); + } + const generateSteps = asSteps(generateMatrix.steps); + requireNoDispatchInputInterpolation(errors, generateSteps); + const generateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); + requireFullShaAction(errors, generateCheckout, "generate-matrix checkout"); + if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { + errors.push("generate-matrix checkout step must set persist-credentials=false"); + } + const generateSetupNode = namedStep(generateSteps, "Set up Node"); + if (!generateSetupNode) errors.push("generate-matrix job missing step: Set up Node"); + requireFullShaAction(errors, generateSetupNode, "generate-matrix setup-node"); + const generate = requireStep(errors, generateSteps, "Generate Vitest scenario matrix"); + const generateEnv = asRecord(generate?.env); + if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") { + errors.push("matrix generation step must pass scenarios through SCENARIOS env"); + } + requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); + requireRunContains(errors, generate, "--emit-live-matrix"); + requireRunContains(errors, generate, "--scenarios"); + requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); + requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); + const liveScenarios = asRecord(jobs["live-scenarios"]); if (Object.keys(liveScenarios).length === 0) errors.push("workflow missing live-scenarios job"); - if (liveScenarios["runs-on"] !== "ubuntu-latest") { - errors.push("live-scenarios job must run on ubuntu-latest"); + if (liveScenarios["runs-on"] !== "${{ matrix.runner }}") { + errors.push("live-scenarios job must run on the matrix runner"); + } + if (liveScenarios.needs !== "generate-matrix") { + errors.push("live-scenarios job must depend on generate-matrix"); + } + const strategy = asRecord(liveScenarios.strategy); + if (strategy["fail-fast"] !== false) { + errors.push("live-scenarios strategy.fail-fast must be false"); + } + const matrix = asRecord(strategy.matrix); + if (matrix.include !== "${{ fromJSON(needs.generate-matrix.outputs.matrix) }}") { + errors.push("live-scenarios matrix.include must come from generate-matrix output"); } const jobEnv = asRecord(liveScenarios.env); if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { errors.push("live-scenarios job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } - if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes(".e2e/vitest")) { - errors.push("live-scenarios job must write artifacts under .e2e/vitest"); + if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes("e2e-artifacts/vitest")) { + errors.push("live-scenarios job must write artifacts under e2e-artifacts/vitest"); + } + if (!stringValue(jobEnv.E2E_ARTIFACT_DIR).includes("${{ matrix.id }}")) { + errors.push("live-scenarios artifacts must be scoped by matrix.id"); + } + if (!stringValue(jobEnv.NEMOCLAW_CLI_BIN).includes("bin/nemoclaw.js")) { + errors.push("live-scenarios job must point NEMOCLAW_CLI_BIN at the repo CLI"); } const steps = asSteps(liveScenarios.steps); @@ -256,30 +303,34 @@ export function validateE2eVitestScenariosWorkflowBoundary( const runVitest = requireStep(errors, steps, "Run Vitest live E2E scenarios"); const runVitestEnv = asRecord(runVitest?.env); - if (runVitestEnv.TEST_FILTER !== "${{ inputs.test_filter }}") { - errors.push("Vitest step must pass test_filter through TEST_FILTER env"); + if (runVitestEnv.SCENARIO_ID !== "${{ matrix.id }}") { + errors.push("Vitest step must pass matrix.id through SCENARIO_ID env"); } requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); - requireRunContains(errors, runVitest, '"${TEST_FILTER}"'); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/registry-scenarios.test.ts"); + requireRunContains(errors, runVitest, '"^${SCENARIO_ID}$"'); const summary = requireStep(errors, steps, "Summarize artifacts"); const summaryEnv = asRecord(summary?.env); - if (summaryEnv.FILTER_LABEL !== "${{ inputs.test_filter || 'all' }}") { - errors.push("summary step must pass display filter through FILTER_LABEL env"); + if (summaryEnv.SCENARIO_ID !== "${{ matrix.id }}") { + errors.push("summary step must pass matrix.id through SCENARIO_ID env"); + } + if (summaryEnv.SCENARIO_LABEL !== "${{ matrix.label }}") { + errors.push("summary step must pass matrix.label through SCENARIO_LABEL env"); } - requireRunContains(errors, summary, "${FILTER_LABEL}"); + requireRunContains(errors, summary, "${SCENARIO_ID}"); const upload = requireStep(errors, steps, "Upload Vitest E2E artifacts"); requireFullShaAction(errors, upload, "upload-artifact"); const uploadWith = asRecord(upload?.with); - if (uploadWith.name !== "e2e-vitest-scenarios") { - errors.push("artifact upload name must be e2e-vitest-scenarios"); + if (uploadWith.name !== "e2e-vitest-scenarios-${{ matrix.id }}") { + errors.push("artifact upload name must include matrix.id"); } - if (uploadWith.path !== ".e2e/vitest/") { - errors.push("artifact upload path must be .e2e/vitest/"); + if (uploadWith.path !== "e2e-artifacts/vitest/${{ matrix.id }}/") { + errors.push("artifact upload path must be non-hidden and scoped by matrix.id"); } - if (uploadWith["include-hidden-files"] !== true) { - errors.push("artifact upload must set include-hidden-files: true"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("artifact upload must set include-hidden-files: false"); } if (uploadWith["if-no-files-found"] !== "ignore") { errors.push("artifact upload must ignore missing fixture artifacts");