ci(e2e): fan out Vitest scenarios from registry

.github/workflows/e2e-vitest-scenarios.yaml

@@ -6,8 +6,8 @@ name: E2E / Vitest Scenarios
 on:
   workflow_dispatch:
     inputs:
-      test_filter:
-        description: "Optional Vitest file/name filter for test/e2e-scenario/live"
+      scenarios:
+        description: "Optional comma-separated typed scenario ids. Empty runs all live Vitest-supported scenarios."
         required: false
         default: ""
         type: string
@@ -16,16 +16,58 @@ permissions:
   contents: read
 
 concurrency:
-  group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.test_filter || 'all' }}
+  group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}
   cancel-in-progress: false
 
 jobs:
-  live-scenarios:
+  generate-matrix:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
+      - id: matrix
+        name: Generate Vitest scenario matrix
+        env:
+          SCENARIOS: ${{ inputs.scenarios }}
+        run: |
+          set -euo pipefail
+          args=(--emit-live-matrix)
+          if [ -n "${SCENARIOS}" ]; then
+            if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
+              echo "::error::Invalid scenario input: ${SCENARIOS}" >&2
+              exit 1
+            fi
+            args+=(--scenarios "${SCENARIOS}")
+          fi
+          matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")"
+          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
+
+  live-scenarios:
+    needs: generate-matrix
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
     env:
-      E2E_ARTIFACT_DIR: ${{ github.workspace }}/.e2e/vitest
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/${{ matrix.id }}
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_E2E_SCENARIOS: "1"
+      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
@@ -45,25 +87,23 @@ jobs:
 
       - name: Run Vitest live E2E scenarios
         env:
-          TEST_FILTER: ${{ inputs.test_filter }}
+          SCENARIO_ID: ${{ matrix.id }}
         run: |
           set -euo pipefail
-          if [ -n "${TEST_FILTER}" ]; then
-            npx vitest run --project e2e-scenarios-live "${TEST_FILTER}" --silent=false --reporter=default
-          else
-            npx vitest run --project e2e-scenarios-live --silent=false --reporter=default
-          fi
+          npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/registry-scenarios.test.ts -t "^${SCENARIO_ID}$" --silent=false --reporter=default
 
       - name: Summarize artifacts
         if: always()
         env:
-          FILTER_LABEL: ${{ inputs.test_filter || 'all' }}
+          SCENARIO_ID: ${{ matrix.id }}
+          SCENARIO_LABEL: ${{ matrix.label }}
         run: |
           {
             echo "## Vitest E2E Scenarios"
             echo
             echo "- Project: \`e2e-scenarios-live\`"
-            printf '%s%s%s\n' '- Filter: `' "${FILTER_LABEL}" '`'
+            printf '%s%s%s\n' '- Scenario: `' "${SCENARIO_ID}" '`'
+            printf '%s%s%s\n' '- Label: `' "${SCENARIO_LABEL}" '`'
             echo "- Artifact root: \`${E2E_ARTIFACT_DIR}\`"
             echo
             if [ -d "${E2E_ARTIFACT_DIR}" ]; then
@@ -78,7 +118,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-vitest-scenarios
-          path: .e2e/vitest/
-          include-hidden-files: true
+          name: e2e-vitest-scenarios-${{ matrix.id }}
+          path: e2e-artifacts/vitest/${{ matrix.id }}/
+          include-hidden-files: false
           if-no-files-found: ignore

test/e2e-scenario/docs/README.md

@@ -190,7 +190,7 @@ test/e2e-scenario/
 - `.github/workflows/e2e-scenarios-all.yaml` fans out typed scenario dry-runs
   from the typed registry matrix.
 - `.github/workflows/e2e-vitest-scenarios.yaml` runs the opt-in Vitest live
-  scenario project and uploads `.e2e/vitest/` fixture artifacts.
+  scenario project and uploads non-hidden `e2e-artifacts/vitest/` fixture artifacts.
 - Existing workflows such as `nightly-e2e.yaml`, `e2e-branch-validation.yaml`,
   `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and
   `regression-e2e.yaml` still run legacy live E2E scripts during the migration.

test/e2e-scenario/framework-tests/e2e-live-skip-name-contract.test.ts

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { listScenarios } from "../scenarios/registry.ts";
+import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts";
+
+/**
+ * Locks the contract that the live registry-scenarios test file registers
+ * each scenario under a name equal to `scenario.id` (no `[not wired: ...]`
+ * suffix), so the workflow's exact `-t "^${SCENARIO_ID}$"` filter matches
+ * supported AND unsupported entries identically. Without this contract,
+ * explicit unsupported selections on `workflow_dispatch` would match zero
+ * tests and Vitest would exit non-zero with no structured skip reason.
+ */
+describe("live registry-scenarios skip-name contract", () => {
+  it("registers every scenario under a name equal to its id", () => {
+    const scenarios = listScenarios();
+    expect(scenarios.length).toBeGreaterThan(0);
+    for (const scenario of scenarios) {
+      expect(liveScenarioTestName(scenario)).toBe(scenario.id);
+    }
+  });
+
+  it('matches the workflow\'s exact `-t "^${SCENARIO_ID}$"` regex for every scenario', () => {
+    for (const scenario of listScenarios()) {
+      const name = liveScenarioTestName(scenario);
+      const filter = new RegExp(`^${scenario.id}$`);
+      expect(
+        filter.test(name),
+        `workflow filter must match registered name for ${scenario.id}`,
+      ).toBe(true);
+    }
+  });
+
+  it("matches an explicit unsupported selection through the workflow filter", () => {
+    const unsupported = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-hermes");
+    expect(
+      unsupported,
+      "ubuntu-repo-cloud-hermes must remain a canonical unsupported example",
+    ).toBeTruthy();
+    const support = liveScenarioSupport(unsupported!);
+    expect(support.supported).toBe(false);
+
+    const name = liveScenarioTestName(unsupported!);
+    const filter = new RegExp(`^${unsupported!.id}$`);
+    expect(filter.test(name)).toBe(true);
+    // Negative: any historical `[not wired: ...]` suffix would break the workflow filter.
+    expect(name).not.toMatch(/\[not wired:/);
+  });
+
+  it("registers the canonical supported scenario under its bare id", () => {
+    const supported = listScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw");
+    expect(supported).toBeTruthy();
+    expect(liveScenarioSupport(supported!).supported).toBe(true);
+    expect(liveScenarioTestName(supported!)).toBe("ubuntu-repo-cloud-openclaw");
+  });
+
+  // Note: the workflow's `-t "^${SCENARIO_ID}$"` filter pattern itself is
+  // locked by `tools/e2e-scenarios/workflow-boundary.mts` and exercised by
+  // `e2e-scenarios-workflow.test.ts`. This file only needs to guarantee
+  // that the test names registered under that filter equal `scenario.id`.
+});

test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts

@@ -5,11 +5,10 @@ import { spawnSync } from "node:child_process";
 import path from "node:path";
 
 import { describe, expect, it } from "vitest";
-
-import { buildScenarioMatrix } from "../scenarios/run.ts";
+import { scenario } from "../scenarios/builder.ts";
 import { listScenarios } from "../scenarios/registry.ts";
+import { buildLiveScenarioMatrix, buildScenarioMatrix } from "../scenarios/run.ts";
 import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts";
-import { scenario } from "../scenarios/builder.ts";
 
 const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
 const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts");
@@ -23,6 +22,14 @@ function runEmitMatrix() {
   });
 }
 
+function runEmitLiveMatrix(args: string[] = []) {
+  return spawnSync(TSX, [RUN_SCENARIOS, "--emit-live-matrix", ...args], {
+    cwd: REPO_ROOT,
+    encoding: "utf8",
+    timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000),
+  });
+}
+
 describe("typed scenario matrix", () => {
   it("emits one matrix entry per registered scenario", () => {
     const matrix = buildScenarioMatrix();
@@ -120,4 +127,54 @@ describe("typed scenario matrix", () => {
       });
     }
   });
+
+  it("builds the default live Vitest matrix from fixture-supported scenarios only", () => {
+    expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([
+      "ubuntu-repo-cloud-openclaw",
+    ]);
+    expect(buildLiveScenarioMatrix()[0]).toMatchObject({
+      id: "ubuntu-repo-cloud-openclaw",
+      runner: "ubuntu-latest",
+      platform: "ubuntu-local",
+      install: "repo-current",
+      runtime: "docker-running",
+      onboarding: "cloud-openclaw",
+      expectedStateId: "cloud-openclaw-ready",
+      requiredSecrets: ["NVIDIA_API_KEY"],
+      supported: true,
+      supportReasons: [],
+      pendingRuntimeSuites: ["smoke", "inference", "credentials"],
+    });
+  });
+
+  it("keeps explicitly selected unsupported live scenarios in the matrix with skip reasons", () => {
+    expect(buildLiveScenarioMatrix(["ubuntu-repo-cloud-hermes"])).toEqual([
+      expect.objectContaining({
+        id: "ubuntu-repo-cloud-hermes",
+        supported: false,
+        supportReasons: ["onboarding 'cloud-hermes' is not wired for live Vitest fixtures"],
+      }),
+    ]);
+  });
+
+  it("--emit-live-matrix prints a single-line JSON array for supported live Vitest scenarios", () => {
+    const result = runEmitLiveMatrix();
+    expect(result.status, result.stderr).toBe(0);
+    const lines = result.stdout.trim().split("\n");
+    expect(lines.length, "live matrix output must be a single line").toBe(1);
+    const parsed = JSON.parse(lines[0]);
+    expect(parsed.map((entry: { id: string }) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]);
+  });
+
+  it("--emit-live-matrix honors explicit scenario selections", () => {
+    const result = runEmitLiveMatrix(["--scenarios", "ubuntu-repo-cloud-hermes"]);
+    expect(result.status, result.stderr).toBe(0);
+    const parsed = JSON.parse(result.stdout.trim());
+    expect(parsed).toEqual([
+      expect.objectContaining({
+        id: "ubuntu-repo-cloud-hermes",
+        supported: false,
+      }),
+    ]);
+  });
 });

test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts

@@ -33,6 +33,20 @@ describe("deterministic scenario registry", () => {
     expect(() => buildScenarioRegistry([first, second])).toThrow(/duplicate-id/);
   });
 
+  it("should reject scenario IDs that are unsafe for workflow regex filters and artifact paths", () => {
+    const unsafe = scenario("bad.id")
+      .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml")
+      .build();
+
+    expect(() => buildScenarioRegistry([unsafe])).toThrow(/not safe for workflow regex filters/);
+
+    const result = runScenarioCli(["--scenarios", "../escape", "--plan-only"]);
+    expect(result.status).not.toBe(0);
+    expect(`${result.stdout}${result.stderr}`).toMatch(
+      /Selected scenario ID '\.\.\/escape' is not safe/,
+    );
+  });
+
   it("should return actionable unknown scenario error", () => {
     const result = runScenarioCli(["--scenarios", "does-not-exist", "--plan-only"]);
 

test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts

@@ -6,13 +6,12 @@ import os from "node:os";
 import path from "node:path";
 
 import { describe, expect, it } from "vitest";
-
-import { listScenarios } from "../scenarios/registry.ts";
-import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts";
 import {
   validateE2eScenariosWorkflowBoundary,
   validateE2eVitestScenariosWorkflowBoundary,
 } from "../../../tools/e2e-scenarios/workflow-boundary.mts";
+import { listScenarios } from "../scenarios/registry.ts";
+import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts";
 
 const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
 const WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e-scenarios.yaml");
@@ -116,7 +115,7 @@ describe("e2e-vitest-scenarios workflow boundary", () => {
     expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]);
   });
 
-  it("flags direct dispatch-input interpolation and missing hidden artifact upload", () => {
+  it("flags direct dispatch-input interpolation and unsafe artifact upload", () => {
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-"));
     const workflowPath = path.join(tmp, "workflow.yaml");
     fs.writeFileSync(
@@ -152,7 +151,7 @@ jobs:
         with:
           name: e2e-vitest-scenarios
           path: .e2e/vitest/
-          include-hidden-files: false
+          include-hidden-files: true
           if-no-files-found: ignore
 `,
     );
@@ -161,14 +160,31 @@ jobs:
       const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath);
       expect(errors).toEqual(
         expect.arrayContaining([
+          "workflow_dispatch missing input: scenarios",
+          "workflow_dispatch must not expose legacy test_filter input",
+          "workflow missing generate-matrix job",
+          "generate-matrix job must run on ubuntu-latest",
+          "live-scenarios job must run on the matrix runner",
+          "live-scenarios job must depend on generate-matrix",
+          "live-scenarios strategy.fail-fast must be false",
+          "live-scenarios matrix.include must come from generate-matrix output",
+          "live-scenarios job must write artifacts under e2e-artifacts/vitest",
+          "live-scenarios artifacts must be scoped by matrix.id",
+          "live-scenarios job must point NEMOCLAW_CLI_BIN at the repo CLI",
           "checkout action must be pinned to a full commit SHA",
           "checkout step must set persist-credentials=false",
           "setup-node action must be pinned to a full commit SHA",
           "run-scenario job missing step: Build CLI",
+          "Vitest step must pass matrix.id through SCENARIO_ID env",
           "step 'Run Vitest live E2E scenarios' run script must not interpolate dispatch inputs directly",
+          "step 'Run Vitest live E2E scenarios' run script must include test/e2e-scenario/live/registry-scenarios.test.ts",
+          "step 'Run Vitest live E2E scenarios' run script must include \"^${SCENARIO_ID}$\"",
           "step 'Summarize artifacts' run script must not interpolate dispatch inputs directly",
-          "summary step must pass display filter through FILTER_LABEL env",
-          "artifact upload must set include-hidden-files: true",
+          "summary step must pass matrix.id through SCENARIO_ID env",
+          "summary step must pass matrix.label through SCENARIO_LABEL env",
+          "artifact upload must set include-hidden-files: false",
+          "artifact upload name must include matrix.id",
+          "artifact upload path must be non-hidden and scoped by matrix.id",
           "upload-artifact action must be pinned to a full commit SHA",
         ]),
       );