From b573c818babf4b6ddb697e416905af01b3a5c8e0 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Wed, 27 May 2026 18:09:54 -0400
Subject: [PATCH 1/2] feat(e2e): generate scenario fan-out matrix from typed
 registry

Replace the hand-wired job list in e2e-scenarios-all.yaml with a
generated matrix sourced from the existing typed scenario registry.
Adding a scenario in test/e2e-scenario/scenarios/scenarios/baseline.ts
now automatically produces a tile in the fan-out workflow on the next
run, with no workflow edits required. This closes the drift gap that
left ~14 of 22 registered scenarios undispatched.

Changes:
- Add --emit-matrix flag to test/e2e-scenario/scenarios/run.ts that
  walks listScenarios() and prints a single-line JSON array of GHA
  matrix include entries: { id, runner, label, platform, suites }.
- Extract runner-routing.ts that derives the runs-on label from
  ScenarioEnvironment.platform, with a runs-on:<label> override path
  via runnerRequirements. Throws on unknown platforms so missing
  mappings fail loudly during matrix generation.
- Refactor .github/workflows/e2e-scenarios-all.yaml to a two-job
  shape: generate-matrix emits the JSON, run-scenario fans out via
  strategy.matrix.include and calls the existing e2e-scenarios.yaml
  reusable workflow. Tile names come from the typed label so the
  sidebar reflects what is actually being tested.
- Guard the run.ts CLI bootstrap so importing buildScenarioMatrix
  from tests does not trigger the side-effecting main() path.
- Add e2e-scenario-matrix.test.ts covering matrix shape, registry
  parity, runner overrides, unknown-platform errors, and CLI output
  contract.
---
 .github/workflows/e2e-scenarios-all.yaml      | 105 ++++++++++--------
 .../e2e-scenario-matrix.test.ts               | 105 ++++++++++++++++++
 test/e2e-scenario/scenarios/run.ts            | 103 ++++++++++++++++-
 test/e2e-scenario/scenarios/runner-routing.ts |  52 +++++++++
 4 files changed, 312 insertions(+), 53 deletions(-)
 create mode 100644 test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
 create mode 100644 test/e2e-scenario/scenarios/runner-routing.ts

diff --git a/.github/workflows/e2e-scenarios-all.yaml b/.github/workflows/e2e-scenarios-all.yaml
index d6a42a08528..b0d2a4a2958 100644
--- a/.github/workflows/e2e-scenarios-all.yaml
+++ b/.github/workflows/e2e-scenarios-all.yaml
@@ -1,15 +1,14 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-# Scenario-based E2E fan-out. Runs every setup scenario from the current
-# migration catalog by calling the single-scenario runner workflow.
+# Scenario-based E2E fan-out. Generates the job matrix from the typed
+# scenario registry (`test/e2e-scenario/scenarios/registry.ts`) so that
+# adding a scenario in `baseline.ts` automatically produces a tile here on
+# the next run -- no workflow edits required.
 #
-# Each child job passes its scenario id through the called workflow's
-# `scenarios` input (comma-separated; one id here per child). The legacy
-# per-call `suite_filter` input was retired when the runner moved to a
-# typed scenario contract, so this fan-out no longer accepts a fan-out-wide
-# suite filter; if per-suite scoping is needed in the future, route it
-# through `e2e-scenarios.yaml` directly with a custom scenario list.
+# The single-scenario runner workflow `e2e-scenarios.yaml` remains the
+# authoritative execution path; this file just fans out one call per
+# scenario id with the runner label resolved by `--emit-matrix`.
 
 name: E2E / Scenario Runner / All
 
@@ -24,51 +23,63 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  ubuntu-repo-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: ubuntu-repo-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.emit.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
 
-  ubuntu-repo-cloud-hermes:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: ubuntu-repo-cloud-hermes
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
 
-  gpu-repo-local-ollama-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: gpu-repo-local-ollama-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
 
-  macos-repo-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: macos-repo-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
-
-  wsl-repo-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: wsl-repo-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - id: emit
+        name: Emit scenario matrix from typed registry
+        run: |
+          set -euo pipefail
+          matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix)"
+          # Sanity-check that the output is non-empty JSON before handing it
+          # to GHA, so a registry error fails this job loudly instead of
+          # producing zero matrix tiles.
+          if [ -z "${matrix}" ] || [ "${matrix}" = "[]" ]; then
+            echo "::error::scenario matrix is empty; check typed registry" >&2
+            exit 1
+          fi
+          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
 
-  brev-launchable-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: brev-launchable-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - name: Render matrix summary
+        env:
+          MATRIX_JSON: ${{ steps.emit.outputs.matrix }}
+        run: |
+          {
+            echo '## E2E scenario matrix'
+            echo ''
+            echo '| Scenario | Runner | Label |'
+            echo '| --- | --- | --- |'
+            python3 - <<'PY'
+          import json, os
+          for e in json.loads(os.environ["MATRIX_JSON"]):
+              print(f"| `{e['id']}` | {e['runner']} | {e['label']} |")
+          PY
+          } >> "$GITHUB_STEP_SUMMARY"
 
-  ubuntu-no-docker-preflight-negative:
+  run-scenario:
+    needs: generate-matrix
+    name: ${{ matrix.label }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     uses: ./.github/workflows/e2e-scenarios.yaml
     with:
-      scenarios: ubuntu-no-docker-preflight-negative
+      scenarios: ${{ matrix.id }}
     secrets:
       NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
new file mode 100644
index 00000000000..3e06cb12a61
--- /dev/null
+++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
@@ -0,0 +1,105 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { buildScenarioMatrix } from "../scenarios/run.ts";
+import { listScenarios } from "../scenarios/registry.ts";
+import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts";
+import { scenario } from "../scenarios/builder.ts";
+
+const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
+const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts");
+const TSX = path.join(REPO_ROOT, "node_modules/.bin/tsx");
+
+function runEmitMatrix() {
+  return spawnSync(TSX, [RUN_SCENARIOS, "--emit-matrix"], {
+    cwd: REPO_ROOT,
+    encoding: "utf8",
+    timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000),
+  });
+}
+
+describe("typed scenario matrix", () => {
+  it("emits one matrix entry per registered scenario", () => {
+    const matrix = buildScenarioMatrix();
+    const ids = listScenarios().map((s) => s.id);
+    expect(matrix.map((entry) => entry.id).sort()).toEqual([...ids].sort());
+  });
+
+  it("resolves a runner label for every scenario", () => {
+    const matrix = buildScenarioMatrix();
+    expect(matrix.length).toBeGreaterThan(0);
+    for (const entry of matrix) {
+      expect(entry.runner, `runner missing for ${entry.id}`).toMatch(/[A-Za-z0-9-]/);
+      expect(entry.label, `label missing for ${entry.id}`).toContain(entry.id);
+    }
+  });
+
+  it("routes platforms to their canonical runners", () => {
+    const byId = new Map(buildScenarioMatrix().map((entry) => [entry.id, entry]));
+    expect(byId.get("ubuntu-repo-cloud-openclaw")?.runner).toBe("ubuntu-latest");
+    expect(byId.get("macos-repo-cloud-openclaw")?.runner).toBe("macos-26");
+    expect(byId.get("wsl-repo-cloud-openclaw")?.runner).toBe("windows-latest");
+    expect(byId.get("gpu-repo-local-ollama-openclaw")?.runner).toBe(
+      "linux-amd64-gpu-rtxpro6000-latest-1",
+    );
+  });
+
+  it("honors an explicit runs-on:<label> requirement override", () => {
+    const custom = scenario("test-runs-on-override")
+      .description("test fixture")
+      .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml")
+      .environment({
+        platform: "ubuntu-local",
+        install: "repo-current",
+        runtime: "docker-running",
+        onboarding: "cloud-openclaw",
+      })
+      .expectedState("cloud-openclaw-ready")
+      .onboardingAssertions(["base-installed"])
+      .suites(["smoke"])
+      .runnerRequirements(["runs-on:custom-self-hosted"])
+      .build();
+    expect(resolveRunnerForScenario(custom).runner).toBe("custom-self-hosted");
+  });
+
+  it("fails loudly when a platform has no default runner mapping", () => {
+    const broken = scenario("test-unknown-platform")
+      .description("test fixture")
+      .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml")
+      .environment({
+        platform: "made-up-platform",
+        install: "repo-current",
+        runtime: "docker-running",
+        onboarding: "cloud-openclaw",
+      })
+      .expectedState("cloud-openclaw-ready")
+      .onboardingAssertions(["base-installed"])
+      .suites(["smoke"])
+      .build();
+    expect(() => resolveRunnerForScenario(broken)).toThrow(/no default for platform/);
+  });
+
+  it("--emit-matrix prints a single-line JSON array compatible with $GITHUB_OUTPUT", () => {
+    const result = runEmitMatrix();
+    expect(result.status, result.stderr).toBe(0);
+    const lines = result.stdout.trim().split("\n");
+    expect(lines.length, "matrix output must be a single line").toBe(1);
+    const parsed = JSON.parse(lines[0]);
+    expect(Array.isArray(parsed)).toBe(true);
+    expect(parsed.length).toBe(listScenarios().length);
+    for (const entry of parsed) {
+      expect(entry).toMatchObject({
+        id: expect.any(String),
+        runner: expect.any(String),
+        label: expect.any(String),
+        platform: expect.any(String),
+        suites: expect.any(Array),
+      });
+    }
+  });
+});
diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts
index e666e078446..b50e7836eae 100644
--- a/test/e2e-scenario/scenarios/run.ts
+++ b/test/e2e-scenario/scenarios/run.ts
@@ -1,26 +1,57 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { realpathSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
 import { compileRunPlans, renderPlanText, writePlanArtifacts } from "./compiler.ts";
 import { ScenarioRunner } from "./orchestrators/runner.ts";
 import { listScenarios } from "./registry.ts";
+import { resolveRunnerForScenario } from "./runner-routing.ts";
+import type { ScenarioDefinition } from "./types.ts";
 
 interface Args {
   list: boolean;
   planOnly: boolean;
   dryRun: boolean;
   validateOnly: boolean;
+  emitMatrix: boolean;
   scenarios: string[];
 }
 
+/**
+ * Shape of a single GitHub Actions matrix `include` entry emitted by
+ * `--emit-matrix`. The fields are kept short and JSON-stable so the consuming
+ * workflow can reference them as `${{ matrix.id }}`, `${{ matrix.runner }}`,
+ * etc. without further parsing.
+ */
+export interface ScenarioMatrixEntry {
+  id: string;
+  runner: string;
+  label: string;
+  platform: string;
+  suites: string[];
+}
+
 function parseArgs(argv: string[]): Args {
-  const args: Args = { list: false, planOnly: false, dryRun: false, validateOnly: false, scenarios: [] };
+  const args: Args = {
+    list: false,
+    planOnly: false,
+    dryRun: false,
+    validateOnly: false,
+    emitMatrix: false,
+    scenarios: [],
+  };
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
     if (arg === "--list") {
       args.list = true;
       continue;
     }
+    if (arg === "--emit-matrix") {
+      args.emitMatrix = true;
+      continue;
+    }
     if (arg === "--plan-only") {
       args.planOnly = true;
       continue;
@@ -54,12 +85,56 @@ function printList() {
   }
 }
 
+function buildLabel(scenario: ScenarioDefinition): string {
+  const platform = scenario.environment?.platform ?? "unknown-platform";
+  const suites = scenario.suiteIds ?? [];
+  if (scenario.expectedFailure) {
+    const cls = scenario.expectedFailure.errorClass ?? "expected-failure";
+    return `${platform} \u00b7 ${scenario.id} \u00b7 expect-fail:${cls}`;
+  }
+  if (suites.length === 0) {
+    return `${platform} \u00b7 ${scenario.id}`;
+  }
+  if (suites.length <= 3) {
+    return `${platform} \u00b7 ${scenario.id} \u00b7 ${suites.join("+")}`;
+  }
+  return `${platform} \u00b7 ${scenario.id} \u00b7 ${suites.length} suites`;
+}
+
+/**
+ * Build the GitHub Actions matrix for every scenario in the typed registry.
+ * Sorted by id so workflow runs are deterministic and diffable.
+ */
+export function buildScenarioMatrix(): ScenarioMatrixEntry[] {
+  return listScenarios().map((scenario): ScenarioMatrixEntry => {
+    const { runner } = resolveRunnerForScenario(scenario);
+    return {
+      id: scenario.id,
+      runner,
+      label: buildLabel(scenario),
+      platform: scenario.environment?.platform ?? "unknown",
+      suites: scenario.suiteIds ?? [],
+    };
+  });
+}
+
+function emitMatrix() {
+  // Single line so GHA's `$GITHUB_OUTPUT` can consume it via
+  //   echo "matrix=$(npx tsx ... --emit-matrix)" >> "$GITHUB_OUTPUT"
+  // without needing heredoc multi-line output handling.
+  process.stdout.write(`${JSON.stringify(buildScenarioMatrix())}\n`);
+}
+
 async function main() {
   const args = parseArgs(process.argv.slice(2));
   if (args.list) {
     printList();
     return;
   }
+  if (args.emitMatrix) {
+    emitMatrix();
+    return;
+  }
 
   const modeCount = [args.planOnly, args.dryRun, args.validateOnly].filter(Boolean).length;
   if (modeCount !== 1) {
@@ -86,9 +161,25 @@ async function main() {
   }
 }
 
-try {
-  await main();
-} catch (error) {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
+// Only execute when invoked directly as a script. Importing this module from
+// tests (e.g. `buildScenarioMatrix`) must not trigger the CLI side-effects.
+// Compare via realpath so symlinked paths (e.g. `/tmp` -> `/private/tmp` on
+// macOS) still resolve as equal.
+function isInvokedDirectly(): boolean {
+  const entry = process.argv[1];
+  if (!entry) return false;
+  try {
+    return realpathSync(entry) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+}
+
+if (isInvokedDirectly()) {
+  try {
+    await main();
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  }
 }
diff --git a/test/e2e-scenario/scenarios/runner-routing.ts b/test/e2e-scenario/scenarios/runner-routing.ts
new file mode 100644
index 00000000000..45b11552818
--- /dev/null
+++ b/test/e2e-scenario/scenarios/runner-routing.ts
@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Maps a typed scenario definition to the GitHub Actions runner label it
+// should execute on. The mapping is derived from the typed
+// `ScenarioEnvironment.platform` and `ScenarioDefinition.runnerRequirements`
+// fields so that adding a scenario to the registry automatically resolves a
+// runner without editing workflow YAML.
+
+import type { ScenarioDefinition } from "./types.ts";
+
+export interface ResolvedRunner {
+  /** GitHub Actions `runs-on` label. */
+  runner: string;
+  /** Reason the runner was selected, surfaced in matrix entries for debugging. */
+  reason: string;
+}
+
+const PLATFORM_DEFAULT_RUNNER: Record<string, string> = {
+  "ubuntu-local": "ubuntu-latest",
+  "gpu-runner": "linux-amd64-gpu-rtxpro6000-latest-1",
+  "macos-local": "macos-26",
+  "wsl-local": "windows-latest",
+  "brev-launchable": "ubuntu-latest",
+};
+
+/**
+ * Resolve the GitHub Actions runner label for a typed scenario.
+ *
+ * Routing precedence:
+ *   1. An explicit `runs-on:<label>` entry in `runnerRequirements`.
+ *   2. The default runner mapped from `environment.platform`.
+ *
+ * Throws when neither path yields a runner so missing platform mappings fail
+ * loudly during matrix generation rather than silently falling back to
+ * `ubuntu-latest` (which used to mask routing bugs in the legacy bash map).
+ */
+export function resolveRunnerForScenario(scenario: ScenarioDefinition): ResolvedRunner {
+  const explicit = (scenario.runnerRequirements ?? []).find((req) => req.startsWith("runs-on:"));
+  if (explicit) {
+    return { runner: explicit.slice("runs-on:".length), reason: "runnerRequirements override" };
+  }
+
+  const platform = scenario.environment?.platform;
+  if (platform && PLATFORM_DEFAULT_RUNNER[platform]) {
+    return { runner: PLATFORM_DEFAULT_RUNNER[platform], reason: `platform:${platform}` };
+  }
+
+  throw new Error(
+    `Cannot resolve runner for scenario '${scenario.id}': no runs-on override and no default for platform '${platform ?? "<missing>"}'.`,
+  );
+}

From e788fbf886732fbad13e661d83886744a0304dba Mon Sep 17 00:00:00 2001
From: Carlos Villela <cvillela@nvidia.com>
Date: Fri, 29 May 2026 15:42:08 -0700
Subject: [PATCH 2/2] test(e2e): reject empty scenario runner labels

---
 .../e2e-scenario-matrix.test.ts                | 18 ++++++++++++++++++
 test/e2e-scenario/scenarios/runner-routing.ts  |  8 +++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
index 3e06cb12a61..95ffa6db495 100644
--- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
+++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts
@@ -67,6 +67,24 @@ describe("typed scenario matrix", () => {
     expect(resolveRunnerForScenario(custom).runner).toBe("custom-self-hosted");
   });
 
+  it("rejects empty runs-on requirement overrides", () => {
+    const broken = scenario("test-empty-runs-on-override")
+      .description("test fixture")
+      .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml")
+      .environment({
+        platform: "ubuntu-local",
+        install: "repo-current",
+        runtime: "docker-running",
+        onboarding: "cloud-openclaw",
+      })
+      .expectedState("cloud-openclaw-ready")
+      .onboardingAssertions(["base-installed"])
+      .suites(["smoke"])
+      .runnerRequirements(["runs-on:   "])
+      .build();
+    expect(() => resolveRunnerForScenario(broken)).toThrow(/empty runs-on override/);
+  });
+
   it("fails loudly when a platform has no default runner mapping", () => {
     const broken = scenario("test-unknown-platform")
       .description("test fixture")
diff --git a/test/e2e-scenario/scenarios/runner-routing.ts b/test/e2e-scenario/scenarios/runner-routing.ts
index 45b11552818..fa59dbcbcf5 100644
--- a/test/e2e-scenario/scenarios/runner-routing.ts
+++ b/test/e2e-scenario/scenarios/runner-routing.ts
@@ -38,7 +38,13 @@ const PLATFORM_DEFAULT_RUNNER: Record<string, string> = {
 export function resolveRunnerForScenario(scenario: ScenarioDefinition): ResolvedRunner {
   const explicit = (scenario.runnerRequirements ?? []).find((req) => req.startsWith("runs-on:"));
   if (explicit) {
-    return { runner: explicit.slice("runs-on:".length), reason: "runnerRequirements override" };
+    const runner = explicit.slice("runs-on:".length).trim();
+    if (!runner) {
+      throw new Error(
+        `Cannot resolve runner for scenario '${scenario.id}': empty runs-on override.`,
+      );
+    }
+    return { runner, reason: "runnerRequirements override" };
   }
 
   const platform = scenario.environment?.platform;