diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index b9e8f2f822b..9e2661a8788 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -60,12 +60,15 @@ jobs: [wsl-repo-cloud-openclaw]=windows-latest [gpu-repo-local-ollama-openclaw]=linux-amd64-gpu-rtxpro6000-latest-1 [brev-launchable-cloud-openclaw]=ubuntu-latest + [ubuntu-gateway-port-conflict-negative]=ubuntu-latest + [ubuntu-invalid-nvidia-key-negative]=ubuntu-latest [ubuntu-no-docker-preflight-negative]=ubuntu-latest [ubuntu-repo-cloud-hermes]=ubuntu-latest [ubuntu-repo-cloud-hermes-discord]=ubuntu-latest [ubuntu-repo-cloud-hermes-slack]=ubuntu-latest [ubuntu-repo-cloud-openclaw]=ubuntu-latest [ubuntu-repo-cloud-openclaw-brave]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-custom-policies]=ubuntu-latest [ubuntu-repo-cloud-openclaw-discord]=ubuntu-latest [ubuntu-repo-cloud-openclaw-double-provider-switch]=ubuntu-latest [ubuntu-repo-cloud-openclaw-double-same-provider]=ubuntu-latest diff --git a/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts b/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts index 109e359f85a..b4a6056db06 100644 --- a/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts @@ -14,6 +14,8 @@ describe("coverage report", () => { it("should_render_single_coverage_table", () => { const meta = loadMetadataFromDir(E2E_DIR); const md = renderCoverageReport(meta); + expect(md).toContain("test/e2e-scenario/nemoclaw_scenarios/{scenarios,expected-states}.yaml"); + expect(md).toContain("test/e2e-scenario/validation_suites/suites.yaml"); // Exactly one primary Scenario Coverage table. const headers = md.match(/\|\s*Scenario\s*\|\s*Platform\s*\|\s*Install\s*\|\s*Runtime\s*\|\s*Onboarding\s*\|\s*Expected state\s*\|\s*Suites\s*\|/g); expect(headers).toBeTruthy(); diff --git a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index eb1be9ae191..604ec1c0334 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -7,13 +7,58 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { listScenarios } from "../scenarios/registry.ts"; +import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts"; import { validateE2eScenariosWorkflowBoundary } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e-scenarios.yaml"); + +function routesFromWorkflow(workflowPath = WORKFLOW_PATH): Map { + const workflow = fs.readFileSync(workflowPath, "utf8"); + const match = /declare -A ROUTES=\(\n(?[\s\S]*?)\n\s*\)/.exec(workflow); + if (!match?.groups?.body) { + throw new Error("Could not find ROUTES table in e2e-scenarios.yaml"); + } + return new Map( + Array.from(match.groups.body.matchAll(/^\s*\[([^\]]+)\]=([^\s)]+)\s*$/gm), ([, id, runner]) => [ + id, + runner, + ]), + ); +} + describe("e2e-scenarios workflow boundary", () => { it("keeps scenario execution manual/reusable and artifact-safe", () => { expect(validateE2eScenariosWorkflowBoundary()).toEqual([]); }); + it("routes_every_typed_scenario_id_to_its_resolved_runner", () => { + const scenarios = listScenarios().sort((left, right) => left.id.localeCompare(right.id)); + const routes = routesFromWorkflow(); + const typedIds = scenarios.map((scenario) => scenario.id); + const routeIds = Array.from(routes.keys()).sort(); + const missing = typedIds.filter((id) => !routeIds.includes(id)); + const extra = routeIds.filter((id) => !typedIds.includes(id)); + const runnerMismatches = scenarios.flatMap((scenario) => { + const workflowRunner = routes.get(scenario.id); + if (!workflowRunner) { + return []; + } + const resolvedRunner = resolveRunnerForScenario(scenario).runner; + return workflowRunner === resolvedRunner + ? [] + : [`${scenario.id}: workflow=${workflowRunner}, typed=${resolvedRunner}`]; + }); + + expect(missing, `workflow ROUTES missing typed scenario IDs: ${missing.join(", ")}`).toEqual([]); + expect(extra, `workflow ROUTES has unknown scenario IDs: ${extra.join(", ")}`).toEqual([]); + expect( + runnerMismatches, + `workflow ROUTES has runner mismatches: ${runnerMismatches.join("; ")}`, + ).toEqual([]); + }); + it("flags unsafe trigger and contract regressions", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-scenarios-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e-scenario/runtime/resolver/coverage.ts b/test/e2e-scenario/runtime/resolver/coverage.ts index 4eefc79dc27..2a3110f40cc 100644 --- a/test/e2e-scenario/runtime/resolver/coverage.ts +++ b/test/e2e-scenario/runtime/resolver/coverage.ts @@ -27,7 +27,7 @@ export function renderCoverageReport( lines.push("# E2E Setup Scenario Coverage"); lines.push(""); lines.push( - "_Generated from `test/e2e/{scenarios,expected-states,suites}.yaml`._", + "_Generated from `test/e2e-scenario/nemoclaw_scenarios/{scenarios,expected-states}.yaml` and `test/e2e-scenario/validation_suites/suites.yaml`._", ); lines.push(""); lines.push("## Base Scenarios");