From dbc672fb4f594d60262949539f46dae5e45620f9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 1 Jun 2026 23:42:01 -0700 Subject: [PATCH 1/3] test(e2e): add scenario aliases for layered plans --- .../e2e-scenario-resolver.test.ts | 15 +++ .../nemoclaw_scenarios/scenarios.yaml | 126 ++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-resolver.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-resolver.test.ts index dc4f105884e..31965cffcb0 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-resolver.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-resolver.test.ts @@ -10,6 +10,7 @@ import yaml from "js-yaml"; import { resolveScenario, type ResolverInput } from "../runtime/resolver/plan.ts"; import { loadMetadataFromDir, loadMetadataFromObjects } from "../runtime/resolver/load.ts"; +import { listScenarios } from "../scenarios/registry.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const E2E_DIR = path.join(REPO_ROOT, "test/e2e-scenario"); @@ -66,6 +67,20 @@ describe("E2E scenario resolver", () => { }); }); + it("should_resolve_every_typed_scenario_id_through_yaml_setup_scenarios", () => { + const meta = realMetadata(); + const failures = listScenarios().flatMap((scenario) => { + try { + resolveScenario(scenario.id, meta); + return []; + } catch (error) { + return [`${scenario.id}: ${(error as Error).message}`]; + } + }); + + expect(failures, failures.join("\n")).toEqual([]); + }); + it("should_fail_for_unknown_scenario", () => { const meta = realMetadata(); expect(() => resolveScenario("does-not-exist", meta)).toThrow(/does-not-exist/); diff --git a/test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml b/test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml index 36bbfb517f4..baaafa3b8c0 100644 --- a/test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml +++ b/test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml @@ -213,6 +213,132 @@ setup_scenarios: - credentials - onboarding-state - baseline-onboarding + ubuntu-repo-openai-compatible-openclaw: + alias_for_plan: ubuntu-repo-docker__openai-compatible-openclaw + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: openai-compatible-openclaw + expected_state: cloud-openclaw-ready + suites: + - smoke + ubuntu-repo-cloud-openclaw-brave: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-brave + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-brave + expected_state: cloud-openclaw-ready + suites: + - smoke + ubuntu-repo-cloud-openclaw-telegram: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-telegram + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-telegram + expected_state: cloud-openclaw-ready + suites: + - smoke + - messaging-telegram + ubuntu-repo-cloud-openclaw-discord: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-discord + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-discord + expected_state: cloud-openclaw-ready + suites: + - smoke + - messaging-discord + ubuntu-repo-cloud-openclaw-slack: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-slack + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-slack + expected_state: cloud-openclaw-ready + suites: + - smoke + - messaging-slack + ubuntu-repo-cloud-hermes-discord: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-hermes-discord + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-hermes-discord + expected_state: cloud-hermes-ready + suites: + - smoke + - messaging-discord + ubuntu-repo-cloud-hermes-slack: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-hermes-slack + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-hermes-slack + expected_state: cloud-hermes-ready + suites: + - smoke + - messaging-slack + ubuntu-repo-cloud-openclaw-resume: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-resume-after-interrupt + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-resume-after-interrupt + expected_state: cloud-openclaw-ready + suites: + - smoke + ubuntu-repo-cloud-openclaw-repair: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-repair-existing-config + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-repair-existing-config + expected_state: cloud-openclaw-ready + suites: + - smoke + ubuntu-repo-cloud-openclaw-double-same-provider: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-double-same-provider + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-double-same-provider + expected_state: cloud-openclaw-ready + suites: + - smoke + ubuntu-repo-cloud-openclaw-double-provider-switch: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-double-provider-switch + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-double-provider-switch + expected_state: cloud-openclaw-ready + suites: + - smoke + ubuntu-repo-cloud-openclaw-token-rotation: + alias_for_plan: ubuntu-repo-docker__cloud-nvidia-openclaw-token-rotation + dimensions: + platform: ubuntu-local + install: repo-current + runtime: docker-running + onboarding: cloud-nvidia-openclaw-token-rotation + expected_state: cloud-openclaw-ready + suites: + - smoke + - messaging-token-rotation ubuntu-invalid-nvidia-key-negative: dimensions: platform: ubuntu-local From c66d2a31f710706cbc1787239f00b4668dc42442 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 2 Jun 2026 00:18:02 -0700 Subject: [PATCH 2/3] docs(e2e): reconcile scenario migration docs --- test/e2e-scenario/docs/MIGRATION.md | 228 ++++++++++------------------ test/e2e-scenario/docs/README.md | 170 ++++++++++++--------- 2 files changed, 182 insertions(+), 216 deletions(-) diff --git a/test/e2e-scenario/docs/MIGRATION.md b/test/e2e-scenario/docs/MIGRATION.md index cf3d0e056dc..4a2e7c212b0 100644 --- a/test/e2e-scenario/docs/MIGRATION.md +++ b/test/e2e-scenario/docs/MIGRATION.md @@ -1,149 +1,85 @@ -# E2E Migration Tracker - -This PR migrates all existing `test/e2e/test-*.sh` scripts into the -scenario-based runner introduced by PR #3363. Full deep migration -(Strategy B). Legacy scripts remain in the repo during this PR and run -in parallel for 1–2 nightly cycles after merge; a follow-up PR retires -them once parity is verified. - -**Merge gate:** All 40 legacy entry points must have a scenario-based -equivalent that produces the same PASS/FAIL outcomes as the legacy -script in a side-by-side CI run. - -## Reuse being absorbed - -Migrating 40 scripts collapses 13 distinct categories of duplication. -Each row maps to a Wave 0 item or an existing helper. - -| # | Category | Fan-in (legacy) | Target absorber | LOC | -|---|---|---|---|---:| -| 1 | Logging helpers (`section` / `info` / `pass` / `fail`) | 28–39 scripts redefine each | `runtime/lib/logging.sh` (Wave 0.B.5) | 1,556 | -| 2 | Non-interactive env exports | 187 inlined lines across 40 scripts | `runtime/lib/env.sh::e2e_env_apply_noninteractive` + convention 0.G.1 | 175 | -| 3 | Repo-root / `SCRIPT_DIR` discovery | 37 lines, 4 competing patterns | One convention (Wave 0.G.2) | 25 | -| 4 | `nemoclaw list` / `status` / gateway state probes | 142 inlined sites | `validation_suites/assert/{gateway,sandbox}-alive.sh` | 500 | -| 5 | `bash install.sh ...` invocations | 24 scripts | `nemoclaw_scenarios/install/dispatch.sh` dispatcher (Wave 0.C.1) | 300 | -| 6 | `nemoclaw onboard ...` variants | 42 invocations, 8+ flag incantations | `nemoclaw_scenarios/onboard/dispatch.sh` + profile handlers | 800 | -| 7 | Docker older-base-image pattern | 3 hand-rolled implementations | `nemoclaw_scenarios/fixtures/older-base-image.sh` (Wave 0.A.1) | 250 | -| 8 | Trap / cleanup / teardown blocks | 112 lines, ~15 patterns | `runtime/lib/cleanup.sh` + convention 0.G.3 | 400 | -| 9 | Fake-endpoint inline setups | 3 inline variants | `nemoclaw_scenarios/fixtures/fake-{openai,telegram,discord,slack}.sh` (Wave 0.A.2–5) | 150 | -| 10 | Sandbox-scoped exec (`nemoclaw shell -- ...`) | 15 scripts reimplement with drift | `validation_suites/sandbox-exec.sh` (Wave 0.A.6) | 200 | -| 11 | Hermes/OpenClaw pair-variant scripts | 7 paired scripts share ~70% | Shared suite steps; scenario agent via `expected_state.sandbox.agent` | 800 | -| 12 | `section "Phase N: X"` markers | Every script inflates logs with phase text | Step-script filename carries the name (convention 0.G.4) | 300 | -| 13 | Log-capture paths (`/tmp/*.log`) | 25 different conventions; CI artifact upload assumes one | `$E2E_CONTEXT_DIR/logs/` convention 0.G.5 | 300 | -| **Total** | | | | **~5,556** | - -About **25% LOC reduction** net after legacy retirement. The larger win -is drift reduction: when `--yes-i-accept-third-party-software` renames -again, it's a 1-file change instead of a 24-file change. - -## Status summary - -| Bucket | Legacy LOC | Status | -|---|---:|---| -| Wave 0 — fixtures, asserts, setup splits, conventions, parity workflow | — | ⬜ not started | -| Wave 1 — onboarding baseline | 1,101 | ⬜ | -| Wave 2 — onboarding lifecycle | 2,013 | ⬜ | -| Wave 3 — sandbox lifecycle | 2,891 | ⬜ | -| Wave 4 — rebuild / upgrade | 1,292 | ⬜ | -| Wave 5 — inference variants | 2,593 | ⬜ | -| Wave 6 — Hermes | 1,646 | ⬜ | -| Wave 7 — messaging | 3,397 | ⬜ | -| Wave 8 — security / policy | 2,241 | ⬜ | -| Wave 9 — runtime / platform services | 1,696 | ⬜ | -| Wave 10 — platform + remote | 1,589 | ⬜ | -| Wave 11 — misc | 405 | ⬜ | -| **Total** | **20,864** | **0 / 40 scripts migrated** | - -## Per-script tracker - -Legend: ⬜ not started · 🟨 in progress · ✅ migrated · 🔵 parity verified - -### Wave 1 — onboarding baseline - -- ⬜ `test-full-e2e.sh` (473) → `onboarding/happy-path/` + scenario `ubuntu-curl-cloud-openclaw` -- ⬜ `test-cloud-onboard-e2e.sh` (337) → `onboarding/public-installer/` -- ⬜ `test-cloud-inference-e2e.sh` (291) → extends `inference/cloud/` - -### Wave 2 — onboarding lifecycle - -- ⬜ `test-double-onboard.sh` (717) → `onboarding/double-onboard/` -- ⬜ `test-gpu-double-onboard.sh` (571) → `onboarding/double-onboard/` on GPU scenario -- ⬜ `test-onboard-repair.sh` (372) → `onboarding/repair/` -- ⬜ `test-onboard-resume.sh` (353) → `onboarding/resume/` - -### Wave 3 — sandbox lifecycle - -- ⬜ `test-sandbox-operations.sh` (828) → `sandbox/operations/` -- ⬜ `test-sandbox-survival.sh` (721) → `sandbox/survival/` -- ⬜ `test-snapshot-commands.sh` (281) → `sandbox/snapshot/` -- ⬜ `test-diagnostics.sh` (452) → `sandbox/diagnostics/` -- ⬜ `test-issue-2478-crash-loop-recovery.sh` (609) → `sandbox/crash-loop-recovery/` - -### Wave 4 — rebuild / upgrade - -- ⬜ `test-rebuild-openclaw.sh` (453) → `sandbox/rebuild-openclaw/` (uses `nemoclaw_scenarios/fixtures/older-base-image.sh`) -- ⬜ `test-rebuild-hermes.sh` (401) → `sandbox/rebuild-hermes/` -- ⬜ `test-upgrade-stale-sandbox.sh` (241) → `sandbox/upgrade-stale/` -- ⬜ `test-sandbox-rebuild.sh` (197) → folded into `sandbox/rebuild-openclaw/` - -### Wave 5 — inference variants - -- ⬜ `test-gpu-e2e.sh` (565) → `inference/ollama-gpu/` (deep port) -- ⬜ `test-ollama-auth-proxy-e2e.sh` (548) → `inference/ollama-auth-proxy/` (deep port) -- ⬜ `test-inference-routing.sh` (715) → `inference/routing-errors/` -- ⬜ `test-kimi-inference-compat.sh` (765) → `inference/kimi-compat/` - -### Wave 6 — Hermes - -- ⬜ `test-hermes-e2e.sh` (591) → `onboarding/hermes/` (deep port; currently 1-step health) -- ⬜ `test-hermes-slack-e2e.sh` (537) → `messaging/slack/hermes/` -- ⬜ `test-hermes-discord-e2e.sh` (518) → `messaging/discord/hermes/` - -### Wave 7 — messaging - -- ⬜ `test-messaging-providers.sh` (1,677) → `messaging/providers/{telegram,discord,slack}/` -- ⬜ `test-token-rotation.sh` (575) → `messaging/token-rotation/` -- ⬜ `test-telegram-injection.sh` (475) → `security/telegram-injection/` -- ⬜ `test-messaging-compatible-endpoint.sh` (670) → `messaging/compatible-endpoint/` - -### Wave 8 — security / policy - -- ⬜ `test-shields-config.sh` (550) → `security/shields/` -- ⬜ `test-network-policy.sh` (579) → `security/network-policy/` -- ⬜ `test-credential-sanitization.sh` (810) → `security/credentials/sanitization/` -- ⬜ `test-credential-migration.sh` (302) → `security/credentials/migration/` - -### Wave 9 — runtime / platform services - -- ⬜ `test-runtime-overrides.sh` (272) → `sandbox/runtime-overrides/` -- ⬜ `test-overlayfs-autofix.sh` (537) → `sandbox/overlayfs-autofix/` -- ⬜ `test-device-auth-health.sh` (373) → `lifecycle/device-auth-health/` -- ⬜ `test-state-backup-restore.sh` (378) → `lifecycle/state-backup-restore/` -- ⬜ `test-tunnel-lifecycle.sh` (472) → `lifecycle/tunnel-lifecycle/` - -### Wave 10 — platform + remote - -- ⬜ `test-spark-install.sh` (157) → `platform/spark/` -- ⬜ `test-launchable-smoke.sh` (589) → `platform/launchable/` -- ⬜ `brev-e2e.test.ts` (843) → `platform/brev-remote/` - -### Wave 11 — misc - -- ⬜ `test-skill-agent-e2e.sh` (244) → `onboarding/skill-agent/` -- ⬜ `test-docs-validation.sh` (161) → `lifecycle/docs-validation/` - -## Migration tracking - -The old workflow-level parity report has been removed. Migration is tracked by -coverage domain under issue #3588 and its child issues. For each domain, add the -missing primitive layer first, then migrate assertions into scenario plans and -post-onboard suites with stable assertion IDs. - -Use the scenario coverage report plus code review to answer: - -- which legacy/nightly behaviors are now represented in scenarios, -- which behaviors remain outstanding for the domain issue, and -- which legacy behaviors should be retired rather than ported. +# E2E scenario migration notes + +This file records the current migration model for contributors. It is not the +source of truth for per-domain status. Keep mutable migration state in GitHub +issues and PRs so reviewers can discuss, update, and close work in one place. + +## Current migration state + +The scenario E2E migration is in a hybrid phase: + +- typed scenario builders drive scenario workflow fan-out and dry-run plans; +- product-facing `NemoClawInstance` manifests describe desired setup and + onboarding state; +- YAML metadata still drives the shell scenario runner and live suite + resolution; +- legacy `test/e2e/test-*.sh` scripts still provide most live nightly and + platform coverage. + +Do not assume legacy scripts are deletion-ready just because a scenario or suite +name exists. The final reconciliation phase must show either evidence-complete +coverage or an explicit audit amendment before legacy executable tests are +removed. + +## Active issue tracking + +Use these GitHub issues for status and follow-up work: + +| Issue | Purpose | +| --- | --- | +| #3588 | Parent architecture epic for layered / hybrid scenario E2E | +| #4347–#4356 | Domain-specific audit-coverage phases | +| #4357 | Final audit reconciliation, placeholder cleanup, and deletion-readiness review | +| #4378 | Friendly `setup_scenarios` aliases for layered test plans | + +If a migration discovery needs durable tracking, add it to the relevant issue or +open a focused child issue. Avoid adding long-lived checklists here. + +## What to migrate next + +When moving behavior from a legacy E2E script into the scenario framework: + +1. Identify the relevant audit issue (#4347–#4356). +2. Add or update the product-facing manifest only when the desired setup or + onboarding state changes. +3. Add typed scenario registry coverage when the workflow matrix needs a new + canonical scenario ID. +4. Add YAML metadata when the shell runner needs to resolve or execute the plan. +5. Add reusable suite or assertion helpers instead of copying entire legacy + scripts. +6. Add framework tests that prevent the typed registry, YAML aliases, workflow + routes, manifests, and suites from drifting. +7. Leave legacy executable scripts in place until #4357 records deletion + readiness. + +## Useful commands + +```bash +# Typed registry inventory and dry-run path +npx tsx test/e2e-scenario/scenarios/run.ts --list +npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix +npx tsx test/e2e-scenario/scenarios/run.ts --scenarios --plan-only +npx tsx test/e2e-scenario/scenarios/run.ts --scenarios --dry-run + +# YAML/shell resolver and live runner path +bash test/e2e-scenario/runtime/run-scenario.sh --plan-only +bash test/e2e-scenario/runtime/run-scenario.sh --dry-run +bash test/e2e-scenario/runtime/run-scenario.sh + +# Metadata and framework validation +bash test/e2e-scenario/runtime/coverage-report.sh +npx vitest run --project e2e-scenario-framework --silent=false --reporter=default +``` + +## Cleanup rules + +- Prefer new scenario-matrix coverage over new legacy-style `test-*.sh` scripts. +- Do not reintroduce the removed workflow-level parity report unless maintainers + explicitly reopen that direction. +- Do not delete legacy executable E2Es as part of ordinary domain migration PRs; + queue deletion candidates for #4357. +- Keep docs focused on how the framework works now. Put changing progress status + in issues and PRs. diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index 15ad01d88d7..c56f8d05d13 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -1,98 +1,128 @@ -# NemoClaw E2E +# NemoClaw E2E scenario framework -End-to-end tests organized around **setup scenarios** rather than -one-off shell scripts. A scenario declares *how you got to a working -NemoClaw* (platform + install + runtime + onboarding); a scenario -resolves to an **expected state** contract; once that state validates, -one or more **suites** run functional assertions against it. +NemoClaw's scenario E2E framework is currently a **hybrid** migration model. +It combines typed scenario builders, product-facing setup manifests, YAML +runtime metadata, and reusable shell suites while the older live E2E scripts +continue to run in parallel. -```text -setup scenario → expected state → suite sequence -``` +## Current sources of truth -The declarative sources of truth live in three files — read these -first, they are short and deliberately not redundant with prose: +Use the source that matches the task: -- [`../nemoclaw_scenarios/scenarios.yaml`](../nemoclaw_scenarios/scenarios.yaml) - — platforms, installs, runtimes, onboarding choices, and the - concrete scenarios that combine them. -- [`../nemoclaw_scenarios/expected-states.yaml`](../nemoclaw_scenarios/expected-states.yaml) - — reusable structural contracts (gateway health, sandbox status, - inference routing, etc.). -- [`../validation_suites/suites.yaml`](../validation_suites/suites.yaml) - — ordered validation steps, each with a `requires_state` predicate. +| Task | Current source | +| --- | --- | +| Scenario workflow fan-out and dry-run planning | `test/e2e-scenario/scenarios/registry.ts`, `test/e2e-scenario/scenarios/scenarios/baseline.ts`, and `test/e2e-scenario/scenarios/run.ts` | +| Product-facing desired setup/onboarding state | `test/e2e-scenario/manifests/*.yaml` | +| Shell runner scenario resolution and live scenario execution | `test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml`, `expected-states.yaml`, and `validation_suites/suites.yaml` | +| Reusable live suite assertions | `test/e2e-scenario/validation_suites/` | +| Existing nightly and platform E2E coverage | legacy `test/e2e/test-*.sh` scripts and their workflows | -## Layered scenario model +The migration goal is to keep these surfaces aligned while progressively moving +coverage into scenario contracts and suites. Do not add new legacy-style +`test/e2e/test-*.sh` entrypoints unless there is a specific maintainer-approved +reason. -The E2E source of truth is layered as base environment, onboarding profile, -test plan, expected state, and post-onboard suites. Test plans can also declare -onboarding assertions that run after install/onboard and before expected-state -validation. +## Layered scenario model -Plan-only resolution accepts either an alias or a test plan ID: +The conceptual model is layered: -```bash -bash test/e2e-scenario/runtime/run-scenario.sh ubuntu-repo-cloud-openclaw --plan-only -bash test/e2e-scenario/runtime/run-scenario.sh ubuntu-repo-docker__cloud-nvidia-openclaw --plan-only +```text +base environment + → onboarding profile / manifest + → onboarding assertions + → expected state + → post-onboard suites ``` +The YAML shell runner expresses this through: + +- `base_scenarios`: platform + install + runtime +- `onboarding_profiles`: user onboarding choices +- `test_plans`: base + onboarding + expected state + suites +- `setup_scenarios`: friendly aliases and compatibility metadata +- `onboarding_assertions`: setup/onboarding checks that run before suites + +The typed scenario registry expresses the same intent as deterministic code and +is used by the scenario workflow matrix and dry-run plan artifacts. + ## How to run ```bash -bash test/e2e-scenario/runtime/run-scenario.sh --plan-only # resolve + print plan, no side effects -bash test/e2e-scenario/runtime/run-scenario.sh --dry-run # helpers short-circuit with trace -bash test/e2e-scenario/runtime/run-scenario.sh --validate-only # assume setup done; validate expected state -bash test/e2e-scenario/runtime/run-scenario.sh # full live run -bash test/e2e-scenario/runtime/run-suites.sh […] -bash test/e2e-scenario/runtime/coverage-report.sh # Markdown matrix of scenario × suite +# YAML/shell scenario runner +bash test/e2e-scenario/runtime/run-scenario.sh --plan-only +bash test/e2e-scenario/runtime/run-scenario.sh --dry-run +bash test/e2e-scenario/runtime/run-scenario.sh --validate-only +bash test/e2e-scenario/runtime/run-scenario.sh + +# Suite runner against an existing scenario context +bash test/e2e-scenario/runtime/run-suites.sh [...] + +# Scenario metadata coverage report +bash test/e2e-scenario/runtime/coverage-report.sh + +# Typed scenario registry / workflow dry-run path +npx tsx test/e2e-scenario/scenarios/run.ts --list +npx tsx test/e2e-scenario/scenarios/run.ts --scenarios --plan-only +npx tsx test/e2e-scenario/scenarios/run.ts --scenarios --dry-run +npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix ``` -Override the runtime context dir with `E2E_CONTEXT_DIR=` (default -`.e2e/`, gitignored). The scenario runner and suites communicate only -through `$E2E_CONTEXT_DIR/context.env` — suites do not rediscover -setup state. +Override the runtime context directory with `E2E_CONTEXT_DIR=` (default +`.e2e/`, gitignored). The shell scenario runner and suites communicate through +`$E2E_CONTEXT_DIR/context.env`; suites should not rediscover setup state. -## Where things live +## Repository layout ```text -test/e2e/ - docs/ # README.md, MIGRATION.md - nemoclaw_scenarios/ # declarative scenario inputs + setup machinery - scenarios.yaml / expected-states.yaml - install/ # install dispatcher + one file per install profile - onboard/ # onboard dispatcher + one file per onboarding profile - fixtures/ # reusable stubs (fake-openai, fake-{telegram,discord,slack}, older-base-image) - helpers/ # scenario-side shell utilities (e.g. emit-context-from-plan.sh) - validation_suites/ # suite definitions and outcome assertions +test/e2e-scenario/ + docs/ # This guide and migration notes + manifests/ # Product-facing NemoClawInstance desired state + scenarios/ # Typed builders, registry, compiler, assertions, dry-run orchestration + nemoclaw_scenarios/ # YAML runtime metadata and setup helpers + scenarios.yaml + expected-states.yaml + install/ + onboard/ + fixtures/ + helpers/ + validation_suites/ # Suite definitions and shell assertion steps suites.yaml - sandbox-exec.sh - assert/ # outcome assertions (inference, credentials, policy, messaging) - smoke/ inference/ hermes/ platform/ security/ # suite scripts grouped by concern - runtime/ # entry points + cross-cutting shared libs - run-scenario.sh / run-suites.sh / coverage-report.sh - resolver/ # TypeScript: load, plan, validate, coverage (invoked via tsx) - lib/ # shared shell helpers: context, env, cleanup, logging, artifacts, sandbox-teardown + smoke/ + inference/ + messaging/ + platform/ + security/ + sandbox/ + runtime/ # Shell runner, suite runner, resolver, coverage report, shared libs + run-scenario.sh + run-suites.sh + coverage-report.sh + resolver/ + lib/ ``` -The CI entry point is `.github/workflows/e2e-scenarios.yaml` (manual dispatch). Existing legacy workflows (`nightly-e2e.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, etc.) remain in place during the migration. - -Migration coverage is tracked through the layered scenario definitions, suite inventory, and the domain migration issues linked from issue #3588. Do not add a workflow-level parity report or assertion-ledger gate; use focused code review and the scenario coverage report to decide what to migrate next. +## CI entry points -## How to add a scenario, state, or suite +- `.github/workflows/e2e-scenarios.yaml` runs typed scenario dry-runs for + manually selected scenario IDs. +- `.github/workflows/e2e-scenarios-all.yaml` fans out typed scenario dry-runs + from the typed registry matrix. +- Existing workflows such as `nightly-e2e.yaml`, `e2e-branch-validation.yaml`, + `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and + `regression-e2e.yaml` still run legacy live E2E scripts during the migration. +- `vitest.config.ts` contains the `e2e-scenario-framework` project for framework + and metadata tests. -Add-a-scenario, add-a-state, and add-a-suite are short edits to the -three YAML files above, plus shell scripts under -`nemoclaw_scenarios/install/`, `nemoclaw_scenarios/onboard/`, -`validation_suites/assert/`, or `validation_suites//`. The -schemas in -[`../runtime/resolver/schema.ts`](../runtime/resolver/schema.ts) -describe the required shape; `run-scenario.sh --plan-only` -validates your change without running anything destructive. +## Migration tracking -When adding a suite assertion, emit or preserve a stable `PASS: ` / -`FAIL: ` log line, and update migration coverage through the scenario coverage report and the domain issues under `#3588`. Sandbox lifecycle assertions should use `validation_suites/lib/sandbox_lifecycle.sh`, consume `$E2E_CONTEXT_DIR/context.env`, and keep destructive snapshot restore checks isolated in the opt-in `snapshot-lifecycle` suite. Platform-specific scenarios such as GPU, macOS, WSL, Brev, or DGX Spark must also list `runner_requirements` in `scenarios.yaml`. +Keep mutable migration state in GitHub issues and PRs, not in repo-local +checklists. The parent architecture issue is #3588. Active audit-coverage work +is tracked by the #4347–#4357 issue set, with focused follow-ups such as #4378 +for specific drift fixes. -Prefer new scenario-matrix coverage over new legacy-style `test-*.sh` scripts. +The old workflow-level parity report has been removed. Use scenario framework +tests, the coverage report, PR review, and the audit issues to decide what to +migrate next. From 401c631217b1864e9f8f7e1f0d29f639d40973ca Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 2 Jun 2026 00:21:20 -0700 Subject: [PATCH 3/3] test(e2e): guard scenario workflow drift --- .github/workflows/e2e-scenarios.yaml | 3 +++ .../e2e-coverage-report.test.ts | 2 ++ .../e2e-scenarios-workflow.test.ts | 23 +++++++++++++++++++ .../e2e-scenario/runtime/resolver/coverage.ts | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 5f1dfc8b376..acadc7754ff 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -60,12 +60,15 @@ jobs: [wsl-repo-cloud-openclaw]=windows-latest [gpu-repo-local-ollama-openclaw]=linux-amd64-gpu-rtxpro6000-latest-1 [brev-launchable-cloud-openclaw]=ubuntu-latest + [ubuntu-gateway-port-conflict-negative]=ubuntu-latest + [ubuntu-invalid-nvidia-key-negative]=ubuntu-latest [ubuntu-no-docker-preflight-negative]=ubuntu-latest [ubuntu-repo-cloud-hermes]=ubuntu-latest [ubuntu-repo-cloud-hermes-discord]=ubuntu-latest [ubuntu-repo-cloud-hermes-slack]=ubuntu-latest [ubuntu-repo-cloud-openclaw]=ubuntu-latest [ubuntu-repo-cloud-openclaw-brave]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-custom-policies]=ubuntu-latest [ubuntu-repo-cloud-openclaw-discord]=ubuntu-latest [ubuntu-repo-cloud-openclaw-double-provider-switch]=ubuntu-latest [ubuntu-repo-cloud-openclaw-double-same-provider]=ubuntu-latest diff --git a/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts b/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts index 109e359f85a..b4a6056db06 100644 --- a/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts @@ -14,6 +14,8 @@ describe("coverage report", () => { it("should_render_single_coverage_table", () => { const meta = loadMetadataFromDir(E2E_DIR); const md = renderCoverageReport(meta); + expect(md).toContain("test/e2e-scenario/nemoclaw_scenarios/{scenarios,expected-states}.yaml"); + expect(md).toContain("test/e2e-scenario/validation_suites/suites.yaml"); // Exactly one primary Scenario Coverage table. const headers = md.match(/\|\s*Scenario\s*\|\s*Platform\s*\|\s*Install\s*\|\s*Runtime\s*\|\s*Onboarding\s*\|\s*Expected state\s*\|\s*Suites\s*\|/g); expect(headers).toBeTruthy(); diff --git a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index eb1be9ae191..7bc03bb96e3 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -7,13 +7,36 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { listScenarios } from "../scenarios/registry.ts"; import { validateE2eScenariosWorkflowBoundary } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e-scenarios.yaml"); + +function routeIdsFromWorkflow(workflowPath = WORKFLOW_PATH): string[] { + const workflow = fs.readFileSync(workflowPath, "utf8"); + const match = /declare -A ROUTES=\(\n(?[\s\S]*?)\n\s*\)/.exec(workflow); + if (!match?.groups?.body) { + throw new Error("Could not find ROUTES table in e2e-scenarios.yaml"); + } + return Array.from(match.groups.body.matchAll(/^\s*\[([^\]]+)\]=/gm), ([, id]) => id).sort(); +} + describe("e2e-scenarios workflow boundary", () => { it("keeps scenario execution manual/reusable and artifact-safe", () => { expect(validateE2eScenariosWorkflowBoundary()).toEqual([]); }); + it("routes_every_typed_scenario_id", () => { + const typedIds = listScenarios().map((scenario) => scenario.id).sort(); + const routeIds = routeIdsFromWorkflow(); + const missing = typedIds.filter((id) => !routeIds.includes(id)); + const extra = routeIds.filter((id) => !typedIds.includes(id)); + + expect(missing, `workflow ROUTES missing typed scenario IDs: ${missing.join(", ")}`).toEqual([]); + expect(extra, `workflow ROUTES has unknown scenario IDs: ${extra.join(", ")}`).toEqual([]); + }); + it("flags unsafe trigger and contract regressions", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-scenarios-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e-scenario/runtime/resolver/coverage.ts b/test/e2e-scenario/runtime/resolver/coverage.ts index 4eefc79dc27..2a3110f40cc 100644 --- a/test/e2e-scenario/runtime/resolver/coverage.ts +++ b/test/e2e-scenario/runtime/resolver/coverage.ts @@ -27,7 +27,7 @@ export function renderCoverageReport( lines.push("# E2E Setup Scenario Coverage"); lines.push(""); lines.push( - "_Generated from `test/e2e/{scenarios,expected-states,suites}.yaml`._", + "_Generated from `test/e2e-scenario/nemoclaw_scenarios/{scenarios,expected-states}.yaml` and `test/e2e-scenario/validation_suites/suites.yaml`._", ); lines.push(""); lines.push("## Base Scenarios");