test(e2e): execute real shell assertions; delete dry-run, --validate-only, and the bash runner

.github/workflows/e2e-scenarios-all.yaml

@@ -83,3 +83,10 @@ jobs:
       scenarios: ${{ matrix.id }}
     secrets:
       NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+
+  ubuntu-rebuild-openclaw:
+    uses: ./.github/workflows/e2e-scenarios.yaml
+    with:
+      scenarios: ubuntu-rebuild-openclaw
+    secrets:
+      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}

.github/workflows/e2e-scenarios.yaml

@@ -63,6 +63,7 @@ jobs:
             [ubuntu-gateway-port-conflict-negative]=ubuntu-latest
             [ubuntu-invalid-nvidia-key-negative]=ubuntu-latest
             [ubuntu-no-docker-preflight-negative]=ubuntu-latest
+            [ubuntu-rebuild-openclaw]=ubuntu-latest
             [ubuntu-repo-cloud-hermes]=ubuntu-latest
             [ubuntu-repo-cloud-hermes-discord]=ubuntu-latest
             [ubuntu-repo-cloud-hermes-slack]=ubuntu-latest
@@ -84,7 +85,6 @@ jobs:
           for raw in "${IDS[@]}"; do
             id="${raw//[[:space:]]/}"
             [ -n "${id}" ] || continue
-            npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${id}" --plan-only >/dev/null
             runner="${ROUTES[$id]:-}"
             if [ -z "${runner}" ]; then
               echo "::error::No runner route for scenario: ${id}" >&2
@@ -138,7 +138,7 @@ jobs:
             echo "::error::Invalid scenario input: ${SCENARIOS}" >&2
             exit 1
           fi
-          npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}" --dry-run
+          npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}"
 
       - name: Resolve workspace paths for WSL
         if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw')
@@ -302,7 +302,7 @@ jobs:
           export E2E_CONTEXT_DIR="`$workdir"
           npm ci --ignore-scripts
           set +e
-          npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "`$scenarios" --dry-run
+          npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "`$scenarios"
           status=`$?
           if [ -d "`$workdir/.e2e" ]; then
             rm -rf "`$checkout_dir/.e2e"
@@ -324,14 +324,14 @@ jobs:
             exit $LASTEXITCODE
           }
 
-      - name: Append typed dry-run summary
+      - name: Append typed scenario summary
         if: always()
         shell: bash
         run: |
           {
-            echo '## E2E typed scenario dry-run'
+            echo '## E2E typed scenario run'
             echo ''
-            echo 'Mode: `test/e2e-scenario/scenarios/run.ts --dry-run`.'
+            echo 'Mode: `test/e2e-scenario/scenarios/run.ts --scenarios <id[,id...]>` (live).'
             echo ''
             if [ -f .e2e/run-plan.json ]; then
               python3 - <<'PY'
@@ -368,14 +368,25 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-scenario-${{ inputs.scenarios || github.event.inputs.scenarios }}
+          # Explicit subpath list, NOT a blanket .e2e/ + hidden files.
+          # The framework redacts every byte that flows from spawned
+          # children into actions/*.log, logs/*.log, and onboard.log via
+          # orchestrators/redaction.ts::pipeRedacted. Anything outside
+          # the listed paths (notably the raw context.env file) is
+          # excluded so secret-bearing key=value lines cannot leak via
+          # the artifact even if a future helper writes there.
+          # Diagnostic dumps of context use e2e_context_dump, which
+          # redacts on emit (runtime/lib/context.sh).
           path: |
             .e2e/run-plan.json
             .e2e/plan.txt
             .e2e/environment.result.json
             .e2e/onboarding.result.json
             .e2e/runtime.result.json
-            .e2e/
+            .e2e/actions/
+            .e2e/logs/
+            .e2e/onboard.log
             test/e2e/logs/
           if-no-files-found: warn
           retention-days: 14
-          include-hidden-files: true
+          include-hidden-files: false

test/e2e-scenario/docs/README.md

@@ -1,12 +1,31 @@
 <!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
 <!-- SPDX-License-Identifier: Apache-2.0 -->
 
-# NemoClaw E2E scenario framework
+# NemoClaw E2E
 
-NemoClaw's scenario E2E framework is currently a **hybrid** migration model.
-It combines typed scenario builders, product-facing setup manifests, YAML
-runtime metadata, and reusable shell suites while the older live E2E scripts
-continue to run in parallel.
+End-to-end tests organized around **setup scenarios** rather than
+one-off shell scripts. A scenario declares *how you got to a working
+NemoClaw* (platform + install + runtime + onboarding); a scenario
+resolves to an **expected state** contract; once that state validates,
+one or more **suites** run functional assertions against it.
+
+```text
+setup scenario → expected state → suite sequence
+```
+
+The declarative sources of truth live in three files — read these
+first, they are short and deliberately not redundant with prose:
+
+- [`../nemoclaw_scenarios/scenarios.yaml`](../nemoclaw_scenarios/scenarios.yaml)
+  — platforms, installs, runtimes, onboarding choices, and the
+  concrete scenarios that combine them.
+- [`../scenarios/expected-states.ts`](../scenarios/expected-states.ts)
+  — typed registry of reusable structural contracts (gateway health,
+  sandbox status, inference routing, etc.). Single source of truth
+  since the legacy YAML resolver was retired.
+
+- [`../validation_suites/suites.yaml`](../validation_suites/suites.yaml)
+  — ordered validation steps, each with a `requires_state` predicate.
 
 This hybrid model is transitional. The target architecture for #3588 is a
 single scenario runner that owns scenario resolution, orchestration, evidence
@@ -20,17 +39,19 @@ Use the source that matches the task while the migration is in progress:
 
 | Task | Current source |
 | --- | --- |
-| Scenario workflow fan-out and dry-run planning | `test/e2e-scenario/scenarios/registry.ts`, `test/e2e-scenario/scenarios/scenarios/baseline.ts`, and `test/e2e-scenario/scenarios/run.ts` |
+| Scenario workflow fan-out and live execution | `test/e2e-scenario/scenarios/registry.ts`, `test/e2e-scenario/scenarios/scenarios/baseline.ts`, and `test/e2e-scenario/scenarios/run.ts` |
+| Typed expected-state registry (single source of truth) | `test/e2e-scenario/scenarios/expected-states.ts` |
 | Product-facing desired setup/onboarding state | `test/e2e-scenario/manifests/*.yaml` |
-| Shell runner scenario resolution and live scenario execution | `test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml`, `expected-states.yaml`, and `validation_suites/suites.yaml` |
 | Reusable live suite assertions | `test/e2e-scenario/validation_suites/` |
 | Existing nightly and platform E2E coverage | legacy `test/e2e/test-*.sh` scripts and their workflows |
 
 The near-term migration goal is to keep these surfaces aligned while coverage is
 being moved into scenario contracts and suites. The long-term goal is to remove
-the split between typed planning and shell execution. Do not add new
-legacy-style `test/e2e/test-*.sh` entrypoints unless there is a specific
-maintainer-approved reason.
+the split between typed planning and shell execution. The legacy YAML resolver
+under `runtime/resolver/` and `nemoclaw_scenarios/expected-states.yaml` have
+been retired; the typed registry is the single source of truth for expected
+states. Do not add new legacy-style `test/e2e/test-*.sh` entrypoints unless
+there is a specific maintainer-approved reason.
 
 ## Target runner model
 
@@ -59,115 +80,95 @@ explicitly decide to keep a bridge path for the current migration step.
 
 ## Layered scenario model
 
-The conceptual model is layered:
+The E2E source of truth is layered as base environment, onboarding profile,
+test plan, expected state, and post-onboard suites. Test plans can also declare
+onboarding assertions that run after install/onboard and before expected-state
+validation.
 
-```text
-base environment
-  → onboarding profile / manifest
-    → onboarding assertions
-      → expected state
-        → post-onboard suites
-```
+The typed TS runner enforces the contract by inserting a dedicated
+`state-validation` phase between onboarding and runtime. Probe actions
+are emitted from the typed expected-state registry
+(`scenarios/expected-states.ts`, mirrored to
+`nemoclaw_scenarios/expected-states.yaml` during transition):
 
-The current YAML shell runner expresses this through:
+- `cli-installed`, `gateway-healthy`, `sandbox-running` for ready states.
+- `gateway-absent`, `sandbox-absent` for negative/preflight-failure states.
 
-- `base_scenarios`: platform + install + runtime
-- `onboarding_profiles`: user onboarding choices
-- `test_plans`: base + onboarding + expected state + suites
-- `setup_scenarios`: friendly aliases and compatibility metadata
-- `onboarding_assertions`: setup/onboarding checks that run before suites
+A failed probe is a phase-action failure; the runner short-circuits
+the runtime phase rather than running suite assertions against a
+missing or wedged environment. An onboarding-phase failure does NOT
+block state-validation — negative scenarios depend on absent-state
+probes running after the deliberate onboarding failure to verify
+forbidden side effects (gateway/sandbox left behind) did not occur.
 
-The typed scenario registry expresses the same intent as deterministic code and
-is used by the scenario workflow matrix and dry-run plan artifacts. The target
-single runner should collapse these parallel expressions into one executable
-plan model.
+The target single runner should collapse the legacy parallel YAML expressions
+(`base_scenarios`, `onboarding_profiles`, `test_plans`, `setup_scenarios`,
+`onboarding_assertions`) into the single executable typed plan model above.
 
 ## How to run
 
+The TypeScript runner is the only supported entrypoint. There is one
+execution mode: live. There is no `--dry-run`, no `--validate-only`, no
+fake-pass code path. Plan output is emitted as a side effect of the
+live run.
+
 ```bash
-# YAML/shell scenario runner
-bash test/e2e-scenario/runtime/run-scenario.sh <id> --plan-only
-bash test/e2e-scenario/runtime/run-scenario.sh <id> --dry-run
-bash test/e2e-scenario/runtime/run-scenario.sh <id> --validate-only
-bash test/e2e-scenario/runtime/run-scenario.sh <id>
-
-# Suite runner against an existing scenario context
-bash test/e2e-scenario/runtime/run-suites.sh <suite-id> [<suite-id>...]
-
-# Scenario metadata coverage report
-bash test/e2e-scenario/runtime/coverage-report.sh
-
-# Typed scenario registry / workflow dry-run path
-npx tsx test/e2e-scenario/scenarios/run.ts --list
-npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id> --plan-only
-npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id> --dry-run
-npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix
+npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id[,id...]>     # live execution (the only mode)
+npx tsx test/e2e-scenario/scenarios/run.ts --list                       # list canonical scenario ids
+npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix                # JSON registry payload for CI matrix fan-out
+npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id> --plan-only # local debug only; MUST NOT appear in any workflow
+bash test/e2e-scenario/runtime/coverage-report.sh                       # Markdown matrix of scenario × suite
 ```
 
-Override the runtime context directory with `E2E_CONTEXT_DIR=<path>` (default
-`.e2e/`, gitignored). The shell scenario runner and suites communicate through
-`$E2E_CONTEXT_DIR/context.env`; suites should not rediscover setup state.
+The deprecated bash entrypoints `runtime/run-scenario.sh` and
+`runtime/run-suites.sh` exist only as fail-fast stubs; they print a
+pointer at `run.ts` and exit non-zero.
 
-## Repository layout
+Override the runtime context dir with `E2E_CONTEXT_DIR=<path>` (default
+`.e2e/`, gitignored). The scenario runner and suites communicate only
+through `$E2E_CONTEXT_DIR/context.env` — suites do not rediscover
+setup state.
+
+## Where things live
 
 ```text
-test/e2e-scenario/
-  docs/                              # This guide and migration notes
-  manifests/                         # Product-facing NemoClawInstance desired state
-  scenarios/                         # Typed builders, registry, compiler, assertions, dry-run orchestration
-  nemoclaw_scenarios/                # YAML runtime metadata and setup helpers
-    scenarios.yaml
-    expected-states.yaml
-    install/
-    onboard/
-    fixtures/
-    helpers/
-  validation_suites/                 # Suite definitions and shell assertion steps
+test/e2e/
+  docs/                              # README.md, MIGRATION.md
+  nemoclaw_scenarios/                # declarative scenario inputs + setup machinery
+    scenarios.yaml / expected-states.yaml
+    install/       # install dispatcher + one file per install profile
+    onboard/       # onboard dispatcher + one file per onboarding profile
+    fixtures/      # reusable stubs (fake-openai, fake-{telegram,discord,slack}, older-base-image)
+    helpers/       # scenario-side shell utilities (e.g. emit-context-from-plan.sh)
+  validation_suites/                 # suite definitions and outcome assertions
     suites.yaml
-    smoke/
-    inference/
-    messaging/
-    platform/
-    security/
-    sandbox/
-  runtime/                           # Shell runner, suite runner, resolver, coverage report, shared libs
-    run-scenario.sh
-    run-suites.sh
+    sandbox-exec.sh
+    assert/        # outcome assertions (inference, credentials, policy, messaging)
+    smoke/ inference/ hermes/ platform/ security/   # suite scripts grouped by concern
+  runtime/                           # entry points + cross-cutting shared libs
+    run-scenario.sh / run-suites.sh    # DEPRECATED fail-fast stubs (see above)
     coverage-report.sh
-    resolver/
-    lib/
+    resolver/      # TypeScript: load, plan, validate, coverage (invoked via tsx)
+    lib/           # shared shell helpers: context, env, cleanup, logging, artifacts, sandbox-teardown
 ```
 
-## CI entry points
-
-- `.github/workflows/e2e-scenarios.yaml` runs typed scenario dry-runs for
-  manually selected scenario IDs.
-- `.github/workflows/e2e-scenarios-all.yaml` fans out typed scenario dry-runs
-  from the typed registry matrix.
-- Existing workflows such as `nightly-e2e.yaml`, `e2e-branch-validation.yaml`,
-  `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and
-  `regression-e2e.yaml` still run legacy live E2E scripts during the migration.
-- `vitest.config.ts` contains the `e2e-scenario-framework` project for framework
-  and metadata tests.
+The CI entry point is `.github/workflows/e2e-scenarios.yaml` (manual dispatch). Existing legacy workflows (`nightly-e2e.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, etc.) remain in place during the migration.
 
-## Migration tracking
+Migration status is tracked outside the repository in GitHub issues and pull requests, not in repo-local checklists. The parent architecture issue is #3588. Do not add a workflow-level parity report or assertion-ledger gate; use focused code review, framework tests, and the scenario coverage report to decide what to migrate next.
 
-Migration status is tracked outside the repository in GitHub issues and PRs,
-not in repo-local checklists. The parent architecture issue is #3588. Active
-audit-coverage work is tracked by the #4347–#4357 issue set, with focused
-follow-ups such as #4378 for specific drift fixes.
+## How to add a scenario, state, or suite
 
-The old workflow-level parity report has been removed. Use scenario framework
-tests, the coverage report, PR review, and the audit issues to decide what to
-migrate next.
+Add-a-scenario, add-a-state, and add-a-suite are short edits to the
+three YAML files above, plus shell scripts under
+`nemoclaw_scenarios/install/`, `nemoclaw_scenarios/onboard/`,
+`validation_suites/assert/`, or `validation_suites/<category>/`. The
+typed contracts in
+[`../scenarios/types.ts`](../scenarios/types.ts) and
+[`../scenarios/expected-states.ts`](../scenarios/expected-states.ts)
+describe the required shape; `npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id> --plan-only`
+validates your change without running anything destructive.
 
 When adding a suite assertion, emit or preserve a stable `PASS: <id>` /
-`FAIL: <id>` log line, and record migration evidence or follow-up state in the
-owning issue or PR. Sandbox lifecycle assertions should use
-`validation_suites/lib/sandbox_lifecycle.sh`, consume
-`$E2E_CONTEXT_DIR/context.env`, and keep destructive snapshot restore checks
-isolated in the opt-in `snapshot-lifecycle` suite. Platform-specific scenarios
-such as GPU, macOS, WSL, Brev, or DGX Spark must also list
-`runner_requirements` in `scenarios.yaml`.
+`FAIL: <id>` log line, and update migration coverage through the scenario coverage report and the domain issues under `#3588`. Sandbox lifecycle assertions should use `validation_suites/lib/sandbox_lifecycle.sh`, consume `$E2E_CONTEXT_DIR/context.env`, and keep destructive snapshot restore checks isolated in the opt-in `snapshot-lifecycle` suite. Platform-specific scenarios such as GPU, macOS, WSL, Brev, or DGX Spark must also list `runner_requirements` in `scenarios.yaml`.
 
 Prefer new scenario-matrix coverage over new legacy-style `test-*.sh` scripts.

test/e2e-scenario/framework-tests/e2e-context-helper.test.ts

@@ -9,7 +9,6 @@ import path from "node:path";
 
 const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
 const CONTEXT_LIB = path.join(REPO_ROOT, "test/e2e-scenario/runtime/lib/context.sh");
-const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e-scenario/runtime/run-scenario.sh");
 
 function runBash(script: string, env: Record<string, string> = {}): SpawnSyncReturns<string> {
   return spawnSync("bash", ["-c", script], {
@@ -86,38 +85,4 @@ describe("E2E context helper (runtime/lib/context.sh)", () => {
     }
   });
 
-  it("scenario_plan_execution_should_emit_context_under_dry_run", () => {
-    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ctx-"));
-    try {
-      const r = spawnSync(
-        "bash",
-        [RUN_SCENARIO, "ubuntu-repo-cloud-openclaw", "--dry-run"],
-        {
-          env: { ...process.env, E2E_CONTEXT_DIR: tmp },
-          encoding: "utf8",
-    timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000),
-          cwd: REPO_ROOT,
-        },
-      );
-      expect(r.status, r.stderr).toBe(0);
-      const ctxPath = path.join(tmp, "context.env");
-      expect(fs.existsSync(ctxPath), `context.env missing in ${tmp}`).toBe(true);
-      const ctx = fs.readFileSync(ctxPath, "utf8");
-      for (const key of [
-        "E2E_SCENARIO",
-        "E2E_PLATFORM_OS",
-        "E2E_INSTALL_METHOD",
-        "E2E_ONBOARDING_PATH",
-        "E2E_AGENT",
-        "E2E_PROVIDER",
-        "E2E_SANDBOX_NAME",
-        "E2E_GATEWAY_URL",
-        "E2E_INFERENCE_ROUTE",
-      ]) {
-        expect(ctx, `${key} missing from context.env`).toMatch(new RegExp(`^${key}=`, "m"));
-      }
-    } finally {
-      fs.rmSync(tmp, { recursive: true, force: true });
-    }
-  });
 });