Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
b7acfb7
test(e2e): execute real shell assertions; delete dry-run, --validate-…
jyaunches May 28, 2026
2716ab0
Merge remote-tracking branch 'origin/main' into feat/e2e-real-execution
jyaunches May 28, 2026
a5aabd7
fix(e2e): flush log writeStream before evidence read; replace source-…
jyaunches May 28, 2026
903f90b
fix(e2e): drop extra trailing newline (end-of-file-fixer)
jyaunches May 28, 2026
628870c
feat(e2e): wire install/onboard into typed phase orchestrators
jyaunches May 28, 2026
f81cf5a
fix(e2e): emit context.env in TS framework + cross-phase short-circuit
jyaunches May 28, 2026
2dc9439
fix(e2e): stop dispatcher from truncating context.env; alias onboard.log
jyaunches May 28, 2026
0aa107a
fix(e2e): tighten preflight-passed regex to actual failure markers
jyaunches May 28, 2026
1d128a1
style(e2e): shfmt indentation on preflight-passed continuation lines
jyaunches May 28, 2026
c359f8a
fix(e2e): address CodeRabbit batch (3 quick wins)
jyaunches May 28, 2026
f923597
Merge origin/main into feat/e2e-real-execution
jyaunches May 28, 2026
34f3ca2
fix(e2e): remove residual E2E_DRY_RUN branch and phase-1-skeleton refs
jyaunches May 28, 2026
083b9b6
test(e2e): wrap openshell sandbox exec with per-call timeout + diagno…
jyaunches May 28, 2026
14ab98f
ci(e2e): silence SC2034 on prefix-env sandbox exec timeout overrides
jyaunches May 28, 2026
c167d35
feat(e2e): framework-owned secret hygiene at the spawn boundary
jyaunches May 28, 2026
cc6b7a2
fix(e2e): fail-closed for required security probes and expected-failu…
jyaunches May 28, 2026
5bb3bc7
test(e2e): prefer openshell sandbox ssh-config + ssh -F transport
jyaunches May 28, 2026
10230d9
test(e2e): route rebuild_upgrade.sh sandbox exec through canonical wr…
jyaunches May 28, 2026
72185d7
test(e2e): align status_fields_present with real nemoclaw status output
jyaunches May 28, 2026
0d531d9
test(e2e): wire negative preflight scenario end-to-end
jyaunches May 28, 2026
3198d37
feat(e2e): typed negative-scenario contract matcher (PR #4380 advisor…
jyaunches May 28, 2026
2008d0d
test(e2e): switch policy_presets_preserved to host-side openshell pol…
jyaunches May 28, 2026
6ecd198
test(e2e): land typed probe registry with diagnostics + docs validation
jyaunches May 28, 2026
ff4b0a8
feat(e2e): typed state-validation phase gates runtime suites (PR #438…
jyaunches May 28, 2026
f61bcd2
test(e2e): land security probes (shields, network policy, injection)
jyaunches May 28, 2026
9da75ac
chore(e2e): retire legacy expected-states.yaml + runtime/resolver pat…
jyaunches May 28, 2026
46e6dde
test(e2e): map inference route slug to inference.local DNS host
jyaunches May 28, 2026
9932bbb
test(e2e): route sandbox_lifecycle.sh sandbox-exec calls through cano…
jyaunches May 28, 2026
9afeedc
test(e2e): land lifecycle phase + ubuntu-rebuild-openclaw scenario
jyaunches May 29, 2026
f899439
fix(e2e): ensure ~/.local/bin on PATH and correct snapshot argv
jyaunches Jun 8, 2026
354dfa2
merge: bring main into feat/e2e-real-execution
jyaunches Jun 8, 2026
863e8eb
merge: fold in #4939 single-runner doc + reconcile retired YAML resolver
jyaunches Jun 8, 2026
3d8adfa
fix(e2e): post-merge CI fixes - probe shebangs + retired-link cleanup
jyaunches Jun 8, 2026
381d654
fix(e2e): remove last 2 stale dry-run references the PR audit missed
jyaunches Jun 8, 2026
fc06168
style(e2e): apply shfmt + executable bits to satisfy static-checks
jyaunches Jun 8, 2026
c8ae688
Merge remote-tracking branch 'origin/main' into feat/e2e-real-execution
jyaunches Jun 8, 2026
b41c17b
test(e2e): refactor redaction parity to behavior-based to clear sourc…
jyaunches Jun 8, 2026
bd4d3ce
fix(e2e): address CodeRabbit major and CodeQL alert 715 on PR review
jyaunches Jun 8, 2026
415c577
Merge remote-tracking branch 'origin/main' into feat/e2e-real-execution
jyaunches Jun 8, 2026
7887adb
fix(e2e): suppress CodeQL alert 792 with explicit safety contract
jyaunches Jun 8, 2026
8c6261d
Merge remote-tracking branch 'origin/main' into feat/e2e-real-execution
jyaunches Jun 8, 2026
70df051
fix(e2e): place lgtm[] directive on the line immediately preceding sp…
jyaunches Jun 8, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions .github/workflows/e2e-scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ jobs:
for raw in "${IDS[@]}"; do
id="${raw//[[:space:]]/}"
[ -n "${id}" ] || continue
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${id}" --plan-only >/dev/null
runner="${ROUTES[$id]:-}"
if [ -z "${runner}" ]; then
echo "::error::No runner route for scenario: ${id}" >&2
Expand Down Expand Up @@ -135,7 +134,7 @@ jobs:
echo "::error::Invalid scenario input: ${SCENARIOS}" >&2
exit 1
fi
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}" --dry-run
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}"

- name: Resolve workspace paths for WSL
if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw')
Expand Down Expand Up @@ -299,7 +298,7 @@ jobs:
export E2E_CONTEXT_DIR="`$workdir"
npm ci --ignore-scripts
set +e
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "`$scenarios" --dry-run
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "`$scenarios"
status=`$?
if [ -d "`$workdir/.e2e" ]; then
rm -rf "`$checkout_dir/.e2e"
Expand Down
32 changes: 17 additions & 15 deletions test/e2e-scenario/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,25 @@ test plan, expected state, and post-onboard suites. Test plans can also declare
onboarding assertions that run after install/onboard and before expected-state
validation.

Plan-only resolution accepts either an alias or a test plan ID:

```bash
bash test/e2e-scenario/runtime/run-scenario.sh ubuntu-repo-cloud-openclaw --plan-only
bash test/e2e-scenario/runtime/run-scenario.sh ubuntu-repo-docker__cloud-nvidia-openclaw --plan-only
```

## How to run

The TypeScript runner is the only supported entrypoint. There is one
execution mode: live. There is no `--dry-run`, no `--validate-only`, no
fake-pass code path. Plan output is emitted as a side effect of the
live run.

```bash
bash test/e2e-scenario/runtime/run-scenario.sh <id> --plan-only # resolve + print plan, no side effects
bash test/e2e-scenario/runtime/run-scenario.sh <id> --dry-run # helpers short-circuit with trace
bash test/e2e-scenario/runtime/run-scenario.sh <id> --validate-only # assume setup done; validate expected state
bash test/e2e-scenario/runtime/run-scenario.sh <id> # full live run
bash test/e2e-scenario/runtime/run-suites.sh <suite-id> [<suite-id>…]
bash test/e2e-scenario/runtime/coverage-report.sh # Markdown matrix of scenario × suite
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id[,id...]> # live execution (the only mode)
npx tsx test/e2e-scenario/scenarios/run.ts --list # list canonical scenario ids
npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix # JSON registry payload for CI matrix fan-out
npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id> --plan-only # local debug only; MUST NOT appear in any workflow
bash test/e2e-scenario/runtime/coverage-report.sh # Markdown matrix of scenario × suite
```

The deprecated bash entrypoints `runtime/run-scenario.sh` and
`runtime/run-suites.sh` exist only as fail-fast stubs; they print a
pointer at `run.ts` and exit non-zero.

Override the runtime context dir with `E2E_CONTEXT_DIR=<path>` (default
`.e2e/`, gitignored). The scenario runner and suites communicate only
through `$E2E_CONTEXT_DIR/context.env` — suites do not rediscover
Expand All @@ -72,7 +73,8 @@ test/e2e/
assert/ # outcome assertions (inference, credentials, policy, messaging)
smoke/ inference/ hermes/ platform/ security/ # suite scripts grouped by concern
runtime/ # entry points + cross-cutting shared libs
run-scenario.sh / run-suites.sh / coverage-report.sh
run-scenario.sh / run-suites.sh # DEPRECATED fail-fast stubs (see above)
coverage-report.sh
resolver/ # TypeScript: load, plan, validate, coverage (invoked via tsx)
lib/ # shared shell helpers: context, env, cleanup, logging, artifacts, sandbox-teardown
```
Expand All @@ -89,7 +91,7 @@ three YAML files above, plus shell scripts under
`validation_suites/assert/`, or `validation_suites/<category>/`. The
schemas in
[`../runtime/resolver/schema.ts`](../runtime/resolver/schema.ts)
describe the required shape; `run-scenario.sh <id> --plan-only`
describe the required shape; `npx tsx test/e2e-scenario/scenarios/run.ts --scenarios <id> --plan-only`
validates your change without running anything destructive.

When adding a suite assertion, emit or preserve a stable `PASS: <id>` /
Expand Down
35 changes: 0 additions & 35 deletions test/e2e-scenario/framework-tests/e2e-context-helper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import path from "node:path";

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
const CONTEXT_LIB = path.join(REPO_ROOT, "test/e2e-scenario/runtime/lib/context.sh");
const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e-scenario/runtime/run-scenario.sh");

function runBash(script: string, env: Record<string, string> = {}): SpawnSyncReturns<string> {
return spawnSync("bash", ["-c", script], {
Expand Down Expand Up @@ -86,38 +85,4 @@ describe("E2E context helper (runtime/lib/context.sh)", () => {
}
});

it("scenario_plan_execution_should_emit_context_under_dry_run", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-ctx-"));
try {
const r = spawnSync(
"bash",
[RUN_SCENARIO, "ubuntu-repo-cloud-openclaw", "--dry-run"],
{
env: { ...process.env, E2E_CONTEXT_DIR: tmp },
encoding: "utf8",
timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000),
cwd: REPO_ROOT,
},
);
expect(r.status, r.stderr).toBe(0);
const ctxPath = path.join(tmp, "context.env");
expect(fs.existsSync(ctxPath), `context.env missing in ${tmp}`).toBe(true);
const ctx = fs.readFileSync(ctxPath, "utf8");
for (const key of [
"E2E_SCENARIO",
"E2E_PLATFORM_OS",
"E2E_INSTALL_METHOD",
"E2E_ONBOARDING_PATH",
"E2E_AGENT",
"E2E_PROVIDER",
"E2E_SANDBOX_NAME",
"E2E_GATEWAY_URL",
"E2E_INFERENCE_ROUTE",
]) {
expect(ctx, `${key} missing from context.env`).toMatch(new RegExp(`^${key}=`, "m"));
}
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});
});
235 changes: 0 additions & 235 deletions test/e2e-scenario/framework-tests/e2e-expected-state-validator.test.ts

This file was deleted.

Loading
Loading