diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 07bb8c54f54..360cbb19fbc 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -11,6 +11,11 @@ on: required: false default: "" type: string + pr_number: + description: Optional PR number for selective-dispatch result comments. + required: false + type: string + default: "" permissions: contents: read @@ -322,3 +327,184 @@ jobs: include-hidden-files: false if-no-files-found: ignore retention-days: 14 + + # ── Free-standing recovery scenarios (#2701) ───────────────────────── + # Recovery / disruption scenarios don't fit the steady-state expected-state + # registry that drives `live-scenarios` above. They run as free-standing + # Vitest test files using the same `e2e-scenarios-live` project, framework + # fixtures, and live-project gate — just outside the matrix. + # + # First failing-test-first guard for #2701 (gateway recovery does not + # restore the /tmp guard chain after pod recreate). Will fail on `main` + # until the #2701 fix lands; flips green afterwards. + gateway-guard-recovery: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/gateway-guard-recovery + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + # nemoclaw onboard registers the gateway under the canonical name + # "nemoclaw" (src/lib/actions/sandbox/connect.ts:NEMOCLAW_GATEWAY_NAME) + # but does not call `openshell gateway select` to mark it active. The + # SandboxClient and recovery probes invoke `openshell sandbox exec` + # directly, which fails with "No active gateway" when no active + # gateway is configured. Setting OPENSHELL_GATEWAY here tells openshell + # to use the named gateway for every invocation (per `openshell` -h + # GATEWAY FLAGS: env: OPENSHELL_GATEWAY=). + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + # Onboard expects openshell to already be on PATH — install.sh handles + # this for the legacy bash E2E suite (which runs `bash install.sh + # --non-interactive` end-to-end). The Vitest fixture path skips + # install.sh and invokes `bin/nemoclaw.js onboard` directly, so we + # need to run the standalone openshell installer here. Mirrors what + # `maybe_install_openshell_during_install` does in install.sh. + run: bash scripts/install-openshell.sh + + - name: Run Vitest gateway-guard-recovery scenario + run: | + set -euo pipefail + # OpenShell installs to /usr/local/bin on GitHub-hosted runners + # (writable by the runner user, no sudo) or to ~/.local/bin in + # NEMOCLAW_NON_INTERACTIVE mode when /usr/local/bin is not writable. + # See scripts/install-openshell.sh:394-425. Cover both paths. + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + # Resolve the actual install path so the framework's SandboxClient + # can spawn it without relying on PATH inheritance from the test + # process (the framework also accepts OPENSHELL_BIN as an override). + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" + "$OPENSHELL_BIN" --version + npx vitest run \ + --project e2e-scenarios-live \ + test/e2e-scenario/live/gateway-guard-recovery.test.ts \ + --reporter=default --silent=false + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-gateway-guard-recovery + path: e2e-artifacts/vitest/gateway-guard-recovery/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + # ── PR result comment (mirrors nightly-e2e.yaml's report-to-pr) ─────────── + # Posts a results table on the open PR for the dispatching branch (or the + # PR identified by `inputs.pr_number`). `if: always()` so the comment lands + # even when scenario jobs failed — that's the whole point of a result + # comment. Same shape as nightly-e2e.yaml:report-to-pr so reviewers see + # consistent comment formatting across both suites. + report-to-pr: + runs-on: ubuntu-latest + needs: + [ + generate-matrix, + live-scenarios, + openshell-version-pin-vitest, + onboard-negative-paths-vitest, + openclaw-tui-chat-correlation-vitest, + gateway-guard-recovery, + ] + if: ${{ always() && github.event_name == 'workflow_dispatch' }} + permissions: + issues: write + pull-requests: write + steps: + - name: Post Vitest scenario results to PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const needs = ${{ toJSON(needs) }}; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const workflowBranch = context.ref.replace('refs/heads/', ''); + const prNumberInput = ${{ toJSON(inputs.pr_number) }} || ''; + const requestedScenarios = ${{ toJSON(inputs.scenarios) }} || ''; + + let prNumber = prNumberInput ? Number.parseInt(prNumberInput, 10) : undefined; + if (!prNumber) { + const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${workflowBranch}`, + state: 'open', + }); + if (prs.length === 0) { + core.info(`No open PR found for branch ${workflowBranch} — skipping comment.`); + return; + } + prNumber = prs[0].number; + } + + const emoji = { success: '✅', failure: '❌', cancelled: '⚠️', skipped: '⏭️' }; + const entries = Object.entries(needs).sort(([a], [b]) => a.localeCompare(b)); + const rows = entries.map( + ([name, { result }]) => `| ${name} | ${emoji[result] || '❓'} ${result} |`, + ); + const ran = entries.filter(([, v]) => v.result !== 'skipped'); + const passed = ran.filter(([, v]) => v.result === 'success'); + const failed = ran.filter(([, v]) => v.result === 'failure'); + const skipped = entries.filter(([, v]) => v.result === 'skipped'); + const status = + failed.length > 0 + ? '❌ Some jobs failed' + : skipped.length > 0 && passed.length === 0 + ? '⚠️ No jobs ran' + : '✅ All jobs passed'; + + const lines = [ + `### Vitest E2E Scenario Results — ${status}`, + '', + `**Run:** [${context.runId}](${runUrl})`, + `**Workflow ref:** \`${workflowBranch}\``, + requestedScenarios + ? `**Requested scenarios:** \`${requestedScenarios}\`` + : '**Requested scenarios:** _(default — all supported)_', + `**Summary:** ${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped`, + '', + '| Job | Result |', + '|-----|--------|', + ...rows, + ]; + if (failed.length > 0) { + const failedNames = failed.map(([name]) => name).join(', '); + lines.push('', `> **Failed jobs:** ${failedNames}. Check [run artifacts](${runUrl}) for logs.`); + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: lines.join('\n'), + }); diff --git a/test/e2e-scenario/fixtures/clients/gateway.ts b/test/e2e-scenario/fixtures/clients/gateway.ts index 6ee756fb735..c82fcd5ae5d 100644 --- a/test/e2e-scenario/fixtures/clients/gateway.ts +++ b/test/e2e-scenario/fixtures/clients/gateway.ts @@ -1,15 +1,72 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import type { NemoClawInstance } from "../phases/onboarding.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { assertExitZero } from "./command.ts"; import type { HostCliClient } from "./host.ts"; +import type { SandboxClient } from "./sandbox.ts"; + +/** + * Build the env passed to in-sandbox probes via `openshell sandbox exec`. + * + * The framework's ShellProbe defaults to `inheritEnv: false` and routes the + * spawned-process env through `buildChildEnv`'s allowlist (HOME, PATH, …). + * `OPENSHELL_GATEWAY` is not in that allowlist, so even when the workflow + * sets it, raw `openshell sandbox exec` invocations fail with + * "× No active gateway" because the openshell binary cannot resolve which + * gateway to talk to. Inject the gateway name read from the test process's + * env (defaulting to the canonical `nemoclaw` registered by + * src/lib/actions/sandbox/connect.ts:NEMOCLAW_GATEWAY_NAME) on top of the + * framework's allowlisted env. + */ +function probeEnv(): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; +} + +/** + * Default expected exports inside `/tmp/nemoclaw-proxy-env.sh` that prove the + * NODE_OPTIONS preload chain is wired. The legacy 2478 test verified guards + * by reading this file rather than `/proc//environ` because + * `kernel.yama.ptrace_scope=1` blocks cross-tree environ reads. We mirror + * that approach here for the same reason. + */ +const DEFAULT_GUARD_MARKERS: ReadonlyArray = [ + "nemoclaw-sandbox-safety-net", + "nemoclaw-ciao-network-guard", +]; + +/** Default gateway log path inside the sandbox. */ +const GATEWAY_LOG_PATH = "/tmp/gateway.log"; + +export interface ExpectGuardChainOptions extends ShellProbeRunOptions { + /** Markers required in `/tmp/nemoclaw-proxy-env.sh`. Defaults to safety-net + ciao. */ + expectedMarkers?: ReadonlyArray; +} + +export interface ExpectLogOptions extends ShellProbeRunOptions { + /** Number of trailing log lines to inspect. Defaults to 200. */ + lines?: number; +} + +export interface ExpectPidStableOptions extends ShellProbeRunOptions { + /** Total observation window in seconds. */ + durationSeconds: number; + /** Polling interval in seconds. Defaults to 3. */ + pollIntervalSeconds?: number; +} export class GatewayClient { private readonly host: HostCliClient; + private readonly sandbox: SandboxClient; - constructor(host: HostCliClient) { + constructor(host: HostCliClient, sandbox: SandboxClient) { this.host = host; + this.sandbox = sandbox; } status(options: ShellProbeRunOptions = {}): Promise { @@ -24,4 +81,181 @@ export class GatewayClient { assertExitZero(result, "nemoclaw gateway status"); return result; } + + // ─── Guard-chain recovery probes (#2478, #2701) ──────────────────── + + /** + * Resolve the running openclaw gateway PID inside the sandbox by parsing + * `ps`. Returns the lowest matching PID, or null if no gateway process is + * running. Mirrors the legacy bash `gateway_pid()` helper. + * + * Two-pass match: first prefer rows whose argv contains "gateway" alongside + * comm "openclaw"; fall back to any "openclaw" comm. The two-pass shape + * tolerates older builds that exposed gateway under a slightly different + * argv but the same comm. + */ + async resolveGatewayPid(instance: NemoClawInstance): Promise { + const script = + "set -e; " + + // Primary: argv contains "gateway" and comm is "openclaw". + 'pid="$(ps -eo pid=,comm=,args= 2>/dev/null | ' + + "awk '($2 == \"openclaw\" && $0 ~ /gateway/) || $0 ~ /openclaw[ -]gateway/ { print $1 }' | " + + 'sort -n | head -n 1)"; ' + + // Fallback: any process with comm "openclaw". + 'if [ -z "$pid" ]; then ' + + 'pid="$(ps -eo pid=,comm=,args= 2>/dev/null | ' + + 'awk \'$2 == "openclaw" { print $1 }\' | sort -n | head -n 1)"; ' + + "fi; " + + 'printf "%s\\n" "$pid"'; + + const result = await this.sandbox.exec(instance.sandboxName, ["sh", "-c", script], { + artifactName: `gateway-resolve-pid-${instance.sandboxName}`, + env: probeEnv(), + }); + const trimmed = result.stdout.trim(); + if (!/^[0-9]+$/.test(trimmed)) return null; + const pid = Number(trimmed); + return Number.isSafeInteger(pid) && pid > 0 ? pid : null; + } + + /** + * Assert that the NODE_OPTIONS guard chain is active for the gateway by + * reading `/tmp/nemoclaw-proxy-env.sh` and verifying it contains the + * expected preload markers (`--require` paths). The proxy-env file is + * the single source of truth — when recovery sources it, the gateway + * inherits the chain. + * + * We deliberately read the file rather than `/proc//environ`: + * `kernel.yama.ptrace_scope=1` blocks reads of /proc/.../environ across + * non-ancestor process trees. This matches the legacy 2478 bash test's + * approach (`gateway_guards_active` -> `proxy_env_contents`). + * + * @throws if the file is missing or any expected marker is absent. + */ + async expectGuardChainActive( + instance: NemoClawInstance, + options: ExpectGuardChainOptions = {}, + ): Promise { + const expected = options.expectedMarkers ?? DEFAULT_GUARD_MARKERS; + const result = await this.sandbox.exec( + instance.sandboxName, + ["sh", "-c", "cat /tmp/nemoclaw-proxy-env.sh 2>/dev/null"], + { + artifactName: `gateway-guard-chain-${instance.sandboxName}`, + env: probeEnv(), + ...options, + }, + ); + + if (result.exitCode !== 0 || result.stdout.trim() === "") { + throw new Error( + `expectGuardChainActive: /tmp/nemoclaw-proxy-env.sh missing or empty in ${instance.sandboxName}`, + ); + } + + const missing = expected.filter((marker) => !result.stdout.includes(marker)); + if (missing.length > 0) { + throw new Error( + `expectGuardChainActive: /tmp/nemoclaw-proxy-env.sh missing markers ${JSON.stringify(missing)} in ${instance.sandboxName}`, + ); + } + } + + /** + * Tail the gateway log inside the sandbox and assert the regex matches. + * Used to verify recovery emitted (or did not emit) specific markers like + * `[gateway-recovery] WARNING`. + */ + async expectLogContains( + instance: NemoClawInstance, + pattern: RegExp, + options: ExpectLogOptions = {}, + ): Promise { + const tail = await this.tailLog(instance, options); + if (!pattern.test(tail)) { + throw new Error( + `expectLogContains: ${GATEWAY_LOG_PATH} did not match ${pattern.source} in ${instance.sandboxName}`, + ); + } + } + + /** Inverse of {@link expectLogContains}. */ + async expectLogDoesNotContain( + instance: NemoClawInstance, + pattern: RegExp, + options: ExpectLogOptions = {}, + ): Promise { + const tail = await this.tailLog(instance, options); + if (pattern.test(tail)) { + throw new Error( + `expectLogDoesNotContain: ${GATEWAY_LOG_PATH} unexpectedly matched ${pattern.source} in ${instance.sandboxName}`, + ); + } + } + + /** + * Verify the gateway PID is stable over `durationSeconds`. A crash loop + * shows up as the PID changing every few seconds because the supervisor + * keeps respawning. We sample at `pollIntervalSeconds` and fail on first + * change (or on the gateway disappearing entirely). + */ + async expectPidStable( + instance: NemoClawInstance, + options: ExpectPidStableOptions, + ): Promise { + const pollIntervalSeconds = options.pollIntervalSeconds ?? 3; + if (options.durationSeconds <= 0) { + throw new Error("expectPidStable: durationSeconds must be > 0"); + } + if (pollIntervalSeconds <= 0) { + throw new Error("expectPidStable: pollIntervalSeconds must be > 0"); + } + + const initialPid = await this.resolveGatewayPid(instance); + if (initialPid === null) { + throw new Error( + `expectPidStable: no gateway process in ${instance.sandboxName} at start of observation window`, + ); + } + + const samples = Math.max(1, Math.floor(options.durationSeconds / pollIntervalSeconds)); + for (let i = 0; i < samples; i += 1) { + await sleepSeconds(pollIntervalSeconds); + const pid = await this.resolveGatewayPid(instance); + if (pid === null) { + throw new Error( + `expectPidStable: gateway disappeared in ${instance.sandboxName} after ${(i + 1) * pollIntervalSeconds}s`, + ); + } + if (pid !== initialPid) { + throw new Error( + `expectPidStable: gateway PID changed ${initialPid}→${pid} in ${instance.sandboxName} after ${(i + 1) * pollIntervalSeconds}s (crash-loop suspected)`, + ); + } + } + return initialPid; + } + + // ─── Internal helpers ────────────────────────────────────────────── + + private async tailLog(instance: NemoClawInstance, options: ExpectLogOptions): Promise { + const lines = options.lines ?? 200; + if (!Number.isInteger(lines) || lines <= 0) { + throw new Error("tailLog: lines must be a positive integer"); + } + const result = await this.sandbox.exec( + instance.sandboxName, + ["sh", "-c", `tail -n ${lines} ${GATEWAY_LOG_PATH} 2>/dev/null`], + { + artifactName: `gateway-log-tail-${instance.sandboxName}`, + env: probeEnv(), + ...options, + }, + ); + return result.stdout; + } +} + +function sleepSeconds(seconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, seconds * 1000)); } diff --git a/test/e2e-scenario/fixtures/clients/sandbox.ts b/test/e2e-scenario/fixtures/clients/sandbox.ts index 7686ce729c1..31e3e4662d9 100644 --- a/test/e2e-scenario/fixtures/clients/sandbox.ts +++ b/test/e2e-scenario/fixtures/clients/sandbox.ts @@ -1,10 +1,27 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { trustedShellCommand } from "../shell-probe.ts"; import { artifactLabel, assertExitZero, type CommandRunner } from "./command.ts"; +/** + * Default env for openshell-targeted spawns. ShellProbe filters env via + * the framework allowlist (HOME, PATH, …) which excludes OPENSHELL_GATEWAY, + * so raw `openshell sandbox exec` invocations would fail with + * "× No active gateway" even when the workflow sets the env var. Inject + * it explicitly from the test process's env (defaulting to the canonical + * `nemoclaw` gateway registered by + * src/lib/actions/sandbox/connect.ts:NEMOCLAW_GATEWAY_NAME). + */ +function openshellProbeEnv(): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; +} + export interface SandboxClientOptions { openshellPath?: string; } @@ -51,7 +68,7 @@ export class SandboxClient { status(name: string, options: ShellProbeRunOptions = {}): Promise { validateSandboxName(name); - return this.openshell(["sandbox", "status", name], { + return this.openshell(["sandbox", "status", "--name", name], { artifactName: `sandbox-status-${name}`, ...options, }); @@ -101,6 +118,72 @@ export class SandboxClient { assertExitZero(result, `openshell sandbox status ${name}`); return result; } + + /** + * Disruption helper: simulate the post-pod-recreate /tmp wipe by removing + * the guard chain files. After this, a sandbox containing a running gateway + * is in the same shape as a fresh container that would only see /tmp + * recreated empty by the OpenShell sandbox controller. + * + * Used exclusively by recovery E2E scenarios (#2701). Removes: + * - /tmp/nemoclaw-proxy-env.sh (the NODE_OPTIONS chain export file) + * - the seven --require preload guard scripts written by the entrypoint + */ + async wipeGuardChain( + name: string, + options: ShellProbeRunOptions = {}, + ): Promise { + validateSandboxName(name); + const removeCommand = [ + "rm", + "-f", + "/tmp/nemoclaw-proxy-env.sh", + "/tmp/nemoclaw-sandbox-safety-net.js", + "/tmp/nemoclaw-ciao-network-guard.js", + "/tmp/nemoclaw-slack-channel-guard.js", + "/tmp/nemoclaw-http-proxy-fix.js", + "/tmp/nemoclaw-ws-proxy-fix.js", + "/tmp/nemoclaw-nemotron-inference-fix.js", + "/tmp/nemoclaw-seccomp-guard.js", + ]; + const result = await this.exec(name, removeCommand, { + artifactName: `sandbox-wipe-guard-chain-${name}`, + env: openshellProbeEnv(), + ...options, + }); + assertExitZero(result, `wipe guard chain in ${name}`); + return result; + } + + /** + * Disruption helper: kill the entire openclaw process tree inside the + * sandbox (gateway + launcher + supervisor watchdog). Used after + * `wipeGuardChain` to force the recovery path to relaunch from scratch. + * + * The bracket pattern `[o]penclaw` is the standard pgrep/pkill trick to + * avoid matching the matcher process itself. + * + * Used exclusively by recovery E2E scenarios (#2701). + */ + async killGatewayTree( + name: string, + options: ShellProbeRunOptions = {}, + ): Promise { + validateSandboxName(name); + // Two-phase kill: SIGKILL the tree, sleep, then verify nothing came back. + // Mirrors the bash test's pkill -9 + verify pattern. + const script = + "pkill -9 -f '[o]penclaw' 2>/dev/null || true; " + + "sleep 2; " + + "pgrep -af '[o]penclaw' >/dev/null 2>&1 && exit 1 || exit 0"; + const result = await this.exec(name, ["sh", "-c", script], { + artifactName: `sandbox-kill-gateway-tree-${name}`, + env: openshellProbeEnv(), + ...options, + }); + assertExitZero(result, `kill gateway tree in ${name}`); + return result; + } } export function validateSandboxName(name: string): void { diff --git a/test/e2e-scenario/fixtures/e2e-test.ts b/test/e2e-scenario/fixtures/e2e-test.ts index 7c3b3ba97b4..211aeae8726 100644 --- a/test/e2e-scenario/fixtures/e2e-test.ts +++ b/test/e2e-scenario/fixtures/e2e-test.ts @@ -77,12 +77,15 @@ export const test = base.extend({ host: async ({ shellProbe }, use) => { await use(new HostCliClient(shellProbe)); }, - gateway: async ({ host }, use) => { - await use(new GatewayClient(host)); - }, sandbox: async ({ shellProbe }, use) => { await use(new SandboxClient(shellProbe)); }, + gateway: async ({ host, sandbox }, use) => { + // GatewayClient depends on `sandbox` for in-sandbox probes + // (guard-chain inspection, log tailing, gateway-PID polling). + // The fixture chain is sandbox → gateway so the dependency stays acyclic. + await use(new GatewayClient(host, sandbox)); + }, provider: async ({ shellProbe }, use) => { await use(new ProviderClient(shellProbe)); }, diff --git a/test/e2e-scenario/live/gateway-guard-recovery.test.ts b/test/e2e-scenario/live/gateway-guard-recovery.test.ts new file mode 100644 index 00000000000..b8a56a83b17 --- /dev/null +++ b/test/e2e-scenario/live/gateway-guard-recovery.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live E2E: gateway guard-chain recovery after pod-recreate /tmp wipe. + * + * Failing-test-first regression guard for NVIDIA/NemoClaw#2701. On `main` at + * the time this test landed, `buildOpenClawRecoveryScript()` takes a + * "warn-and-proceed" branch when `/tmp/nemoclaw-proxy-env.sh` is missing — + * it logs `[gateway-recovery] WARNING` and launches the gateway naked. On + * aarch64 / DGX Spark this triggers an infinite crash loop in + * `@homebridge/ciao` (`os.networkInterfaces()` throws because the OpenShell + * netns blocks the syscall). The only manual recovery is a 5-min + * `nemoclaw rebuild --yes`. + * + * This test asserts the desired contract — recovery RESTORES the guard + * chain before launching, no WARNING line, gateway PID stable. It will fail + * on `main` (proving the bug), pass once the fix lands. + * + * The contract is platform-independent: we don't need aarch64 to assert + * "guards are present after recovery." The aarch64 ciao crash is a + * downstream consequence of the same broken contract. + * + * The corresponding legacy bash phase remains in + * `test/e2e/test-issue-2478-crash-loop-recovery.sh` Phase 4 with both the + * #2478 WARNING assertion (current contract) and the new #2701 guard-chain + * assertion (failing today, green after fix). + */ + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { ubuntuRepoDocker } from "../scenarios/matrix.ts"; + +// Reuses the standard ubuntu-repo-docker environment with the +// `cloud-openclaw` onboarding profile (the only one the framework's +// OnboardingPhaseFixture currently supports per +// `test/e2e-scenario/scenarios/runtime-support.ts:SUPPORTED_ONBOARDING`). +// We don't route through the typed scenario registry because the registry +// is keyed on steady-state expected-state probes (cli-installed, +// gateway-healthy, ...); recovery scenarios are behavioral and don't fit +// that mold. +const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw"); + +const SANDBOX_NAME = "e2e-2701"; + +test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701)", async ({ + artifacts, + environment, + onboard, + host, + gateway, + sandbox, + secrets, + cleanup, +}) => { + secrets.required("NVIDIA_API_KEY"); + + await artifacts.writeJson("scenario.json", { + id: "gateway-guard-recovery", + runner: "vitest", + boundary: "sandbox-lifecycle", + issues: ["#2701", "#2478"], + }); + + // ── Setup ──────────────────────────────────────────────────────── + const ready = await environment.assertReady(ENVIRONMENT); + const instance = await onboard.from(ready, { sandboxName: SANDBOX_NAME }); + + // Baseline: a freshly-onboarded sandbox must already have the guard + // chain wired. If this fails, the bug isn't #2701 — it's a regression of + // the entrypoint guard install path. + await gateway.expectGuardChainActive(instance); + + // ── Disrupt ────────────────────────────────────────────────────── + // Same shape as a fresh container after pod recreate: /tmp is empty of + // the guard chain, and the openclaw process tree is gone. + await sandbox.wipeGuardChain(instance.sandboxName); + await sandbox.killGatewayTree(instance.sandboxName); + + // ── Trigger recovery ───────────────────────────────────────────── + // `connect --probe-only` invokes checkAndRecoverSandboxProcesses(), + // which is the production code path that runs every time a user + // reconnects to a sandbox. This is the failure surface end-users hit + // after a host reboot on DGX Spark. + const recoveryResult = await host.nemoclaw([instance.sandboxName, "connect", "--probe-only"], { + artifactName: "nemoclaw-connect-probe-only", + // ShellProbe defaults to inheritEnv: false; without an explicit env + // the spawned `nemoclaw` (= `node bin/nemoclaw.js`) cannot find node + // on PATH and exits 127. Pass the framework's allowlisted env so PATH, + // HOME, and the OPENSHELL_GATEWAY override flow through. + env: { + ...buildAvailabilityProbeEnv(), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }, + timeoutMs: 90_000, + }); + cleanup.add(`recovery-result-${instance.sandboxName}`, async () => { + await artifacts.writeJson("recovery-result.json", { + exitCode: recoveryResult.exitCode, + }); + }); + + // ── Assert #2701 contract ──────────────────────────────────────── + // After recovery completes, the guard chain MUST be restored. Today + // this fails: recovery emits a WARNING but launches the gateway + // naked, leaving /tmp/nemoclaw-proxy-env.sh absent. After the fix + // lands, recovery re-emits the chain before launching. + await gateway.expectGuardChainActive(instance); + + // No WARNING line should appear in the gateway log — the fix turns + // the warn-and-proceed branch into a re-emit-and-continue branch. + await gateway.expectLogDoesNotContain(instance, /\[gateway-recovery\] WARNING/); + + // Gateway must be steady-state — no crash loop. This assertion is + // the "would have caught DGX Spark" check, even on x86 runners, + // because a naked gateway crash would also flake on x86 occasionally + // and a fix that restores the chain trivially holds the PID. + const stablePid = await gateway.expectPidStable(instance, { + durationSeconds: 30, + pollIntervalSeconds: 5, + }); + + expect(stablePid).toBeGreaterThan(0); +}); diff --git a/test/e2e-scenario/support-tests/e2e-clients.test.ts b/test/e2e-scenario/support-tests/e2e-clients.test.ts index ac26979fef1..c0b8043b47b 100644 --- a/test/e2e-scenario/support-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/support-tests/e2e-clients.test.ts @@ -104,7 +104,8 @@ describe("E2E fixture clients", () => { it("gateway client delegates through NemoClaw gateway status", async () => { const runner = new FakeRunner(); const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); - const gateway = new GatewayClient(host); + const sandbox = new SandboxClient(runner); + const gateway = new GatewayClient(host, sandbox); await gateway.expectHealthy(); @@ -118,7 +119,8 @@ describe("E2E fixture clients", () => { it("gateway client preserves caller-provided probe options", async () => { const runner = new FakeRunner(); const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); - const gateway = new GatewayClient(host); + const sandbox = new SandboxClient(runner); + const gateway = new GatewayClient(host, sandbox); await gateway.status({ artifactName: "custom-gateway-status", @@ -167,7 +169,7 @@ describe("E2E fixture clients", () => { expect(runner.calls[0]).toEqual({ command: "openshell", - args: ["sandbox", "status", "assistant"], + args: ["sandbox", "status", "--name", "assistant"], options: { artifactName: "custom-sandbox-status", env: { NEMOCLAW_TEST_VALUE: "1" }, diff --git a/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts index 3a5a45fbc84..6372ca48fb2 100644 --- a/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts @@ -123,10 +123,11 @@ function fixture( artifacts?: ArtifactSink, ): StateValidationPhaseFixture { const host = new HostCliClient(runner); + const sandbox = new SandboxClient(runner); return new StateValidationPhaseFixture( host, - new GatewayClient(host), - new SandboxClient(runner), + new GatewayClient(host, sandbox), + sandbox, io, artifacts, ); diff --git a/test/e2e-scenario/support-tests/e2e-recovery-helpers.test.ts b/test/e2e-scenario/support-tests/e2e-recovery-helpers.test.ts new file mode 100644 index 00000000000..8df05da4dd5 --- /dev/null +++ b/test/e2e-scenario/support-tests/e2e-recovery-helpers.test.ts @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { GatewayClient, HostCliClient, SandboxClient } from "../fixtures/clients/index.ts"; +import type { CommandRunner } from "../fixtures/clients/index.ts"; +import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts"; +import type { + ShellProbeResult, + ShellProbeRunOptions, + TrustedShellCommand, +} from "../fixtures/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +interface ScriptedReply { + stdout?: string; + stderr?: string; + exitCode?: number | null; +} + +/** + * Test runner that returns scripted replies in order. Each `run()` call + * advances through the queue; falls back to a benign success once exhausted. + * + * Designed for the recovery helpers because they issue multiple shell + * probes per assertion (e.g. `expectPidStable` polls N times) and the + * test needs to control each reply independently. + */ +class ScriptedRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private replies: ScriptedReply[] = []; + + queue(...replies: ScriptedReply[]): void { + this.replies.push(...replies); + } + + async run( + command: TrustedShellCommand, + options?: ShellProbeRunOptions, + ): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const reply = this.replies.shift() ?? {}; + return { + command: [command.command, ...command.args], + exitCode: reply.exitCode ?? 0, + signal: null, + timedOut: false, + stdout: reply.stdout ?? "", + stderr: reply.stderr ?? "", + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; + } +} + +function fakeInstance(sandboxName = "e2e-2701"): NemoClawInstance { + return { + onboarding: "openclaw-nvidia", + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + platformOs: "ubuntu", + gatewayUrl: "https://localhost:18789", + result: { + command: ["nemoclaw", "onboard"], + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + artifacts: { stdout: "", stderr: "", result: "" }, + }, + }; +} + +function buildGateway(runner: ScriptedRunner): GatewayClient { + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + const sandbox = new SandboxClient(runner); + return new GatewayClient(host, sandbox); +} + +describe("GatewayClient recovery helpers (#2701)", () => { + describe("expectGuardChainActive", () => { + it("passes when proxy-env.sh contains the default safety-net + ciao markers", async () => { + const runner = new ScriptedRunner(); + runner.queue({ + stdout: + 'export NODE_OPTIONS="--require /tmp/nemoclaw-sandbox-safety-net.js ' + + '--require /tmp/nemoclaw-ciao-network-guard.js"\n', + }); + const gateway = buildGateway(runner); + + await gateway.expectGuardChainActive(fakeInstance()); + + expect(runner.calls[0]?.args.slice(-1)[0]).toContain("cat /tmp/nemoclaw-proxy-env.sh"); + }); + + it("fails when proxy-env.sh is empty (post pod-recreate scenario)", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "" }); + const gateway = buildGateway(runner); + + await expect(gateway.expectGuardChainActive(fakeInstance())).rejects.toThrow( + /missing or empty/, + ); + }); + + it("fails when proxy-env.sh exists but a marker is absent", async () => { + const runner = new ScriptedRunner(); + runner.queue({ + stdout: 'export NODE_OPTIONS="--require /tmp/nemoclaw-sandbox-safety-net.js"\n', + }); + const gateway = buildGateway(runner); + + await expect(gateway.expectGuardChainActive(fakeInstance())).rejects.toThrow( + /missing markers.*nemoclaw-ciao-network-guard/, + ); + }); + + it("honors a caller-supplied marker list", async () => { + const runner = new ScriptedRunner(); + runner.queue({ + stdout: 'export NODE_OPTIONS="--require /tmp/nemoclaw-slack-channel-guard.js"\n', + }); + const gateway = buildGateway(runner); + + await gateway.expectGuardChainActive(fakeInstance(), { + expectedMarkers: ["nemoclaw-slack-channel-guard"], + }); + }); + }); + + describe("expectLogContains / expectLogDoesNotContain", () => { + it("expectLogContains passes when the tail matches", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "[gateway-recovery] WARNING: /tmp/nemoclaw-proxy-env.sh missing\n" }); + const gateway = buildGateway(runner); + + await gateway.expectLogContains(fakeInstance(), /\[gateway-recovery\] WARNING/); + }); + + it("expectLogContains fails when the tail does not match", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "boring log line\n" }); + const gateway = buildGateway(runner); + + await expect( + gateway.expectLogContains(fakeInstance(), /\[gateway-recovery\] WARNING/), + ).rejects.toThrow(/did not match/); + }); + + it("expectLogDoesNotContain passes when the tail is clean", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "openclaw started\n" }); + const gateway = buildGateway(runner); + + await gateway.expectLogDoesNotContain(fakeInstance(), /\[gateway-recovery\] WARNING/); + }); + + it("expectLogDoesNotContain fails when the forbidden marker appears", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "[gateway-recovery] WARNING\n" }); + const gateway = buildGateway(runner); + + await expect( + gateway.expectLogDoesNotContain(fakeInstance(), /\[gateway-recovery\] WARNING/), + ).rejects.toThrow(/unexpectedly matched/); + }); + + it("rejects non-positive line counts", async () => { + const runner = new ScriptedRunner(); + const gateway = buildGateway(runner); + + await expect(gateway.expectLogContains(fakeInstance(), /x/, { lines: 0 })).rejects.toThrow( + /positive integer/, + ); + }); + }); + + describe("resolveGatewayPid", () => { + it("returns the parsed PID when the script prints a number", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "1234\n" }); + const gateway = buildGateway(runner); + + await expect(gateway.resolveGatewayPid(fakeInstance())).resolves.toBe(1234); + }); + + it("returns null when the script prints non-numeric output", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "" }); + const gateway = buildGateway(runner); + + await expect(gateway.resolveGatewayPid(fakeInstance())).resolves.toBeNull(); + }); + }); + + describe("expectPidStable", () => { + it("returns the PID when it is stable across all samples", async () => { + const runner = new ScriptedRunner(); + // initial sample + 3 stable samples + runner.queue( + { stdout: "100\n" }, + { stdout: "100\n" }, + { stdout: "100\n" }, + { stdout: "100\n" }, + ); + const gateway = buildGateway(runner); + + const pid = await gateway.expectPidStable(fakeInstance(), { + durationSeconds: 3, + pollIntervalSeconds: 1, + }); + expect(pid).toBe(100); + }); + + it("throws when the PID changes (crash-loop)", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "100\n" }, { stdout: "201\n" }); + const gateway = buildGateway(runner); + + await expect( + gateway.expectPidStable(fakeInstance(), { + durationSeconds: 2, + pollIntervalSeconds: 1, + }), + ).rejects.toThrow(/PID changed 100→201.*crash-loop/); + }); + + it("throws when the gateway disappears mid-window", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "100\n" }, { stdout: "" }); + const gateway = buildGateway(runner); + + await expect( + gateway.expectPidStable(fakeInstance(), { + durationSeconds: 2, + pollIntervalSeconds: 1, + }), + ).rejects.toThrow(/gateway disappeared/); + }); + + it("throws when no gateway exists at the start of the window", async () => { + const runner = new ScriptedRunner(); + runner.queue({ stdout: "" }); + const gateway = buildGateway(runner); + + await expect( + gateway.expectPidStable(fakeInstance(), { + durationSeconds: 1, + pollIntervalSeconds: 1, + }), + ).rejects.toThrow(/no gateway process.*at start/); + }); + + it("rejects non-positive durations", async () => { + const runner = new ScriptedRunner(); + const gateway = buildGateway(runner); + + await expect( + gateway.expectPidStable(fakeInstance(), { + durationSeconds: 0, + pollIntervalSeconds: 1, + }), + ).rejects.toThrow(/durationSeconds must be > 0/); + }); + }); +}); + +describe("SandboxClient disruption helpers (#2701)", () => { + it("wipeGuardChain removes the seven guard files plus proxy-env.sh", async () => { + const runner = new ScriptedRunner(); + const sandbox = new SandboxClient(runner); + + await sandbox.wipeGuardChain("e2e-2701"); + + const call = runner.calls[0]; + expect(call?.args).toContain("--"); + const removeArgs = call?.args.slice(call.args.indexOf("--") + 1) ?? []; + expect(removeArgs[0]).toBe("rm"); + expect(removeArgs[1]).toBe("-f"); + expect(removeArgs).toContain("/tmp/nemoclaw-proxy-env.sh"); + expect(removeArgs).toContain("/tmp/nemoclaw-ciao-network-guard.js"); + expect(removeArgs).toContain("/tmp/nemoclaw-sandbox-safety-net.js"); + expect(removeArgs).toContain("/tmp/nemoclaw-slack-channel-guard.js"); + expect(removeArgs).toContain("/tmp/nemoclaw-http-proxy-fix.js"); + expect(removeArgs).toContain("/tmp/nemoclaw-ws-proxy-fix.js"); + expect(removeArgs).toContain("/tmp/nemoclaw-nemotron-inference-fix.js"); + expect(removeArgs).toContain("/tmp/nemoclaw-seccomp-guard.js"); + }); + + it("wipeGuardChain throws when the sandbox returns a non-zero exit", async () => { + const runner = new ScriptedRunner(); + runner.queue({ exitCode: 1, stderr: "permission denied" }); + const sandbox = new SandboxClient(runner); + + await expect(sandbox.wipeGuardChain("e2e-2701")).rejects.toThrow(/wipe guard chain/); + }); + + it("killGatewayTree pkills the openclaw tree and verifies nothing remains", async () => { + const runner = new ScriptedRunner(); + const sandbox = new SandboxClient(runner); + + await sandbox.killGatewayTree("e2e-2701"); + + const args = runner.calls[0]?.args ?? []; + const script = args[args.length - 1]; + expect(script).toContain("pkill -9 -f '[o]penclaw'"); + expect(script).toContain("pgrep -af '[o]penclaw'"); + }); + + it("killGatewayTree throws if openclaw processes survive the kill", async () => { + const runner = new ScriptedRunner(); + runner.queue({ exitCode: 1 }); + const sandbox = new SandboxClient(runner); + + await expect(sandbox.killGatewayTree("e2e-2701")).rejects.toThrow(/kill gateway tree/); + }); + + it("rejects sandbox names that fail validation", async () => { + const runner = new ScriptedRunner(); + const sandbox = new SandboxClient(runner); + + await expect(sandbox.wipeGuardChain("../bad")).rejects.toThrow(/sandbox name is invalid/); + await expect(sandbox.killGatewayTree("../bad")).rejects.toThrow(/sandbox name is invalid/); + }); +}); diff --git a/test/e2e/test-issue-2478-crash-loop-recovery.sh b/test/e2e/test-issue-2478-crash-loop-recovery.sh index 59bb09f46f5..1b39f8e5a59 100755 --- a/test/e2e/test-issue-2478-crash-loop-recovery.sh +++ b/test/e2e/test-issue-2478-crash-loop-recovery.sh @@ -510,6 +510,26 @@ if [ -z "$NEGATIVE_PID" ]; then fi info "Negative-case recovery respawned gateway pid=$NEGATIVE_PID" +# ── #2701 contract assertion ───────────────────────────────────────── +# After recovery, the guard chain MUST be restored. Today this fails on +# `main`: recovery emits the WARNING above and then launches the gateway +# naked, leaving /tmp/nemoclaw-proxy-env.sh absent. On aarch64 / DGX Spark +# this triggers the @homebridge/ciao crash loop documented in #2701; on +# x86 the gateway boots fine but the guard chain is still missing, which +# is the failure shape this assertion catches. +# +# Once the #2701 fix lands, recovery re-emits the chain before launching +# and this assertion flips green. Will fail on origin/main as of 2026-06-09. +if gateway_guards_active "$NEGATIVE_PID"; then + pass "#2701: recovery restored guard chain (proxy-env.sh + safety-net + ciao)" +else + fail "#2701: recovery did NOT restore guard chain — gateway respawned naked (DGX Spark crash-loop scenario)" + gateway_diagnostics "$NEGATIVE_PID" + # Do not exit 1 yet — we still want Phase 4's restore + Phase 5 soak to + # run so the artifact bundle is comparable to historical runs. Defer the + # failure decision to the test-level fail counter at the end of the file. +fi + # Restore proxy-env.sh by base64-injecting the snapshot via argv. `openshell # sandbox exec` does not pipe stdin from the caller through to the subshell, # so a `printf | sandbox_exec sh -c 'cat > file'` would leave an empty file. diff --git a/vitest.config.ts b/vitest.config.ts index ad8f7d88042..80ddf2f6a50 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -37,6 +37,10 @@ export default defineConfig({ "**/node_modules/**", "**/.claude/**", "test/e2e/**", + // Live scenario tests own their own gated project (e2e-scenarios-live) + // and require Docker + a real onboard to pass. Excluding here keeps + // the cli project (and pre-commit `Test (cli)`) green locally. + "test/e2e-scenario/live/**", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-openshell-version-check.test.ts",