diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 14ecfe6aed5..7b2ad322b2e 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,sandbox-survival-vitest" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -93,12 +93,12 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,sandbox-survival-vitest" args=(--emit-live-matrix) matrix="" hermes_selected=false registry_scenarios=() - free_standing_scenarios=(openshell-version-pin onboard-negative-paths skill-agent inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference) + free_standing_scenarios=(openshell-version-pin onboard-negative-paths skill-agent inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference sandbox-survival) is_free_standing_scenario() { local id="$1" local known @@ -652,7 +652,6 @@ jobs: if-no-files-found: ignore retention-days: 14 - credential-migration-vitest: needs: validate-jobs if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',credential-migration-vitest,') }} @@ -729,8 +728,6 @@ jobs: if-no-files-found: ignore retention-days: 14 - - runtime-overrides-vitest: needs: [validate-jobs, generate-matrix] if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',runtime-overrides-vitest,') || contains(format(',{0},', inputs.scenarios), ',runtime-overrides,') }} @@ -1311,6 +1308,94 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + sandbox-survival-vitest: + needs: [validate-jobs, generate-matrix] + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-survival-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-survival,') }} + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-sandbox-survival + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-survival + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-survival" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run sandbox survival live test + # Migrated from test/e2e/test-sandbox-survival.sh. This intentionally + # runs install.sh from Vitest to preserve installer/OpenShell/onboard + # fidelity before exercising gateway restart, state survival, and live + # inference.local before and after restart. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/sandbox-survival.test.ts \ + --silent=false --reporter=default + + - name: Upload sandbox survival artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-sandbox-survival + path: e2e-artifacts/vitest/sandbox-survival/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -1506,6 +1591,7 @@ jobs: launchable-smoke-vitest, double-onboard-vitest, model-router-provider-routed-inference-vitest, + sandbox-survival-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, issue-4434-tui-unreachable-inference-vitest, diff --git a/test/e2e-scenario/fixtures/clients/command.ts b/test/e2e-scenario/fixtures/clients/command.ts index c7be2773748..c1ae0846c8f 100644 --- a/test/e2e-scenario/fixtures/clients/command.ts +++ b/test/e2e-scenario/fixtures/clients/command.ts @@ -11,6 +11,18 @@ export interface CommandRunner { run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise; } +export function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +export function outputContainsSandbox( + result: Pick, + sandboxName: string, +): boolean { + const escaped = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`(^|\\s)${escaped}(\\s|$)`, "m").test(resultText(result)); +} + export function assertExitZero(result: ShellProbeResult, label: string): void { if (result.exitCode === 0) return; const fallback = result.signal diff --git a/test/e2e-scenario/fixtures/clients/gateway.ts b/test/e2e-scenario/fixtures/clients/gateway.ts index c82fcd5ae5d..5958a17cb57 100644 --- a/test/e2e-scenario/fixtures/clients/gateway.ts +++ b/test/e2e-scenario/fixtures/clients/gateway.ts @@ -42,6 +42,14 @@ const DEFAULT_GUARD_MARKERS: ReadonlyArray = [ /** Default gateway log path inside the sandbox. */ const GATEWAY_LOG_PATH = "/tmp/gateway.log"; +const DOCKER_DRIVER_GATEWAY_PID_RELPATH = [ + ".local", + "state", + "nemoclaw", + "openshell-docker-gateway", + "openshell-gateway.pid", +] as const; +const DEFAULT_GATEWAY_CONTAINER = "openshell-cluster-nemoclaw"; export interface ExpectGuardChainOptions extends ShellProbeRunOptions { /** Markers required in `/tmp/nemoclaw-proxy-env.sh`. Defaults to safety-net + ciao. */ @@ -60,6 +68,11 @@ export interface ExpectPidStableOptions extends ShellProbeRunOptions { pollIntervalSeconds?: number; } +export interface HostGatewayRuntime { + kind: "pid" | "container"; + id: string; +} + export class GatewayClient { private readonly host: HostCliClient; private readonly sandbox: SandboxClient; @@ -82,6 +95,74 @@ export class GatewayClient { return result; } + async resolveHostRuntime(): Promise { + const pid = await this.host.command( + "sh", + [ + "-lc", + `pid_file=\"$HOME/${DOCKER_DRIVER_GATEWAY_PID_RELPATH.join("/")}\"; ` + + `if [ -f \"$pid_file\" ]; then ` + + `pid=\"$(tr -d '[:space:]' <\"$pid_file\" 2>/dev/null || true)\"; ` + + `if [ -n \"$pid\" ] && kill -0 \"$pid\" 2>/dev/null; then printf '%s\\n' \"$pid\"; exit 0; fi; ` + + `fi; exit 1`, + ], + { + artifactName: "gateway-runtime-pid-probe", + env: probeEnv(), + timeoutMs: 15_000, + }, + ); + if (pid.exitCode === 0 && pid.stdout.trim()) { + return { kind: "pid", id: pid.stdout.trim() }; + } + + const container = await this.host.command( + "docker", + ["ps", "-qf", `name=${DEFAULT_GATEWAY_CONTAINER}`], + { + artifactName: "gateway-runtime-container-probe", + env: probeEnv(), + timeoutMs: 15_000, + }, + ); + const id = container.stdout.trim().split(/\r?\n/).find(Boolean); + return id ? { kind: "container", id } : null; + } + + async expectHostRuntimeStopped(options: ShellProbeRunOptions = {}): Promise { + const runtime = await this.resolveHostRuntime(); + if (runtime) { + throw new Error( + `gateway runtime still appears to be running after stop: ${runtime.kind}:${runtime.id}`, + ); + } + if (options.artifactName) { + await this.host.command("true", [], { + artifactName: options.artifactName, + env: probeEnv(), + timeoutMs: 5_000, + }); + } + } + + async expectOpenshellStatusConnected( + gatewayName = "nemoclaw", + options: ShellProbeRunOptions = {}, + ): Promise { + const result = await this.host.command("openshell", ["status"], { + artifactName: `openshell-status-${gatewayName}`, + env: probeEnv(), + timeoutMs: 30_000, + ...options, + }); + assertExitZero(result, "openshell status"); + const text = `${result.stdout}\n${result.stderr}`; + if (!/connected/i.test(text) || !new RegExp(gatewayName, "i").test(text)) { + throw new Error(`openshell status did not report connected gateway '${gatewayName}'.`); + } + return result; + } + // ─── Guard-chain recovery probes (#2478, #2701) ──────────────────── /** diff --git a/test/e2e-scenario/fixtures/clients/host.ts b/test/e2e-scenario/fixtures/clients/host.ts index e33684760a6..5fbb37b8876 100644 --- a/test/e2e-scenario/fixtures/clients/host.ts +++ b/test/e2e-scenario/fixtures/clients/host.ts @@ -4,7 +4,13 @@ import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { trustedShellCommand } from "../shell-probe.ts"; -import { artifactLabel, assertExitZero, type CommandRunner } from "./command.ts"; +import { + artifactLabel, + assertExitZero, + outputContainsSandbox, + resultText, + type CommandRunner, +} from "./command.ts"; export interface HostClientOptions { cliPath?: string; @@ -60,4 +66,70 @@ export class HostCliClient { assertExitZero(result, "nemoclaw --version"); return result; } + + async expectListed( + sandboxName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + const result = await this.nemoclaw(["list"], { + artifactName: `nemoclaw-list-${artifactLabel(sandboxName)}`, + env: buildAvailabilityProbeEnv(), + ...options, + }); + assertExitZero(result, "nemoclaw list"); + if (!outputContainsSandbox(result, sandboxName)) { + throw new Error(`nemoclaw list did not include '${sandboxName}': ${resultText(result)}`); + } + return result; + } + + async expectStatus( + sandboxName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + const result = await this.nemoclaw([sandboxName, "status"], { + artifactName: `nemoclaw-status-${artifactLabel(sandboxName)}`, + env: buildAvailabilityProbeEnv(), + ...options, + }); + assertExitZero(result, `nemoclaw ${sandboxName} status`); + return result; + } + + async destroySandbox( + sandboxName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + return await this.nemoclaw([sandboxName, "destroy", "--yes"], { + artifactName: `destroy-sandbox-${artifactLabel(sandboxName)}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 15 * 60_000, + ...options, + }); + } + + async cleanupSandbox(sandboxName: string, options: ShellProbeRunOptions = {}): Promise { + const result = await this.destroySandbox(sandboxName, options); + if (result.exitCode === 0) return; + const text = resultText(result); + if ( + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/i.test( + text, + ) + ) { + return; + } + assertExitZero(result, `cleanup destroy sandbox ${sandboxName}`); + } + + async bestEffortCleanupSandbox( + sandboxName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + try { + await this.cleanupSandbox(sandboxName, options); + } catch { + // Best-effort cleanup must not mask the primary setup or assertion failure. + } + } } diff --git a/test/e2e-scenario/fixtures/clients/index.ts b/test/e2e-scenario/fixtures/clients/index.ts index 65c2e0c7cb3..2715350a33a 100644 --- a/test/e2e-scenario/fixtures/clients/index.ts +++ b/test/e2e-scenario/fixtures/clients/index.ts @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -export { assertExitZero, type CommandRunner } from "./command.ts"; +export { + assertExitZero, + outputContainsSandbox, + resultText, + type CommandRunner, +} from "./command.ts"; export { GatewayClient } from "./gateway.ts"; export { HostCliClient } from "./host.ts"; export { @@ -13,6 +18,7 @@ export { } from "./provider.ts"; export { SandboxClient, + sandboxAccessEnv, trustedSandboxShellScript, type TrustedSandboxShellScript, validateSandboxName, diff --git a/test/e2e-scenario/fixtures/clients/sandbox.ts b/test/e2e-scenario/fixtures/clients/sandbox.ts index 31e3e4662d9..57e2a9db892 100644 --- a/test/e2e-scenario/fixtures/clients/sandbox.ts +++ b/test/e2e-scenario/fixtures/clients/sandbox.ts @@ -4,7 +4,12 @@ import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; import { trustedShellCommand } from "../shell-probe.ts"; -import { artifactLabel, assertExitZero, type CommandRunner } from "./command.ts"; +import { + artifactLabel, + assertExitZero, + outputContainsSandbox, + type CommandRunner, +} from "./command.ts"; /** * Default env for openshell-targeted spawns. ShellProbe filters env via @@ -22,6 +27,10 @@ function openshellProbeEnv(): NodeJS.ProcessEnv { }; } +export function sandboxAccessEnv(): NodeJS.ProcessEnv { + return openshellProbeEnv(); +} + export interface SandboxClientOptions { openshellPath?: string; } @@ -63,7 +72,10 @@ export class SandboxClient { } list(options: ShellProbeRunOptions = {}): Promise { - return this.openshell(["sandbox", "list"], { artifactName: "sandbox-list", ...options }); + return this.openshell(["sandbox", "list"], { + artifactName: "sandbox-list", + ...options, + }); } status(name: string, options: ShellProbeRunOptions = {}): Promise { @@ -119,6 +131,16 @@ export class SandboxClient { return result; } + async expectListed(name: string, options: ShellProbeRunOptions = {}): Promise { + validateSandboxName(name); + const result = await this.list({ env: openshellProbeEnv(), ...options }); + assertExitZero(result, "openshell sandbox list"); + if (!outputContainsSandbox(result, name)) { + throw new Error(`openshell sandbox list did not include '${name}'.`); + } + return result; + } + /** * Disruption helper: simulate the post-pod-recreate /tmp wipe by removing * the guard chain files. After this, a sandbox containing a running gateway diff --git a/test/e2e-scenario/fixtures/e2e-test.ts b/test/e2e-scenario/fixtures/e2e-test.ts index 211aeae8726..5969892a943 100644 --- a/test/e2e-scenario/fixtures/e2e-test.ts +++ b/test/e2e-scenario/fixtures/e2e-test.ts @@ -98,8 +98,8 @@ export const test = base.extend({ onboard: async ({ artifacts, cleanup, host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets, cleanup, artifacts)); }, - lifecycle: async ({ cleanup, host, sandbox }, use) => { - await use(new LifecyclePhaseFixture(host, sandbox, cleanup)); + lifecycle: async ({ cleanup, gateway, host, sandbox }, use) => { + await use(new LifecyclePhaseFixture(host, sandbox, cleanup, gateway)); }, runtime: async ({ provider, sandbox }, use) => { await use(new RuntimePhaseFixture(sandbox, provider)); diff --git a/test/e2e-scenario/fixtures/phases/lifecycle.ts b/test/e2e-scenario/fixtures/phases/lifecycle.ts index 263f7c6f965..1ca747678bd 100644 --- a/test/e2e-scenario/fixtures/phases/lifecycle.ts +++ b/test/e2e-scenario/fixtures/phases/lifecycle.ts @@ -3,6 +3,7 @@ import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { assertExitZero } from "../clients/command.ts"; +import type { GatewayClient, HostGatewayRuntime } from "../clients/gateway.ts"; import type { HostCliClient } from "../clients/host.ts"; import type { SandboxClient } from "../clients/sandbox.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; @@ -61,6 +62,7 @@ export class LifecyclePhaseFixture { private readonly host: HostCliClient, private readonly sandbox: SandboxClient, private readonly cleanup: LifecycleCleanup, + private readonly gateway?: GatewayClient, ) {} async simulate( @@ -152,7 +154,10 @@ export class LifecyclePhaseFixture { timeoutMs: DOCKER_PROBE_TIMEOUT_MS, }); assertExitZero(rename, `docker rename ${originalName} ${backupName}`); - steps.push({ id: `docker-rename:${originalName}->${backupName}`, results: [rename] }); + steps.push({ + id: `docker-rename:${originalName}->${backupName}`, + results: [rename], + }); this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { await this.host.command("docker", ["rename", backupName, originalName], { artifactName: `lifecycle-cleanup-docker-rename-back-${backupName}`, @@ -174,11 +179,144 @@ export class LifecyclePhaseFixture { env: buildAvailabilityProbeEnv(), timeoutMs: STATUS_TIMEOUT_MS, }); - steps.push({ id: `nemoclaw-status:${instance.sandboxName}`, results: [statusResult] }); + steps.push({ + id: `nemoclaw-status:${instance.sandboxName}`, + results: [statusResult], + }); return { profile: "post-reboot-recovery", steps }; } + async stopGatewayRuntime(): Promise { + const runtime = (await this.gateway?.resolveHostRuntime()) ?? null; + await this.host.command( + "sh", + ["-lc", "command -v openshell >/dev/null 2>&1 && openshell forward stop 18789 || true"], + { + artifactName: "lifecycle-gateway-forward-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + await this.host.command( + "sh", + ["-lc", "command -v openshell >/dev/null 2>&1 && openshell gateway stop -g nemoclaw || true"], + { + artifactName: "lifecycle-gateway-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + + const pidFileStop = await this.host.command( + "sh", + [ + "-lc", + `pid_file="$HOME/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid"; ` + + `if [ -f "$pid_file" ]; then ` + + `pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)"; ` + + `if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then ` + + `kill "$pid" 2>/dev/null || true; ` + + `for _ in 1 2 3 4 5 6 7 8 9 10; do kill -0 "$pid" 2>/dev/null || exit 0; sleep 1; done; ` + + `kill -9 "$pid" 2>/dev/null || true; ` + + `fi; fi`, + ], + { + artifactName: "lifecycle-gateway-pid-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(pidFileStop, "stop Docker-driver gateway PID"); + + const containerStop = await this.host.command( + "sh", + [ + "-lc", + `cid="$(docker ps -qf 'name=openshell-cluster-nemoclaw' 2>/dev/null | head -1)"; ` + + `if [ -n "$cid" ]; then docker stop "$cid" >/dev/null; fi`, + ], + { + artifactName: "lifecycle-gateway-container-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + assertExitZero(containerStop, "stop OpenShell gateway container"); + return runtime; + } + + async startGatewayRuntime( + previousRuntime: HostGatewayRuntime | null, + options: { sandboxName?: string } = {}, + ): Promise { + if (previousRuntime?.kind === "pid") { + const args = options.sandboxName ? [options.sandboxName, "status"] : ["status"]; + return await this.host.nemoclaw(args, { + artifactName: options.sandboxName + ? `lifecycle-gateway-recover-through-nemoclaw-status-${options.sandboxName}` + : "lifecycle-gateway-recover-through-nemoclaw-status", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + } + return await this.host.command("openshell", ["gateway", "start", "--name", "nemoclaw"], { + artifactName: "lifecycle-gateway-start", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + } + + async restartGatewayRuntime( + options: { delayMs?: number; sandboxName?: string } = {}, + ): Promise { + const previousRuntime = await this.stopGatewayRuntime(); + if (this.gateway) { + await this.gateway.expectHostRuntimeStopped({ + artifactName: "lifecycle-gateway-stopped", + }); + } + const delayMs = options.delayMs ?? 5_000; + if (delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + await this.startGatewayRuntime(previousRuntime, { + sandboxName: options.sandboxName, + }); + return previousRuntime; + } + + async waitForGatewayConnected( + options: { attempts?: number; intervalMs?: number } = {}, + ): Promise { + const attempts = options.attempts ?? 60; + const intervalMs = options.intervalMs ?? 5_000; + let lastError: unknown; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + if (this.gateway) { + await this.gateway.expectOpenshellStatusConnected("nemoclaw", { + artifactName: `lifecycle-gateway-health-${attempt}`, + }); + return; + } + const status = await this.host.command("openshell", ["status"], { + artifactName: `lifecycle-gateway-health-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertExitZero(status, "openshell status"); + if (/connected/i.test(`${status.stdout}\n${status.stderr}`)) return; + } catch (error) { + lastError = error; + } + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error( + `gateway did not become healthy after restart: ${lastError instanceof Error ? lastError.message : String(lastError ?? "unknown")}`, + ); + } + private async discoverLabeledContainerNames(instance: NemoClawInstance): Promise { const result = await this.host.command( "docker", diff --git a/test/e2e-scenario/fixtures/phases/runtime.ts b/test/e2e-scenario/fixtures/phases/runtime.ts index e417d30c1ea..c950aedf926 100644 --- a/test/e2e-scenario/fixtures/phases/runtime.ts +++ b/test/e2e-scenario/fixtures/phases/runtime.ts @@ -176,13 +176,35 @@ function hasChoiceContent(choice: unknown): boolean { if (!choice || typeof choice !== "object") return false; const message = (choice as { message?: unknown }).message; if (message && typeof message === "object") { - const content = (message as { content?: unknown }).content; + const content = (message as { content?: unknown; reasoning_content?: unknown }).content; + const reasoning = (message as { reasoning_content?: unknown }).reasoning_content; if (typeof content === "string" && content.length > 0) return true; + if (typeof reasoning === "string" && reasoning.length > 0) return true; } const text = (choice as { text?: unknown }).text; return typeof text === "string" && text.length > 0; } +function chatCompletionText(json: unknown): string { + if (!json || typeof json !== "object") return ""; + const choices = (json as { choices?: unknown }).choices; + if (!Array.isArray(choices)) return ""; + const parts: string[] = []; + for (const choice of choices) { + if (!choice || typeof choice !== "object") continue; + const message = (choice as { message?: unknown }).message; + if (message && typeof message === "object") { + const content = (message as { content?: unknown; reasoning_content?: unknown }).content; + const reasoning = (message as { reasoning_content?: unknown }).reasoning_content; + if (typeof content === "string") parts.push(content); + if (typeof reasoning === "string") parts.push(reasoning); + } + const text = (choice as { text?: unknown }).text; + if (typeof text === "string") parts.push(text); + } + return parts.join("\n"); +} + function assertChatCompletionShape(json: unknown, label: string): void { if (!json || typeof json !== "object") { throw new Error(`${label} response was not an object`); @@ -273,7 +295,9 @@ export class RuntimePhaseFixture { async expectInferenceLocalChatCompletion( instance: NemoClawInstance, - options: InferenceRuntimeChatOptions & { readonly route?: InferenceRoute } = {}, + options: InferenceRuntimeChatOptions & { + readonly route?: InferenceRoute; + } = {}, ): Promise { const endpoint = inferenceRouteUrl(options.route, CHAT_COMPLETIONS_PATH); const payload = openAiChatPayload(options); @@ -301,6 +325,38 @@ export class RuntimePhaseFixture { return { endpoint, result }; } + async expectInferenceLocalPong( + instance: NemoClawInstance, + options: InferenceRuntimeChatOptions & { + readonly attempts?: number; + readonly retryDelayMs?: number; + readonly route?: InferenceRoute; + } = {}, + ): Promise { + const attempts = options.attempts ?? 3; + const retryDelayMs = options.retryDelayMs ?? 5_000; + let last: InferenceRuntimeProbeResult | undefined; + let lastText = ""; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + last = await this.expectInferenceLocalChatCompletion(instance, { + maxTokens: 100, + prompt: "Reply with exactly one word: PONG", + ...options, + artifactName: `${options.artifactName ?? "runtime-inference-local-pong"}-${attempt}`, + }); + lastText = chatCompletionText(parseJsonBody(last.result.stdout, "inference.local PONG")); + if (/\bPONG\b/i.test(lastText)) return last; + } catch (error) { + lastText = error instanceof Error ? error.message : String(error); + } + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + throw new Error( + `inference.local PONG probe failed after ${attempts} attempts: ${lastText || last?.result.stdout || "empty response"}`, + ); + } + async expectInferenceLocalStatus( instance: NemoClawInstance, options: InferenceRuntimeStatusOptions = {}, diff --git a/test/e2e-scenario/fixtures/phases/state-validation.ts b/test/e2e-scenario/fixtures/phases/state-validation.ts index 47651635f27..0122390ae28 100644 --- a/test/e2e-scenario/fixtures/phases/state-validation.ts +++ b/test/e2e-scenario/fixtures/phases/state-validation.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ArtifactSink } from "../artifacts.ts"; import { + resultText, trustedProviderEndpoint, type GatewayClient, type HostCliClient, @@ -78,6 +79,11 @@ function outputContainsSandbox(result: ShellProbeResult, sandboxName: string): b return new RegExp(`(^|\\s)${escapeRegExp(sandboxName)}(\\s|$)`, "m").test(output); } +export interface SandboxMarker { + path: string; + value: string; +} + function statusProbeEnv(): NodeJS.ProcessEnv { return buildAvailabilityProbeEnv(); } @@ -94,6 +100,10 @@ function resultHttpCode(result: ShellProbeResult): string { return result.stdout.trim(); } +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'"'"'`)}'`; +} + function resultHasHttpCode(result: ShellProbeResult, allowedCodes: readonly string[]): boolean { return result.exitCode === 0 && allowedCodes.includes(resultHttpCode(result)); } @@ -142,6 +152,93 @@ export class StateValidationPhaseFixture { } } + expectLocalRegistryContains(sandboxName: string): void { + const reader = this.io.readRegistry ?? defaultReadRegistry; + const registry = reader(); + if (!registry) { + throw new Error( + `expected local registry entry for '${sandboxName}', but ${defaultRegistryPath()} does not exist.`, + ); + } + if (!Object.prototype.hasOwnProperty.call(registry.entries, sandboxName)) { + const present = Object.keys(registry.entries).sort().join(", ") || "(none)"; + throw new Error( + `expected local registry entry for '${sandboxName}', but registry contains: ${present}`, + ); + } + } + + async writeSandboxMarkers( + instance: NemoClawInstance, + markers: readonly SandboxMarker[], + ): Promise { + for (const marker of markers) { + const result = await this.sandbox.exec( + instance.sandboxName, + [ + "sh", + "-lc", + `mkdir -p "$(dirname ${shellQuote(marker.path)})" && printf '%s\\n' ${shellQuote(marker.value)} > ${shellQuote(marker.path)}`, + ], + { + artifactName: `state-marker-write-${path.basename(marker.path)}`, + env: statusProbeEnv(), + timeoutMs: 30_000, + }, + ); + if (result.exitCode !== 0) { + throw new Error(`failed to write sandbox marker ${marker.path}: ${resultText(result)}`); + } + } + } + + async expectSandboxMarkers( + instance: NemoClawInstance, + markers: readonly SandboxMarker[], + artifactPrefix = "state-marker-read", + ): Promise { + for (const marker of markers) { + const result = await this.sandbox.exec( + instance.sandboxName, + ["sh", "-lc", `cat ${shellQuote(marker.path)} 2>/dev/null`], + { + artifactName: `${artifactPrefix}-${path.basename(marker.path)}`, + env: statusProbeEnv(), + timeoutMs: 30_000, + }, + ); + const actual = result.stdout.trim(); + if (result.exitCode !== 0 || actual !== marker.value) { + throw new Error( + `sandbox marker ${marker.path} mismatch: expected '${marker.value}', got '${actual || ""}'`, + ); + } + } + } + + async expectSandboxDirectoryPopulated( + instance: NemoClawInstance, + directory: string, + artifactName = "state-directory-populated", + ): Promise { + const result = await this.sandbox.exec( + instance.sandboxName, + [ + "sh", + "-lc", + `find ${shellQuote(directory)} -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null`, + ], + { + artifactName, + env: statusProbeEnv(), + timeoutMs: 30_000, + }, + ); + if (result.exitCode !== 0 || result.stdout.trim() === "") { + throw new Error(`sandbox directory ${directory} is not populated after restart.`); + } + } + private async writeResult( status: "passed" | "failed", expectedState: string | ExpectedState, @@ -280,7 +377,11 @@ export class StateValidationPhaseFixture { `state-validation expected gateway to be absent, but ${healthUrl} responded healthy.`, ); } - return { id: "gateway-absent", status: "passed", results: [result, health] }; + return { + id: "gateway-absent", + status: "passed", + results: [result, health], + }; } private async expectSandboxRunning( @@ -319,7 +420,11 @@ export class StateValidationPhaseFixture { `but the registry contains: ${present}.`, ); } - return { id: "local-registry-entry-present", status: "passed", results: [] }; + return { + id: "local-registry-entry-present", + status: "passed", + results: [], + }; } private async expectDockerSandboxContainerPresent( @@ -358,7 +463,11 @@ export class StateValidationPhaseFixture { `*-nemoclaw-gpu-backup-* sibling), but docker ps -a returned none.`, ); } - return { id: "docker-sandbox-container-present", status: "passed", results: [result] }; + return { + id: "docker-sandbox-container-present", + status: "passed", + results: [result], + }; } private async expectSandboxAbsent( diff --git a/test/e2e-scenario/live/sandbox-survival.test.ts b/test/e2e-scenario/live/sandbox-survival.test.ts new file mode 100644 index 00000000000..6fe50badc57 --- /dev/null +++ b/test/e2e-scenario/live/sandbox-survival.test.ts @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-sandbox-survival.sh. + * + * Preserves the legacy real boundaries: install.sh/onboard, Docker, OpenShell + * gateway stop/start, NemoClaw registry/list/status, sandbox SSH/exec, durable + * /sandbox/.openclaw state markers, and inference.local chat completion before + * and after gateway restart. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero, resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { NemoClawInstance } from "../fixtures/phases/index.ts"; +import type { SandboxMarker } from "../fixtures/phases/state-validation.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-survival"; +const MIN_OPENSHELL_VERSION = "0.0.24"; +const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b"; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function versionGte(actual: string, minimum: string): boolean { + const actualParts = actual.split(".").map((part) => Number.parseInt(part, 10)); + const minimumParts = minimum.split(".").map((part) => Number.parseInt(part, 10)); + for (let index = 0; index < Math.max(actualParts.length, minimumParts.length); index += 1) { + const a = Number.isFinite(actualParts[index]) ? actualParts[index] : 0; + const b = Number.isFinite(minimumParts[index]) ? minimumParts[index] : 0; + if (a > b) return true; + if (a < b) return false; + } + return true; +} + +function extractSemver(raw: string): string | undefined { + return raw.match(/\d+\.\d+\.\d+/)?.[0]; +} + +function installEnv(apiKey: string): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NVIDIA_API_KEY: apiKey, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + }; +} + +async function expectSandboxExecAlive( + sandboxName: string, + exec: ( + script: string, + artifactName: string, + ) => Promise<{ stdout: string; stderr: string; exitCode: number | null }>, + artifactName: string, +): Promise { + const alive = await exec("echo alive", artifactName); + expect(alive.exitCode, `${sandboxName} exec failed: ${resultText(alive)}`).toBe(0); + expect(alive.stdout.trim(), resultText(alive)).toBe("alive"); +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "sandbox survives gateway restart with registry, state, SSH, and live inference intact", + async ({ + artifacts, + cleanup, + host, + lifecycle, + runtime, + sandbox, + secrets, + skip, + stateValidation, + }) => { + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "sandbox-survival", + runner: "vitest", + boundary: "install-sh-docker-openshell-gateway-sandbox-inference", + legacySource: "test/e2e/test-sandbox-survival.sh", + contracts: [ + "install.sh --non-interactive creates the named OpenClaw sandbox", + "NemoClaw registry, nemoclaw list/status, and openshell sandbox list discover the sandbox", + "OpenShell version supports gateway resume and state persistence", + "sandbox exec/SSH-equivalent access works before and after gateway restart", + "inference.local returns a live PONG before and after gateway restart", + "markers under /sandbox/.openclaw survive the gateway stop/start cycle", + "final destroy removes the sandbox from NemoClaw registry/list state", + ], + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-sandbox-survival", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for sandbox survival E2E: ${resultText(docker)}`); + } + skip("Docker is required for sandbox survival E2E"); + } + + const modelsReachable = await host.command( + "curl", + ["-sf", "--max-time", "10", "https://integrate.api.nvidia.com/v1/models"], + { + artifactName: "prereq-integrate-api-models", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 15_000, + }, + ); + expect(modelsReachable.exitCode, resultText(modelsReachable)).toBe(0); + expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); + + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "pre-cleanup-nemoclaw-destroy-sandbox-survival", + }); + await host.command( + "sh", + [ + "-lc", + `command -v openshell >/dev/null 2>&1 && openshell sandbox delete ${SANDBOX_NAME} || true`, + ], + { + artifactName: "pre-cleanup-openshell-delete-sandbox-survival", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }, + ); + await lifecycle.stopGatewayRuntime(); + await host.command( + "sh", + [ + "-lc", + "command -v openshell >/dev/null 2>&1 && openshell gateway destroy -g nemoclaw || true", + ], + { + artifactName: "pre-cleanup-openshell-gateway-destroy", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }, + ); + fs.rmSync(path.join(process.env.HOME ?? "", ".nemoclaw", "onboard.lock"), { + force: true, + }); + + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-sandbox-survival", + }); + }); + cleanup.add("destroy shared NemoClaw gateway", async () => { + await host.command( + "sh", + [ + "-lc", + "command -v openshell >/dev/null 2>&1 && openshell gateway destroy -g nemoclaw || true", + ], + { + artifactName: "cleanup-openshell-gateway-destroy", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }, + ); + }); + + const install = await host.command("bash", ["install.sh", "--non-interactive"], { + artifactName: "install-sh-sandbox-survival", + cwd: REPO_ROOT, + env: installEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: 20 * 60_000, + }); + expect(install.exitCode, resultText(install)).toBe(0); + + await host.expectNemoclawAvailable(); + const openshellVersion = await host.command("openshell", ["--version"], { + artifactName: "openshell-version-sandbox-survival", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertExitZero(openshellVersion, "openshell --version"); + const version = extractSemver(resultText(openshellVersion)); + expect(version, resultText(openshellVersion)).toBeTruthy(); + expect(versionGte(version!, MIN_OPENSHELL_VERSION), resultText(openshellVersion)).toBe(true); + + const instance: NemoClawInstance = { + onboarding: "cloud-openclaw", + sandboxName: SANDBOX_NAME, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + platformOs: "ubuntu", + gatewayUrl: "http://127.0.0.1:18789", + result: install, + }; + + stateValidation.expectLocalRegistryContains(SANDBOX_NAME); + await host.expectListed(SANDBOX_NAME, { + artifactName: "post-install-nemoclaw-list", + }); + await sandbox.expectListed(SANDBOX_NAME, { + artifactName: "post-install-openshell-sandbox-list", + }); + await host.expectStatus(SANDBOX_NAME, { + artifactName: "post-install-nemoclaw-status", + }); + + const execShell = (script: string, artifactName: string) => + sandbox.exec(SANDBOX_NAME, ["sh", "-lc", script], { + artifactName, + env: sandboxAccessEnv(), + timeoutMs: 60_000, + }); + await expectSandboxExecAlive(SANDBOX_NAME, execShell, "baseline-sandbox-exec-alive"); + + await runtime.expectInferenceLocalPong(instance, { + artifactName: "baseline-inference-local-pong", + model: MODEL, + curlMaxTimeSeconds: 60, + timeoutMs: 90_000, + redactionValues: [apiKey], + }); + + const markerValue = `nemoclaw-survival-${Date.now()}`; + const markers: SandboxMarker[] = [ + { + path: "/sandbox/.openclaw/.survival-marker-workspace", + value: markerValue, + }, + { path: "/sandbox/.openclaw/.survival-marker", value: markerValue }, + { + path: "/sandbox/.openclaw/test-data/nested-marker.txt", + value: markerValue, + }, + ]; + await stateValidation.writeSandboxMarkers(instance, markers); + await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read"); + + await lifecycle.restartGatewayRuntime({ + delayMs: 5_000, + sandboxName: SANDBOX_NAME, + }); + await lifecycle.waitForGatewayConnected({ + attempts: 60, + intervalMs: 5_000, + }); + + await sandbox.expectListed(SANDBOX_NAME, { + artifactName: "post-restart-openshell-sandbox-list", + }); + stateValidation.expectLocalRegistryContains(SANDBOX_NAME); + await host.expectListed(SANDBOX_NAME, { + artifactName: "post-restart-nemoclaw-list", + }); + await host.expectStatus(SANDBOX_NAME, { + artifactName: "post-restart-nemoclaw-status", + timeoutMs: 120_000, + }); + await expectSandboxExecAlive(SANDBOX_NAME, execShell, "post-restart-sandbox-exec-alive"); + await stateValidation.expectSandboxMarkers(instance, markers, "post-restart-marker-read"); + await stateValidation.expectSandboxDirectoryPopulated( + instance, + "/sandbox/.openclaw", + "post-restart-openclaw-directory-populated", + ); + + await runtime.expectInferenceLocalPong(instance, { + artifactName: "post-restart-inference-local-pong", + model: MODEL, + curlMaxTimeSeconds: 60, + timeoutMs: 90_000, + redactionValues: [apiKey], + }); + + await host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "final-destroy-sandbox-survival", + timeoutMs: 15 * 60_000, + }); + const afterDestroyList = await host.nemoclaw(["list"], { + artifactName: "post-destroy-nemoclaw-list", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expect(resultText(afterDestroyList), "sandbox still listed after destroy").not.toMatch( + new RegExp(`(^|\\s)${SANDBOX_NAME}(\\s|$)`, "m"), + ); + + await artifacts.writeJson("scenario-result.json", { + id: "sandbox-survival", + status: "passed", + legacySource: "test/e2e/test-sandbox-survival.sh", + assertions: { + installCompleted: install.exitCode === 0, + registryListedBeforeRestart: true, + inferenceLocalBeforeRestart: true, + markersPersistedAfterRestart: true, + inferenceLocalAfterRestart: true, + destroyedAtEnd: true, + }, + }); + }, + 30 * 60_000, +); diff --git a/test/e2e-scenario/support-tests/e2e-clients.test.ts b/test/e2e-scenario/support-tests/e2e-clients.test.ts index c0b8043b47b..4525566b323 100644 --- a/test/e2e-scenario/support-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/support-tests/e2e-clients.test.ts @@ -41,7 +41,11 @@ class FakeRunner implements CommandRunner { command: TrustedShellCommand, options?: ShellProbeRunOptions, ): Promise { - this.calls.push({ command: command.command, args: [...command.args], options }); + this.calls.push({ + command: command.command, + args: [...command.args], + options, + }); return { command: [command.command, ...command.args], exitCode: this.exitCode, @@ -80,9 +84,28 @@ describe("E2E fixture clients", () => { ]); }); + it("host client validates list/status and cleans up sandbox destroys", async () => { + const runner = new FakeRunner(); + runner.stdout = "NAME\nassistant\n"; + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await host.expectListed("assistant"); + await host.expectStatus("assistant"); + await host.cleanupSandbox("assistant"); + + expect(runner.calls.map((call) => ({ command: call.command, args: call.args }))).toEqual([ + { command: "nemoclaw", args: ["list"] }, + { command: "nemoclaw", args: ["assistant", "status"] }, + { command: "nemoclaw", args: ["assistant", "destroy", "--yes"] }, + ]); + }); + it("host client propagates cwd, env, and timeout options", async () => { const runner = new FakeRunner(); - const host = new HostCliClient(runner, { cliPath: "nemoclaw", cwd: "/tmp/project" }); + const host = new HostCliClient(runner, { + cliPath: "nemoclaw", + cwd: "/tmp/project", + }); await host.nemoclaw(["status"], { env: { NEMOCLAW_TEST_VALUE: "1" }, @@ -141,6 +164,37 @@ describe("E2E fixture clients", () => { }); }); + it("gateway client resolves host runtime and validates OpenShell status", async () => { + const pidRunner = new FakeRunner(); + pidRunner.stdout = "12345\n"; + const pidHost = new HostCliClient(pidRunner, { cliPath: "nemoclaw" }); + await expect( + new GatewayClient(pidHost, new SandboxClient(pidRunner)).resolveHostRuntime(), + ).resolves.toEqual({ + kind: "pid", + id: "12345", + }); + + const containerRunner = new FakeRunner(); + containerRunner.exitCode = 1; + const containerHost = new HostCliClient(containerRunner, { + cliPath: "nemoclaw", + }); + const containerGateway = new GatewayClient(containerHost, new SandboxClient(containerRunner)); + const runtime = containerGateway.resolveHostRuntime(); + containerRunner.exitCode = 0; + containerRunner.stdout = "abc123\n"; + await expect(runtime).resolves.toEqual({ kind: "container", id: "abc123" }); + + const statusRunner = new FakeRunner(); + statusRunner.stdout = "Connected to nemoclaw\n"; + const statusHost = new HostCliClient(statusRunner, { cliPath: "nemoclaw" }); + await new GatewayClient( + statusHost, + new SandboxClient(statusRunner), + ).expectOpenshellStatusConnected(); + }); + it("sandbox client builds OpenShell sandbox commands", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); @@ -156,6 +210,23 @@ describe("E2E fixture clients", () => { }); }); + it("sandbox client validates list output using the OpenShell gateway env", async () => { + const runner = new FakeRunner(); + runner.stdout = "NAME\nassistant\n"; + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + await sandbox.expectListed("assistant"); + + expect(runner.calls[0]).toMatchObject({ + command: "openshell", + args: ["sandbox", "list"], + options: { + artifactName: "sandbox-list", + env: expect.objectContaining({ OPENSHELL_GATEWAY: "nemoclaw" }), + }, + }); + }); + it("sandbox client preserves caller-provided probe options", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); @@ -367,10 +438,14 @@ describe("E2E fixture clients", () => { }), ).toThrow(/not allowed/); expect(() => - trustedProviderEndpoint("https://10.0.0.1/models", { allowedHosts: ["10.0.0.1"] }), + trustedProviderEndpoint("https://10.0.0.1/models", { + allowedHosts: ["10.0.0.1"], + }), ).toThrow(/private or link-local/); expect(() => - trustedProviderEndpoint("https://[fd00::1]/models", { allowedHosts: ["fd00::1"] }), + trustedProviderEndpoint("https://[fd00::1]/models", { + allowedHosts: ["fd00::1"], + }), ).toThrow(/private or link-local/); }); @@ -430,6 +505,18 @@ describe("E2E fixture clients", () => { await expect(provider.getJson(endpoint)).rejects.not.toThrow(/query-token-value|api_key/); }); + it("shared command helpers match complete sandbox names", async () => { + const { outputContainsSandbox, resultText } = await import("../fixtures/clients/index.ts"); + const result = { + stdout: "NAME\nassistant-old\nassistant\n", + stderr: "", + }; + + expect(resultText(result)).toContain("assistant"); + expect(outputContainsSandbox(result, "assistant")).toBe(true); + expect(outputContainsSandbox(result, "assist")).toBe(false); + }); + it("assertExitZero reports non-zero and signaled commands", () => { const result: ShellProbeResult = { command: ["cmd"], @@ -456,7 +543,9 @@ describe("E2E fixture clients", () => { await expect(state.exists(file)).resolves.toBe(true); await expect(state.exists(path.join(tmp, "missing.json"))).resolves.toBe(false); - await expect(state.readJson(file)).resolves.toEqual({ sandbox: "assistant" }); + await expect(state.readJson(file)).resolves.toEqual({ + sandbox: "assistant", + }); await expect(state.exists(`bad${"\0"}path`)).rejects.toThrow(); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/e2e-scenario/support-tests/e2e-phase-lifecycle.test.ts b/test/e2e-scenario/support-tests/e2e-phase-lifecycle.test.ts index a960495412d..2f038729296 100644 --- a/test/e2e-scenario/support-tests/e2e-phase-lifecycle.test.ts +++ b/test/e2e-scenario/support-tests/e2e-phase-lifecycle.test.ts @@ -3,7 +3,12 @@ import { describe, expect, expectTypeOf, it } from "vitest"; -import { HostCliClient, SandboxClient, type CommandRunner } from "../fixtures/clients/index.ts"; +import { + GatewayClient, + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../fixtures/clients/index.ts"; import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; import { buildBackupContainerName, @@ -56,7 +61,11 @@ class FakeRunner implements CommandRunner { command: TrustedShellCommand, options?: ShellProbeRunOptions, ): Promise { - this.calls.push({ command: command.command, args: [...command.args], options }); + this.calls.push({ + command: command.command, + args: [...command.args], + options, + }); const response = this.responses.shift(); if (!response) { throw new Error( @@ -121,8 +130,14 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", "{{.Names}}", ], }, - { command: "docker", args: ["stop", "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw"] }, - { command: "nemoclaw", args: ["e2e-ubuntu-repo-cloud-openclaw", "status"] }, + { + command: "docker", + args: ["stop", "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw"], + }, + { + command: "nemoclaw", + args: ["e2e-ubuntu-repo-cloud-openclaw", "status"], + }, ]); expect(cleanup.calls.map((call) => call.name)).toEqual([ "lifecycle.docker-start:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", @@ -197,6 +212,64 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-bac }); }); +describe("LifecyclePhaseFixture gateway runtime restart helpers", () => { + it("stops PID/container runtimes, starts the previous runtime shape, and polls health", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "12345\n")); // resolveHostRuntime pid probe + runner.enqueue(shellResult(0)); // forward stop + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(0)); // pid stop + runner.enqueue(shellResult(0)); // container stop + runner.enqueue(shellResult(1, "")); // expectHostRuntimeStopped pid probe + runner.enqueue(shellResult(0, "")); // expectHostRuntimeStopped container probe + runner.enqueue(shellResult(0)); // lifecycle-gateway-stopped true artifact + runner.enqueue(shellResult(0, "status recovered\n")); // start through nemoclaw status + runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // waitForGatewayConnected + const cleanup = new FakeCleanup(); + const host = new HostCliClient(runner); + const sandbox = new SandboxClient(runner); + const fx = new LifecyclePhaseFixture(host, sandbox, cleanup, new GatewayClient(host, sandbox)); + + await expect(fx.restartGatewayRuntime({ delayMs: 0 })).resolves.toEqual({ + kind: "pid", + id: "12345", + }); + await fx.waitForGatewayConnected({ attempts: 1, intervalMs: 1 }); + + expect(runner.calls.map((call) => `${call.command} ${call.args.join(" ")}`)).toEqual([ + expect.stringContaining("sh -lc pid_file="), + "sh -lc command -v openshell >/dev/null 2>&1 && openshell forward stop 18789 || true", + "sh -lc command -v openshell >/dev/null 2>&1 && openshell gateway stop -g nemoclaw || true", + expect.stringContaining("sh -lc pid_file="), + expect.stringContaining("sh -lc cid="), + expect.stringContaining("sh -lc pid_file="), + "docker ps -qf name=openshell-cluster-nemoclaw", + "true ", + "nemoclaw status", + "openshell status", + ]); + }); + + it("can recover a PID runtime through sandbox-specific status", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "status recovered\n")); + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).startGatewayRuntime( + { kind: "pid", id: "12345" }, + { + sandboxName: "e2e-survival", + }, + ), + ).resolves.toMatchObject({ exitCode: 0 }); + + expect(runner.calls.map((call) => `${call.command} ${call.args.join(" ")}`)).toEqual([ + "nemoclaw e2e-survival status", + ]); + }); +}); + describe("LifecyclePhaseFixture profile dispatch", () => { it("rejects unknown lifecycle profiles", async () => { const runner = new FakeRunner(); diff --git a/test/e2e-scenario/support-tests/e2e-phase-runtime.test.ts b/test/e2e-scenario/support-tests/e2e-phase-runtime.test.ts index f1dd5b44856..e6266ee53a3 100644 --- a/test/e2e-scenario/support-tests/e2e-phase-runtime.test.ts +++ b/test/e2e-scenario/support-tests/e2e-phase-runtime.test.ts @@ -55,7 +55,11 @@ class FakeRunner implements CommandRunner { command: TrustedShellCommand, options?: ShellProbeRunOptions, ): Promise { - this.calls.push({ command: command.command, args: [...command.args], options }); + this.calls.push({ + command: command.command, + args: [...command.args], + options, + }); const response = this.responses.shift(); if (!response) { throw new Error( @@ -194,6 +198,33 @@ describe("runtime phase fixture", () => { expect(call?.options?.artifactName).toBe("custom-chat"); }); + it("retries inference.local PONG chat completions and accepts reasoning content", async () => { + const runner = new FakeRunner(); + runner.enqueue( + shellResult(0, JSON.stringify({ choices: [{ message: { content: "not yet" } }] })), + ); + runner.enqueue( + shellResult( + 0, + JSON.stringify({ + choices: [{ message: { reasoning_content: "PONG" } }], + }), + ), + ); + + const result = await fixture(runner).expectInferenceLocalPong(instance(), { + artifactName: "pong-probe", + attempts: 2, + retryDelayMs: 1, + }); + + expect(result.result.stdout).toContain("PONG"); + expect(runner.calls.map((call) => call.options?.artifactName)).toEqual([ + "pong-probe-1", + "pong-probe-2", + ]); + }); + it("accepts configured status codes for auth-proxy and route-health checks", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "403")); @@ -327,7 +358,9 @@ describe("runtime phase fixture", () => { const runner = new FakeRunner(); await expect( - fixture(runner).expectInferenceLocalModels(instance(), { curlMaxTimeSeconds }), + fixture(runner).expectInferenceLocalModels(instance(), { + curlMaxTimeSeconds, + }), ).rejects.toThrow("inference request curlMaxTimeSeconds must be a finite positive number"); expect(runner.calls).toEqual([]); } diff --git a/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts index 6372ca48fb2..5ec6a59db35 100644 --- a/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/support-tests/e2e-phase-state-validation.test.ts @@ -95,7 +95,11 @@ class FakeRunner implements CommandRunner { command: TrustedShellCommand, options?: ShellProbeRunOptions, ): Promise { - this.calls.push({ command: command.command, args: [...command.args], options }); + this.calls.push({ + command: command.command, + args: [...command.args], + options, + }); const response = this.responses.shift(); if (response instanceof Error) { throw response; @@ -472,6 +476,34 @@ describe("state-validation phase fixture", () => { ); }); + it("exposes direct registry and sandbox marker helpers for survival checks", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ + entries: { "e2e-ubuntu-repo-cloud-openclaw": {} }, + }), + }); + + expect(() => fx.expectLocalRegistryContains("e2e-ubuntu-repo-cloud-openclaw")).not.toThrow(); + runner.enqueue(shellResult(0)); + runner.enqueue(shellResult(0, "survived\n")); + runner.enqueue(shellResult(0, "/sandbox/.openclaw/config.json\n")); + + const marker = { + path: "/sandbox/.openclaw/.survival-marker", + value: "survived", + }; + await fx.writeSandboxMarkers(instance(), [marker]); + await fx.expectSandboxMarkers(instance(), [marker]); + await fx.expectSandboxDirectoryPopulated(instance(), "/sandbox/.openclaw"); + + expect(runner.calls.map((call) => call.args.slice(0, 7))).toEqual([ + ["sandbox", "exec", "-n", "e2e-ubuntu-repo-cloud-openclaw", "--", "sh", "-lc"], + ["sandbox", "exec", "-n", "e2e-ubuntu-repo-cloud-openclaw", "--", "sh", "-lc"], + ["sandbox", "exec", "-n", "e2e-ubuntu-repo-cloud-openclaw", "--", "sh", "-lc"], + ]); + }); + it("writes a state-validation phase result artifact on success", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-state-validation-artifacts-")); try { @@ -532,7 +564,11 @@ describe("state-validation host-side probes", () => { const runner = new FakeRunner(); const fx = fixture(runner, { readRegistry: () => ({ - entries: { "e2e-ubuntu-repo-cloud-openclaw": { name: "e2e-ubuntu-repo-cloud-openclaw" } }, + entries: { + "e2e-ubuntu-repo-cloud-openclaw": { + name: "e2e-ubuntu-repo-cloud-openclaw", + }, + }, }), });