test(e2e): migrate test-sandbox-survival.sh to vitest

.github/workflows/e2e-vitest-scenarios.yaml

@@ -40,7 +40,7 @@ jobs:
           SCENARIOS: ${{ inputs.scenarios }}
         run: |
           set -euo pipefail
-          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest"
+          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,sandbox-survival-vitest"
           if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then
             echo "::error::Use either scenarios or jobs, not both." >&2
             exit 1
@@ -93,12 +93,12 @@ jobs:
           SCENARIOS: ${{ inputs.scenarios }}
         run: |
           set -euo pipefail
-          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest"
+          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,sandbox-survival-vitest"
           args=(--emit-live-matrix)
           matrix=""
           hermes_selected=false
           registry_scenarios=()
-          free_standing_scenarios=(openshell-version-pin onboard-negative-paths skill-agent inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference)
+          free_standing_scenarios=(openshell-version-pin onboard-negative-paths skill-agent inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference sandbox-survival)
           is_free_standing_scenario() {
             local id="$1"
             local known
@@ -652,7 +652,6 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
-
   credential-migration-vitest:
     needs: validate-jobs
     if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',credential-migration-vitest,') }}
@@ -729,8 +728,6 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
-
-
   runtime-overrides-vitest:
     needs: [validate-jobs, generate-matrix]
     if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',runtime-overrides-vitest,') || contains(format(',{0},', inputs.scenarios), ',runtime-overrides,') }}
@@ -1311,6 +1308,94 @@ jobs:
           docker logout docker.io || true
           rm -rf "${DOCKER_CONFIG}"
 
+  sandbox-survival-vitest:
+    needs: [validate-jobs, generate-matrix]
+    if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-survival-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-survival,') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-sandbox-survival
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-survival
+      NEMOCLAW_RUN_E2E_SCENARIOS: "1"
+      NEMOCLAW_NON_INTERACTIVE: "1"
+      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+      NEMOCLAW_SANDBOX_NAME: "e2e-survival"
+      OPENSHELL_GATEWAY: "nemoclaw"
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Authenticate to Docker Hub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then
+            echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls."
+            exit 0
+          fi
+          mkdir -p "${DOCKER_CONFIG}"
+          chmod 700 "${DOCKER_CONFIG}"
+          login_succeeded=0
+          for attempt in 1 2 3; do
+            if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
+              login_succeeded=1
+              break
+            fi
+            if [[ "$attempt" -lt 3 ]]; then
+              echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
+              sleep 5
+            fi
+          done
+          if [[ "$login_succeeded" -ne 1 ]]; then
+            echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls."
+          fi
+
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Build CLI
+        run: npm run build:cli
+
+      - name: Run sandbox survival live test
+        # Migrated from test/e2e/test-sandbox-survival.sh. This intentionally
+        # runs install.sh from Vitest to preserve installer/OpenShell/onboard
+        # fidelity before exercising gateway restart, state survival, and live
+        # inference.local before and after restart.
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+        run: |
+          set -euo pipefail
+          npx vitest run --project e2e-scenarios-live \
+            test/e2e-scenario/live/sandbox-survival.test.ts \
+            --silent=false --reporter=default
+
+      - name: Upload sandbox survival artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-vitest-scenarios-sandbox-survival
+          path: e2e-artifacts/vitest/sandbox-survival/
+          include-hidden-files: false
+          if-no-files-found: ignore
+          retention-days: 14
+
+      - name: Clean up Docker auth
+        if: always()
+        run: |
+          set -euo pipefail
+          docker logout docker.io || true
+          rm -rf "${DOCKER_CONFIG}"
+
   # Focused coverage slice for the #2603/#3145 OpenClaw websocket
   # protocol/history contract. The retained legacy bash lane remains the
   # source for full closeout until a later PR proves replacement and deletes it.
@@ -1506,6 +1591,7 @@ jobs:
         launchable-smoke-vitest,
         double-onboard-vitest,
         model-router-provider-routed-inference-vitest,
+        sandbox-survival-vitest,
         openclaw-tui-chat-correlation-vitest,
         gateway-guard-recovery,
         issue-4434-tui-unreachable-inference-vitest,

test/e2e-scenario/fixtures/clients/command.ts

@@ -11,6 +11,18 @@ export interface CommandRunner {
   run(command: TrustedShellCommand, options?: ShellProbeRunOptions): Promise<ShellProbeResult>;
 }
 
+export function resultText(result: Pick<ShellProbeResult, "stdout" | "stderr">): string {
+  return [result.stdout, result.stderr].filter(Boolean).join("\n");
+}
+
+export function outputContainsSandbox(
+  result: Pick<ShellProbeResult, "stdout" | "stderr">,
+  sandboxName: string,
+): boolean {
+  const escaped = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return new RegExp(`(^|\\s)${escaped}(\\s|$)`, "m").test(resultText(result));
+}
+
 export function assertExitZero(result: ShellProbeResult, label: string): void {
   if (result.exitCode === 0) return;
   const fallback = result.signal

test/e2e-scenario/fixtures/clients/gateway.ts

@@ -42,6 +42,14 @@ const DEFAULT_GUARD_MARKERS: ReadonlyArray<string> = [
 
 /** Default gateway log path inside the sandbox. */
 const GATEWAY_LOG_PATH = "/tmp/gateway.log";
+const DOCKER_DRIVER_GATEWAY_PID_RELPATH = [
+  ".local",
+  "state",
+  "nemoclaw",
+  "openshell-docker-gateway",
+  "openshell-gateway.pid",
+] as const;
+const DEFAULT_GATEWAY_CONTAINER = "openshell-cluster-nemoclaw";
 
 export interface ExpectGuardChainOptions extends ShellProbeRunOptions {
   /** Markers required in `/tmp/nemoclaw-proxy-env.sh`. Defaults to safety-net + ciao. */
@@ -60,6 +68,11 @@ export interface ExpectPidStableOptions extends ShellProbeRunOptions {
   pollIntervalSeconds?: number;
 }
 
+export interface HostGatewayRuntime {
+  kind: "pid" | "container";
+  id: string;
+}
+
 export class GatewayClient {
   private readonly host: HostCliClient;
   private readonly sandbox: SandboxClient;
@@ -82,6 +95,74 @@ export class GatewayClient {
     return result;
   }
 
+  async resolveHostRuntime(): Promise<HostGatewayRuntime | null> {
+    const pid = await this.host.command(
+      "sh",
+      [
+        "-lc",
+        `pid_file=\"$HOME/${DOCKER_DRIVER_GATEWAY_PID_RELPATH.join("/")}\"; ` +
+          `if [ -f \"$pid_file\" ]; then ` +
+          `pid=\"$(tr -d '[:space:]' <\"$pid_file\" 2>/dev/null || true)\"; ` +
+          `if [ -n \"$pid\" ] && kill -0 \"$pid\" 2>/dev/null; then printf '%s\\n' \"$pid\"; exit 0; fi; ` +
+          `fi; exit 1`,
+      ],
+      {
+        artifactName: "gateway-runtime-pid-probe",
+        env: probeEnv(),
+        timeoutMs: 15_000,
+      },
+    );
+    if (pid.exitCode === 0 && pid.stdout.trim()) {
+      return { kind: "pid", id: pid.stdout.trim() };
+    }
+
+    const container = await this.host.command(
+      "docker",
+      ["ps", "-qf", `name=${DEFAULT_GATEWAY_CONTAINER}`],
+      {
+        artifactName: "gateway-runtime-container-probe",
+        env: probeEnv(),
+        timeoutMs: 15_000,
+      },
+    );
+    const id = container.stdout.trim().split(/\r?\n/).find(Boolean);
+    return id ? { kind: "container", id } : null;
+  }
+
+  async expectHostRuntimeStopped(options: ShellProbeRunOptions = {}): Promise<void> {
+    const runtime = await this.resolveHostRuntime();
+    if (runtime) {
+      throw new Error(
+        `gateway runtime still appears to be running after stop: ${runtime.kind}:${runtime.id}`,
+      );
+    }
+    if (options.artifactName) {
+      await this.host.command("true", [], {
+        artifactName: options.artifactName,
+        env: probeEnv(),
+        timeoutMs: 5_000,
+      });
+    }
+  }
+
+  async expectOpenshellStatusConnected(
+    gatewayName = "nemoclaw",
+    options: ShellProbeRunOptions = {},
+  ): Promise<ShellProbeResult> {
+    const result = await this.host.command("openshell", ["status"], {
+      artifactName: `openshell-status-${gatewayName}`,
+      env: probeEnv(),
+      timeoutMs: 30_000,
+      ...options,
+    });
+    assertExitZero(result, "openshell status");
+    const text = `${result.stdout}\n${result.stderr}`;
+    if (!/connected/i.test(text) || !new RegExp(gatewayName, "i").test(text)) {
+      throw new Error(`openshell status did not report connected gateway '${gatewayName}'.`);
+    }
+    return result;
+  }
+
   // ─── Guard-chain recovery probes (#2478, #2701) ────────────────────
 
   /**

test/e2e-scenario/fixtures/clients/host.ts

@@ -4,7 +4,13 @@
 import { buildAvailabilityProbeEnv } from "../availability-env.ts";
 import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts";
 import { trustedShellCommand } from "../shell-probe.ts";
-import { artifactLabel, assertExitZero, type CommandRunner } from "./command.ts";
+import {
+  artifactLabel,
+  assertExitZero,
+  outputContainsSandbox,
+  resultText,
+  type CommandRunner,
+} from "./command.ts";
 
 export interface HostClientOptions {
   cliPath?: string;
@@ -60,4 +66,70 @@ export class HostCliClient {
     assertExitZero(result, "nemoclaw --version");
     return result;
   }
+
+  async expectListed(
+    sandboxName: string,
+    options: ShellProbeRunOptions = {},
+  ): Promise<ShellProbeResult> {
+    const result = await this.nemoclaw(["list"], {
+      artifactName: `nemoclaw-list-${artifactLabel(sandboxName)}`,
+      env: buildAvailabilityProbeEnv(),
+      ...options,
+    });
+    assertExitZero(result, "nemoclaw list");
+    if (!outputContainsSandbox(result, sandboxName)) {
+      throw new Error(`nemoclaw list did not include '${sandboxName}': ${resultText(result)}`);
+    }
+    return result;
+  }
+
+  async expectStatus(
+    sandboxName: string,
+    options: ShellProbeRunOptions = {},
+  ): Promise<ShellProbeResult> {
+    const result = await this.nemoclaw([sandboxName, "status"], {
+      artifactName: `nemoclaw-status-${artifactLabel(sandboxName)}`,
+      env: buildAvailabilityProbeEnv(),
+      ...options,
+    });
+    assertExitZero(result, `nemoclaw ${sandboxName} status`);
+    return result;
+  }
+
+  async destroySandbox(
+    sandboxName: string,
+    options: ShellProbeRunOptions = {},
+  ): Promise<ShellProbeResult> {
+    return await this.nemoclaw([sandboxName, "destroy", "--yes"], {
+      artifactName: `destroy-sandbox-${artifactLabel(sandboxName)}`,
+      env: buildAvailabilityProbeEnv(),
+      timeoutMs: 15 * 60_000,
+      ...options,
+    });
+  }
+
+  async cleanupSandbox(sandboxName: string, options: ShellProbeRunOptions = {}): Promise<void> {
+    const result = await this.destroySandbox(sandboxName, options);
+    if (result.exitCode === 0) return;
+    const text = resultText(result);
+    if (
+      /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/i.test(
+        text,
+      )
+    ) {
+      return;
+    }
+    assertExitZero(result, `cleanup destroy sandbox ${sandboxName}`);
+  }
+
+  async bestEffortCleanupSandbox(
+    sandboxName: string,
+    options: ShellProbeRunOptions = {},
+  ): Promise<void> {
+    try {
+      await this.cleanupSandbox(sandboxName, options);
+    } catch {
+      // Best-effort cleanup must not mask the primary setup or assertion failure.
+    }
+  }
 }

test/e2e-scenario/fixtures/clients/index.ts

@@ -1,7 +1,12 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-export { assertExitZero, type CommandRunner } from "./command.ts";
+export {
+  assertExitZero,
+  outputContainsSandbox,
+  resultText,
+  type CommandRunner,
+} from "./command.ts";
 export { GatewayClient } from "./gateway.ts";
 export { HostCliClient } from "./host.ts";
 export {
@@ -13,6 +18,7 @@ export {
 } from "./provider.ts";
 export {
   SandboxClient,
+  sandboxAccessEnv,
   trustedSandboxShellScript,
   type TrustedSandboxShellScript,
   validateSandboxName,