test(e2e): migrate onboard repair flow to Vitest

.github/workflows/e2e-vitest-scenarios.yaml

@@ -2048,6 +2048,68 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+  onboard-repair-vitest:
+    needs: generate-matrix
+    if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',onboard-repair-vitest,') || contains(format(',{0},', inputs.scenarios), ',onboard-repair,') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    env:
+      FREE_STANDING_VITEST_JOB: "1"
+      FREE_STANDING_SCENARIO_ID: "onboard-repair"
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-repair
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
+      NEMOCLAW_RUN_E2E_SCENARIOS: "1"
+      NEMOCLAW_NON_INTERACTIVE: "1"
+      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+      NEMOCLAW_SANDBOX_NAME: "e2e-repair-vitest"
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Build CLI
+        run: npm run build:cli
+
+      - name: Install OpenShell CLI
+        run: bash scripts/install-openshell.sh
+
+      - name: Run onboard-repair live Vitest test
+        run: |
+          set -euo pipefail
+          export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
+          if command -v openshell >/dev/null 2>&1; then
+            OPENSHELL_BIN="$(command -v openshell)"
+          elif [ -x "$HOME/.local/bin/openshell" ]; then
+            OPENSHELL_BIN="$HOME/.local/bin/openshell"
+          else
+            echo "::error::OpenShell CLI not found after install"
+            exit 1
+          fi
+          export OPENSHELL_BIN
+          "$OPENSHELL_BIN" --version
+          npx vitest run --project e2e-scenarios-live \
+            test/e2e-scenario/live/onboard-repair.test.ts \
+            --silent=false --reporter=default
+
+      - name: Upload onboard-repair artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-vitest-scenarios-onboard-repair
+          path: e2e-artifacts/vitest/onboard-repair/
+          include-hidden-files: false
+          if-no-files-found: ignore
+          retention-days: 14
+
   token-rotation-vitest:
     needs: generate-matrix
     if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') || contains(format(',{0},', inputs.scenarios), ',token-rotation,') }}
@@ -3253,6 +3315,7 @@ jobs:
         full-e2e-vitest,
         cloud-onboard-vitest,
         gpu-double-onboard-vitest,
+        onboard-repair-vitest,
         onboard-resume-vitest,
         model-router-provider-routed-inference-vitest,
         sandbox-survival-vitest,

test/e2e-scenario/live/onboard-repair.test.ts

@@ -0,0 +1,236 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/** Live Vitest replacement for test/e2e/test-onboard-repair.sh. */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import { type HostCliClient } from "../fixtures/clients/host.ts";
+import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts";
+import { expect, test } from "../fixtures/e2e-test.ts";
+import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts";
+import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
+import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
+
+const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
+const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js");
+const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-repair-vitest";
+const OTHER_SANDBOX_NAME = process.env.NEMOCLAW_OTHER_SANDBOX_NAME ?? "e2e-repair-other-vitest";
+const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json");
+const LIVE_TIMEOUT_MS = 70 * 60_000;
+const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip;
+
+validateSandboxName(SANDBOX_NAME);
+validateSandboxName(OTHER_SANDBOX_NAME);
+process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT;
+
+function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
+  return {
+    ...buildAvailabilityProbeEnv(),
+    PATH: `${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`,
+    NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
+    NEMOCLAW_NON_INTERACTIVE: "1",
+    OPENSHELL_GATEWAY: "nemoclaw",
+    ...extra,
+  };
+}
+
+function resultText(result: Pick<ShellProbeResult, "stdout" | "stderr">): string {
+  return [result.stdout, result.stderr].filter(Boolean).join("\n");
+}
+
+async function nemoclaw(
+  host: HostCliClient,
+  args: string[],
+  artifactName: string,
+  extraEnv: NodeJS.ProcessEnv = {},
+  timeoutMs = 20 * 60_000,
+): Promise<ShellProbeResult> {
+  return await host.command(process.execPath, [CLI_ENTRYPOINT, ...args], {
+    artifactName,
+    env: env(extraEnv),
+    timeoutMs,
+  });
+}
+
+function onboardEnv(sandboxName: string, fakeBaseUrl: string, extra: NodeJS.ProcessEnv = {}) {
+  return env({
+    COMPATIBLE_API_KEY: "dummy",
+    NEMOCLAW_ENDPOINT_URL: fakeBaseUrl,
+    NEMOCLAW_MODEL: "test-model",
+    NEMOCLAW_PROVIDER: "custom",
+    NEMOCLAW_SANDBOX_NAME: sandboxName,
+    ...extra,
+  });
+}
+
+async function cleanup(host: HostCliClient, sandbox: SandboxClient): Promise<void> {
+  for (const name of [SANDBOX_NAME, OTHER_SANDBOX_NAME]) {
+    await nemoclaw(host, [name, "destroy", "--yes"], `cleanup-destroy-${name}`).catch(
+      () => undefined,
+    );
+    await sandbox
+      .openshell(["sandbox", "delete", name], {
+        artifactName: `cleanup-openshell-delete-${name}`,
+        env: env(),
+        timeoutMs: 60_000,
+      })
+      .catch(() => undefined);
+  }
+  await sandbox
+    .openshell(["forward", "stop", "18789"], {
+      artifactName: "cleanup-forward-stop-18789",
+      env: env(),
+      timeoutMs: 30_000,
+    })
+    .catch(() => undefined);
+  await sandbox
+    .openshell(["gateway", "destroy", "-g", "nemoclaw"], {
+      artifactName: "cleanup-gateway-destroy",
+      env: env(),
+      timeoutMs: 60_000,
+    })
+    .catch(() => undefined);
+  fs.rmSync(SESSION_FILE, { force: true });
+}
+
+async function waitSandboxAbsent(sandbox: SandboxClient, name: string): Promise<void> {
+  for (let attempt = 1; attempt <= 60; attempt += 1) {
+    const result = await sandbox.openshell(["sandbox", "get", name], {
+      artifactName: `wait-${name}-absent-${attempt}`,
+      env: env(),
+      timeoutMs: 30_000,
+    });
+    if (result.exitCode !== 0 && /NotFound|not found/i.test(resultText(result))) return;
+    await new Promise((resolve) => setTimeout(resolve, 1_000));
+  }
+  throw new Error(`${name} still exists after forced deletion`);
+}
+
+liveTest(
+  "onboard repair resumes missing sandbox and rejects conflicting resume inputs",
+  { timeout: LIVE_TIMEOUT_MS },
+  async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, skip }) => {
+    await artifacts.writeJson("scenario.json", {
+      id: "onboard-repair",
+      legacySource: "test/e2e/test-onboard-repair.sh",
+      sandboxName: SANDBOX_NAME,
+      otherSandboxName: OTHER_SANDBOX_NAME,
+      contracts: [
+        "forced policy-step failure leaves a resumable session",
+        "resume recreates a recorded sandbox that was removed underneath it",
+        "resume rejects a different requested sandbox name",
+        "resume rejects provider/model overrides that conflict with recorded state",
+      ],
+    });
+
+    const docker = await host.command("docker", ["info"], {
+      artifactName: "phase-0-docker-info",
+      env: env(),
+      timeoutMs: 30_000,
+    });
+    if (docker.exitCode !== 0) {
+      if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker));
+      skip(`Docker is required: ${resultText(docker)}`);
+    }
+
+    const fake = await startFakeOpenAiCompatibleServer();
+    cleanupRegistry.add("close fake OpenAI-compatible endpoint", async () => fake.close());
+    cleanupRegistry.add("remove repair sandboxes", () => cleanup(host, sandbox));
+    await cleanup(host, sandbox);
+
+    const first = await nemoclaw(
+      host,
+      ["onboard", "--non-interactive"],
+      "phase-1-forced-failure",
+      onboardEnv(SANDBOX_NAME, fake.baseUrl, {
+        NEMOCLAW_E2E_FAILURE_INJECTION: "1",
+        NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies",
+        NEMOCLAW_POLICY_MODE: "suggested",
+        NEMOCLAW_RECREATE_SANDBOX: "1",
+      }),
+    );
+    expect(first.exitCode, resultText(first)).toBe(1);
+    expect(resultText(first)).toContain("Forced onboarding failure at step 'policies'");
+    expect(fs.existsSync(SESSION_FILE)).toBe(true);
+
+    const sandboxAfterFailure = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], {
+      artifactName: "phase-1-sandbox-get-after-failure",
+      env: env(),
+      timeoutMs: 60_000,
+    });
+    expect(sandboxAfterFailure.exitCode, resultText(sandboxAfterFailure)).toBe(0);
+
+    await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], {
+      artifactName: "phase-2-delete-recorded-sandbox",
+      env: env(),
+      timeoutMs: 60_000,
+    });
+    await waitSandboxAbsent(sandbox, SANDBOX_NAME);
+
+    const repair = await nemoclaw(
+      host,
+      ["onboard", "--resume", "--non-interactive"],
+      "phase-2-resume-repair",
+      onboardEnv(SANDBOX_NAME, fake.baseUrl, {
+        NEMOCLAW_POLICY_MODE: "skip",
+      }),
+    );
+    expect(repair.exitCode, resultText(repair)).toBe(0);
+    expect(resultText(repair)).toContain("[resume] Skipping preflight (cached)");
+    expect(resultText(repair)).toContain("Recorded sandbox state is unavailable; recreating it");
+    expect(resultText(repair)).toContain("Creating sandbox");
+
+    const status = await nemoclaw(host, [SANDBOX_NAME, "status"], "phase-2-status-after-repair");
+    expect(status.exitCode, resultText(status)).toBe(0);
+
+    const reinject = await nemoclaw(
+      host,
+      ["onboard", "--non-interactive"],
+      "phase-3-reinject-failure",
+      onboardEnv(SANDBOX_NAME, fake.baseUrl, {
+        NEMOCLAW_E2E_FAILURE_INJECTION: "1",
+        NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies",
+        NEMOCLAW_POLICY_MODE: "suggested",
+        NEMOCLAW_RECREATE_SANDBOX: "1",
+      }),
+    );
+    expect(reinject.exitCode, resultText(reinject)).toBe(1);
+
+    const sandboxConflict = await nemoclaw(
+      host,
+      ["onboard", "--resume", "--non-interactive"],
+      "phase-4-conflicting-sandbox",
+      onboardEnv(OTHER_SANDBOX_NAME, fake.baseUrl, {
+        NEMOCLAW_POLICY_MODE: "skip",
+      }),
+    );
+    expect(sandboxConflict.exitCode, resultText(sandboxConflict)).toBe(1);
+    expect(resultText(sandboxConflict)).toContain(
+      `Resumable state belongs to sandbox '${SANDBOX_NAME}', not '${OTHER_SANDBOX_NAME}'`,
+    );
+
+    const providerConflict = await nemoclaw(
+      host,
+      ["onboard", "--resume", "--non-interactive"],
+      "phase-5-conflicting-provider-model",
+      onboardEnv(SANDBOX_NAME, fake.baseUrl, {
+        NEMOCLAW_MODEL: "gpt-5.4",
+        NEMOCLAW_POLICY_MODE: "skip",
+        NEMOCLAW_PROVIDER: "openai",
+      }),
+    );
+    expect(providerConflict.exitCode, resultText(providerConflict)).toBe(1);
+    expect(resultText(providerConflict)).toMatch(
+      /Resumable state recorded provider '.*', not '.*'\./,
+    );
+    expect(resultText(providerConflict)).toContain("not 'gpt-5.4'");
+
+    await cleanup(host, sandbox);
+    expect(fs.existsSync(SESSION_FILE)).toBe(false);
+    await artifacts.writeJson("scenario-result.json", { id: "onboard-repair", status: "passed" });
+  },
+);