diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 40e956a68a4..3a5d87d00ea 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -2048,6 +2048,68 @@ jobs: if-no-files-found: ignore retention-days: 14 + onboard-repair-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',onboard-repair-vitest,') || contains(format(',{0},', inputs.scenarios), ',onboard-repair,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "onboard-repair" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-repair + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-repair-vitest" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run onboard-repair live Vitest test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/onboard-repair.test.ts \ + --silent=false --reporter=default + + - name: Upload onboard-repair artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-onboard-repair + path: e2e-artifacts/vitest/onboard-repair/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + token-rotation-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') || contains(format(',{0},', inputs.scenarios), ',token-rotation,') }} @@ -3253,6 +3315,7 @@ jobs: full-e2e-vitest, cloud-onboard-vitest, gpu-double-onboard-vitest, + onboard-repair-vitest, onboard-resume-vitest, model-router-provider-routed-inference-vitest, sandbox-survival-vitest, diff --git a/test/e2e-scenario/live/onboard-repair.test.ts b/test/e2e-scenario/live/onboard-repair.test.ts new file mode 100644 index 00000000000..a53f150a6c6 --- /dev/null +++ b/test/e2e-scenario/live/onboard-repair.test.ts @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Live Vitest replacement for test/e2e/test-onboard-repair.sh. */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { type HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-repair-vitest"; +const OTHER_SANDBOX_NAME = process.env.NEMOCLAW_OTHER_SANDBOX_NAME ?? "e2e-repair-other-vitest"; +const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); +const LIVE_TIMEOUT_MS = 70 * 60_000; +const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +validateSandboxName(SANDBOX_NAME); +validateSandboxName(OTHER_SANDBOX_NAME); +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; + +function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + PATH: `${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + OPENSHELL_GATEWAY: "nemoclaw", + ...extra, + }; +} + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +async function nemoclaw( + host: HostCliClient, + args: string[], + artifactName: string, + extraEnv: NodeJS.ProcessEnv = {}, + timeoutMs = 20 * 60_000, +): Promise { + return await host.command(process.execPath, [CLI_ENTRYPOINT, ...args], { + artifactName, + env: env(extraEnv), + timeoutMs, + }); +} + +function onboardEnv(sandboxName: string, fakeBaseUrl: string, extra: NodeJS.ProcessEnv = {}) { + return env({ + COMPATIBLE_API_KEY: "dummy", + NEMOCLAW_ENDPOINT_URL: fakeBaseUrl, + NEMOCLAW_MODEL: "test-model", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_SANDBOX_NAME: sandboxName, + ...extra, + }); +} + +async function cleanup(host: HostCliClient, sandbox: SandboxClient): Promise { + for (const name of [SANDBOX_NAME, OTHER_SANDBOX_NAME]) { + await nemoclaw(host, [name, "destroy", "--yes"], `cleanup-destroy-${name}`).catch( + () => undefined, + ); + await sandbox + .openshell(["sandbox", "delete", name], { + artifactName: `cleanup-openshell-delete-${name}`, + env: env(), + timeoutMs: 60_000, + }) + .catch(() => undefined); + } + await sandbox + .openshell(["forward", "stop", "18789"], { + artifactName: "cleanup-forward-stop-18789", + env: env(), + timeoutMs: 30_000, + }) + .catch(() => undefined); + await sandbox + .openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-gateway-destroy", + env: env(), + timeoutMs: 60_000, + }) + .catch(() => undefined); + fs.rmSync(SESSION_FILE, { force: true }); +} + +async function waitSandboxAbsent(sandbox: SandboxClient, name: string): Promise { + for (let attempt = 1; attempt <= 60; attempt += 1) { + const result = await sandbox.openshell(["sandbox", "get", name], { + artifactName: `wait-${name}-absent-${attempt}`, + env: env(), + timeoutMs: 30_000, + }); + if (result.exitCode !== 0 && /NotFound|not found/i.test(resultText(result))) return; + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + throw new Error(`${name} still exists after forced deletion`); +} + +liveTest( + "onboard repair resumes missing sandbox and rejects conflicting resume inputs", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, skip }) => { + await artifacts.writeJson("scenario.json", { + id: "onboard-repair", + legacySource: "test/e2e/test-onboard-repair.sh", + sandboxName: SANDBOX_NAME, + otherSandboxName: OTHER_SANDBOX_NAME, + contracts: [ + "forced policy-step failure leaves a resumable session", + "resume recreates a recorded sandbox that was removed underneath it", + "resume rejects a different requested sandbox name", + "resume rejects provider/model overrides that conflict with recorded state", + ], + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: env(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); + skip(`Docker is required: ${resultText(docker)}`); + } + + const fake = await startFakeOpenAiCompatibleServer(); + cleanupRegistry.add("close fake OpenAI-compatible endpoint", async () => fake.close()); + cleanupRegistry.add("remove repair sandboxes", () => cleanup(host, sandbox)); + await cleanup(host, sandbox); + + const first = await nemoclaw( + host, + ["onboard", "--non-interactive"], + "phase-1-forced-failure", + onboardEnv(SANDBOX_NAME, fake.baseUrl, { + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_RECREATE_SANDBOX: "1", + }), + ); + expect(first.exitCode, resultText(first)).toBe(1); + expect(resultText(first)).toContain("Forced onboarding failure at step 'policies'"); + expect(fs.existsSync(SESSION_FILE)).toBe(true); + + const sandboxAfterFailure = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-1-sandbox-get-after-failure", + env: env(), + timeoutMs: 60_000, + }); + expect(sandboxAfterFailure.exitCode, resultText(sandboxAfterFailure)).toBe(0); + + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "phase-2-delete-recorded-sandbox", + env: env(), + timeoutMs: 60_000, + }); + await waitSandboxAbsent(sandbox, SANDBOX_NAME); + + const repair = await nemoclaw( + host, + ["onboard", "--resume", "--non-interactive"], + "phase-2-resume-repair", + onboardEnv(SANDBOX_NAME, fake.baseUrl, { + NEMOCLAW_POLICY_MODE: "skip", + }), + ); + expect(repair.exitCode, resultText(repair)).toBe(0); + expect(resultText(repair)).toContain("[resume] Skipping preflight (cached)"); + expect(resultText(repair)).toContain("Recorded sandbox state is unavailable; recreating it"); + expect(resultText(repair)).toContain("Creating sandbox"); + + const status = await nemoclaw(host, [SANDBOX_NAME, "status"], "phase-2-status-after-repair"); + expect(status.exitCode, resultText(status)).toBe(0); + + const reinject = await nemoclaw( + host, + ["onboard", "--non-interactive"], + "phase-3-reinject-failure", + onboardEnv(SANDBOX_NAME, fake.baseUrl, { + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_RECREATE_SANDBOX: "1", + }), + ); + expect(reinject.exitCode, resultText(reinject)).toBe(1); + + const sandboxConflict = await nemoclaw( + host, + ["onboard", "--resume", "--non-interactive"], + "phase-4-conflicting-sandbox", + onboardEnv(OTHER_SANDBOX_NAME, fake.baseUrl, { + NEMOCLAW_POLICY_MODE: "skip", + }), + ); + expect(sandboxConflict.exitCode, resultText(sandboxConflict)).toBe(1); + expect(resultText(sandboxConflict)).toContain( + `Resumable state belongs to sandbox '${SANDBOX_NAME}', not '${OTHER_SANDBOX_NAME}'`, + ); + + const providerConflict = await nemoclaw( + host, + ["onboard", "--resume", "--non-interactive"], + "phase-5-conflicting-provider-model", + onboardEnv(SANDBOX_NAME, fake.baseUrl, { + NEMOCLAW_MODEL: "gpt-5.4", + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_PROVIDER: "openai", + }), + ); + expect(providerConflict.exitCode, resultText(providerConflict)).toBe(1); + expect(resultText(providerConflict)).toMatch( + /Resumable state recorded provider '.*', not '.*'\./, + ); + expect(resultText(providerConflict)).toContain("not 'gpt-5.4'"); + + await cleanup(host, sandbox); + expect(fs.existsSync(SESSION_FILE)).toBe(false); + await artifacts.writeJson("scenario-result.json", { id: "onboard-repair", status: "passed" }); + }, +);