diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index fb84fb57dee..b1f59b4a65b 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1854,6 +1854,71 @@ jobs: if-no-files-found: ignore retention-days: 14 + full-e2e-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',full-e2e-vitest,') || contains(format(',{0},', inputs.scenarios), ',full-e2e,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "full-e2e" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/full-e2e + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-full-vitest" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run full-e2e live Vitest test + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/full-e2e.test.ts \ + --silent=false --reporter=default + + - name: Upload full-e2e artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-full-e2e + path: e2e-artifacts/vitest/full-e2e/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + token-rotation-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') || contains(format(',{0},', inputs.scenarios), ',token-rotation,') }} @@ -3056,6 +3121,7 @@ jobs: messaging-providers-vitest, launchable-smoke-vitest, double-onboard-vitest, + full-e2e-vitest, onboard-resume-vitest, model-router-provider-routed-inference-vitest, sandbox-survival-vitest, diff --git a/test/e2e-scenario/live/full-e2e.test.ts b/test/e2e-scenario/live/full-e2e.test.ts new file mode 100644 index 00000000000..6eefd0d818d --- /dev/null +++ b/test/e2e-scenario/live/full-e2e.test.ts @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Live Vitest replacement for test/e2e/test-full-e2e.sh. */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { type HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full-vitest"; +const LIVE_TIMEOUT_MS = 50 * 60_000; +const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; +validateSandboxName(SANDBOX_NAME); + +function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + PATH: `${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: "nemoclaw", + ...extra, + }; +} + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +async function repoNemoclaw( + host: HostCliClient, + args: string[], + artifactName: string, + extraEnv: NodeJS.ProcessEnv = {}, + timeoutMs = 120_000, +): Promise { + return await host.command(process.execPath, [CLI_ENTRYPOINT, ...args], { + artifactName, + env: env(extraEnv), + timeoutMs, + }); +} + +async function cleanup(host: HostCliClient, sandbox: SandboxClient): Promise { + await repoNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], "cleanup-nemoclaw-destroy").catch( + () => undefined, + ); + await sandbox + .openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: env(), + timeoutMs: 60_000, + }) + .catch(() => undefined); + await sandbox + .openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: env(), + timeoutMs: 60_000, + }) + .catch(() => undefined); +} + +function chatRequest(model: string): string { + return JSON.stringify({ + model, + messages: [ + { + role: "user", + content: "What is 6 multiplied by 7? Reply with only the integer, no extra words.", + }, + ], + max_tokens: 100, + }); +} + +function parseReplyCommand(): string { + return String.raw`python3 -c 'import json,sys; d=json.load(sys.stdin); m=d["choices"][0]["message"]; print((m.get("content") or m.get("reasoning_content") or "").strip())'`; +} + +liveTest( + "full e2e: install, onboard, inference, cli operations, and cleanup", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, secrets, skip }) => { + const hosted = requireHostedInferenceConfig(secrets); + const redactionValues = [hosted.apiKey]; + await artifacts.writeJson("scenario.json", { + id: "full-e2e", + legacySource: "test/e2e/test-full-e2e.sh", + sandboxName: SANDBOX_NAME, + endpointUrl: hosted.endpointUrl, + model: hosted.model, + contracts: [ + "install.sh --non-interactive completes onboarding", + "nemoclaw and openshell are installed and usable", + "sandbox appears in list/status and has policy/inference configuration", + "direct hosted inference and sandbox inference.local both respond", + "nemoclaw logs produces output and cleanup removes registry state", + ], + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: env(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); + skip(`Docker is required: ${resultText(docker)}`); + } + + cleanupRegistry.add("remove full-e2e sandbox", () => cleanup(host, sandbox)); + await cleanup(host, sandbox); + + const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { + artifactName: "phase-1-install-sh", + cwd: REPO_ROOT, + env: env({ ...hosted.env, NVIDIA_INFERENCE_API_KEY: hosted.apiKey }), + redactionValues, + timeoutMs: 25 * 60_000, + }); + expect(install.exitCode, resultText(install)).toBe(0); + + const pathProbe = await host.command( + "bash", + [ + "-lc", + 'export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"; command -v nemoclaw; command -v openshell; nemoclaw --help >/dev/null', + ], + { artifactName: "phase-2-path-probe", env: env(), timeoutMs: 60_000 }, + ); + expect(pathProbe.exitCode, resultText(pathProbe)).toBe(0); + expect(pathProbe.stdout).toContain("nemoclaw"); + expect(pathProbe.stdout).toContain("openshell"); + + const list = await repoNemoclaw(host, ["list"], "phase-3-nemoclaw-list"); + expect(list.exitCode, resultText(list)).toBe(0); + expect(list.stdout).toContain(SANDBOX_NAME); + const status = await repoNemoclaw(host, [SANDBOX_NAME, "status"], "phase-3-nemoclaw-status"); + expect(status.exitCode, resultText(status)).toBe(0); + + const inference = await sandbox.openshell(["inference", "get"], { + artifactName: "phase-3-openshell-inference-get", + env: env(), + timeoutMs: 60_000, + }); + expect(inference.exitCode, resultText(inference)).toBe(0); + expect(resultText(inference)).toContain(hosted.model); + + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-3-openshell-policy-get", + env: env(), + timeoutMs: 60_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toMatch(/network_policies|egress/i); + + const direct = await host.command( + "curl", + [ + "-fsS", + "--max-time", + "60", + "-H", + `Authorization: Bearer ${hosted.apiKey}`, + `${hosted.endpointUrl}/models`, + ], + { + artifactName: "phase-4-direct-hosted-inference-models", + env: env(), + redactionValues, + timeoutMs: 90_000, + }, + ); + expect(direct.exitCode, resultText(direct)).toBe(0); + expect(resultText(direct)).toContain("data"); + + const sandboxInference = await sandbox.exec( + SANDBOX_NAME, + [ + "sh", + "-lc", + `curl -fsS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${chatRequest(hosted.model)}' | ${parseReplyCommand()}`, + ], + { + artifactName: "phase-4-sandbox-inference-local", + env: env(), + redactionValues, + timeoutMs: 120_000, + }, + ); + expect(sandboxInference.exitCode, resultText(sandboxInference)).toBe(0); + expect(sandboxInference.stdout).toMatch(/(^|[^0-9])42([^0-9]|$)/); + + const logs = await repoNemoclaw( + host, + [SANDBOX_NAME, "logs"], + "phase-5-nemoclaw-logs", + {}, + 90_000, + ); + expect(logs.exitCode, resultText(logs)).toBe(0); + expect(resultText(logs).trim().length, resultText(logs)).toBeGreaterThan(0); + + await cleanup(host, sandbox); + const registry = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); + const registryText = fs.existsSync(registry) ? fs.readFileSync(registry, "utf8") : ""; + expect(registryText).not.toContain(SANDBOX_NAME); + + await artifacts.writeJson("scenario-result.json", { id: "full-e2e", status: "passed" }); + }, +);