diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 51d8dbcc32a..c8bd277b3a4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -3566,7 +3566,6 @@ jobs: E2E_TARGET_ID: "snapshot-commands" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/snapshot-commands NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-snapshot" @@ -3585,11 +3584,8 @@ jobs: build-cli: "false" - name: Run snapshot commands live test - # Preserves the - # original ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY runner - # contract by driving install.sh and the real snapshot CLI lifecycle. - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + # Drives install.sh and the real snapshot CLI lifecycle while the test + # supplies a sandbox-reachable fake OpenAI-compatible endpoint. run: | set -euo pipefail npx vitest run --project e2e-live \ diff --git a/test/e2e/live/snapshot-commands-helpers.ts b/test/e2e/live/snapshot-commands-helpers.ts new file mode 100644 index 00000000000..9f7f078009e --- /dev/null +++ b/test/e2e/live/snapshot-commands-helpers.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; + +export interface SnapshotInferenceFixture { + apiKey: string; + endpointUrl: string; + model: string; +} + +/** + * Builds the child env for the snapshot-commands live target. + * + * `NEMOCLAW_E2E_USE_HOSTED_INFERENCE` is in the fixture env allowlist, so an + * ambient value is forwarded into the child. `stageHostedInferenceSourceSecretEnv` + * treats that flag alone as sufficient to force hosted-custom staging even when + * `COMPATIBLE_API_KEY` names an explicit endpoint, which would silently route + * this target back at hosted inference. Strip it so the target stays hermetic. + */ +export function buildSnapshotCommandEnv( + sandboxName: string, + inference?: SnapshotInferenceFixture, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: sandboxName, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; + delete env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE; + if (inference) { + Object.assign(env, { + COMPATIBLE_API_KEY: inference.apiKey, + NEMOCLAW_COMPAT_MODEL: inference.model, + NEMOCLAW_ENDPOINT_URL: inference.endpointUrl, + NEMOCLAW_MODEL: inference.model, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + }); + } + return env; +} diff --git a/test/e2e/live/snapshot-commands.test.ts b/test/e2e/live/snapshot-commands.test.ts index 5e526880078..0452a31beef 100644 --- a/test/e2e/live/snapshot-commands.test.ts +++ b/test/e2e/live/snapshot-commands.test.ts @@ -15,11 +15,18 @@ import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; -import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; +import { + buildSnapshotCommandEnv, + type SnapshotInferenceFixture, +} from "./snapshot-commands-helpers.ts"; import { scanSnapshotCredentialLeaks } from "./snapshot-credential-scanner.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-snapshot"; @@ -32,19 +39,11 @@ if (!BACKUP_DIR.startsWith(`${path.resolve(BACKUP_ROOT)}${path.sep}`)) { const MARKER_FILE = "/sandbox/.openclaw/workspace/snapshot-marker.txt"; const SECOND_MARKER = "/sandbox/.openclaw/workspace/snapshot-marker-2.txt"; const LIVE_TIMEOUT_MS = 30 * 60_000; -const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const INFERENCE_API_KEY = "nvapi-snapshot-commands-fixture-credential"; +const INFERENCE_MODEL = "snapshot-commands-model"; -function commandEnv(apiKey?: string): NodeJS.ProcessEnv { - const env: NodeJS.ProcessEnv = { - ...buildAvailabilityProbeEnv(), - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", - }; - if (apiKey) env.NVIDIA_INFERENCE_API_KEY = apiKey; - return env; +function commandEnv(inference?: SnapshotInferenceFixture): NodeJS.ProcessEnv { + return buildSnapshotCommandEnv(SANDBOX_NAME, inference); } async function bestEffort(run: () => Promise): Promise { @@ -107,8 +106,7 @@ function firstSnapshotTimestamp(listOutput: string): string { test("snapshot commands preserve create/list/latest restore/targeted restore/no-leak lifecycle", { timeout: LIVE_TIMEOUT_MS, -}, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); +}, async ({ artifacts, cleanup, host, sandbox, skip }) => { await artifacts.target.declare({ id: "snapshot-commands", boundary: "install.sh + nemoclaw snapshot commands + openshell sandbox exec", @@ -116,6 +114,7 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no- backupDir: BACKUP_DIR, contracts: [ "install.sh onboards a live OpenClaw sandbox", + "onboard authenticates to a hermetic compatible inference endpoint", "snapshot create reports Snapshot v created", "snapshot list shows versioned snapshots and parseable timestamps", "latest snapshot restore recovers latest workspace state", @@ -137,6 +136,24 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no- skip(`Docker is required for snapshot commands E2E: ${resultText(dockerInfo)}`); } + const inference = await startFakeOpenAiCompatibleServer({ + apiKey: INFERENCE_API_KEY, + host: "0.0.0.0", + model: INFERENCE_MODEL, + publicHost: "host.openshell.internal", + requireAuth: true, + requireAuthModels: true, + }); + cleanup.add("close snapshot commands compatible inference fixture", async () => { + await artifacts.writeJson("compatible-inference-requests.json", inference.requests()); + await inference.close(); + }); + const inferenceConfig = { + apiKey: INFERENCE_API_KEY, + endpointUrl: inference.baseUrl, + model: INFERENCE_MODEL, + }; + cleanup.add(`destroy snapshot sandbox ${SANDBOX_NAME}`, () => cleanupSnapshotSandbox(host, sandbox, "cleanup"), ); @@ -144,36 +161,43 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no- await cleanupSnapshotSandbox(host, sandbox, "pre-cleanup"); fs.rmSync(BACKUP_DIR, { recursive: true, force: true }); - let install: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { - install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { - artifactName: - attempt === 1 ? "phase-1-install-nemoclaw" : `phase-1-install-nemoclaw-attempt-${attempt}`, - cwd: REPO_ROOT, - env: commandEnv(apiKey), - redactionValues: [apiKey], - timeoutMs: 20 * 60_000, - }); - if (install.exitCode === 0) break; - if (isTransientProviderValidationFailure(install) && attempt < INSTALL_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); - continue; - } - if (isTransientProviderValidationFailure(install) && process.env.GITHUB_ACTIONS === "true") { - await artifacts.writeJson("transient-provider-validation.skip.json", { - reason: "transient NVIDIA Endpoints validation failure during install.sh onboard", - attempts: INSTALL_ATTEMPTS, - sourceBoundary: "external NVIDIA Endpoints provider availability", - removalCondition: - "remove once CI endpoint validation is stable for a release cycle or covered by a hermetic provider-validation fixture", - }); - skip( - `NVIDIA Endpoints validation hit a transient upstream/rate-limit failure after ${INSTALL_ATTEMPTS} attempts`, - ); - } - break; - } - expect(install?.exitCode, install ? resultText(install) : "install did not run").toBe(0); + const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { + artifactName: "phase-1-install-nemoclaw", + cwd: REPO_ROOT, + env: commandEnv(inferenceConfig), + redactionValues: [INFERENCE_API_KEY], + timeoutMs: 20 * 60_000, + }); + expect(install.exitCode, resultText(install)).toBe(0); + + const authenticatedInference = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `curl -fsS --max-time 60 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${JSON.stringify( + { + model: INFERENCE_MODEL, + messages: [{ role: "user", content: "reply with OK" }], + max_tokens: 8, + }, + )}'`, + ), + { + artifactName: "phase-1-authenticated-inference-post", + env: commandEnv(), + timeoutMs: 90_000, + }, + ); + expect( + authenticatedInference.exitCode, + `${authenticatedInference.stdout}\n${authenticatedInference.stderr}`, + ).toBe(0); + expect(inference.requests()).toContainEqual( + expect.objectContaining({ + auth: "ok", + model: INFERENCE_MODEL, + path: "/v1/chat/completions", + }), + ); const cliProbe = await host.command( "bash", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 68a3560b523..8a48af3502b 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -59,6 +59,13 @@ "test/e2e/support/rlimit-connect-workflow-boundary.test.ts", "test/sandbox-rlimit-hooks.test.ts" ] + }, + { + "live": "test/e2e/live/snapshot-commands.test.ts", + "fast": [ + "test/e2e/support/e2e-workflow.test.ts", + "test/e2e/support/snapshot-commands-helpers.test.ts" + ] } ] } diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 78eb6c2dc13..70669a57716 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -1088,11 +1088,17 @@ jobs: snapshotJob["timeout-minutes"] = 30; snapshotJob.env.DOCKER_CONFIG = "${{ github.workspace }}/.docker-config-shared"; snapshotJob.env.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; + snapshotJob.env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE = "1"; for (const step of snapshotJob.steps) { if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) { step.with = { ...(step.with as Record), "persist-credentials": true }; } if (step.name === "Run snapshot commands live test") { + step.env = { + ...((step.env as Record | undefined) ?? {}), + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; step.run = String(step.run).replace( "test/e2e/live/snapshot-commands.test.ts", "test/e2e/live/registry-targets.test.ts", @@ -1113,8 +1119,11 @@ jobs: expect.arrayContaining([ "snapshot-commands job must keep a 40 minute timeout", "snapshot-commands job must not set DOCKER_CONFIG at job level", + "snapshot-commands job must not enable hosted inference", "snapshot-commands checkout step must set persist-credentials=false", "snapshot-commands job env must not include NVIDIA_INFERENCE_API_KEY", + "snapshot-commands step 'Run snapshot commands live test' env must not include NEMOCLAW_E2E_USE_HOSTED_INFERENCE", + "snapshot-commands step 'Run snapshot commands live test' env must not include NVIDIA_API_KEY", "snapshot-commands upload-e2e-artifacts invocation must not override its contract", "snapshot-commands upload-e2e-artifacts must use the action defaults", "step 'Run snapshot commands live test' run script must include test/e2e/live/snapshot-commands.test.ts", diff --git a/test/e2e/support/snapshot-commands-helpers.test.ts b/test/e2e/support/snapshot-commands-helpers.test.ts new file mode 100644 index 00000000000..fb629828a6a --- /dev/null +++ b/test/e2e/support/snapshot-commands-helpers.test.ts @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { buildSnapshotCommandEnv } from "../live/snapshot-commands-helpers.ts"; + +const HOSTED_FLAG = "NEMOCLAW_E2E_USE_HOSTED_INFERENCE"; +const SANDBOX_NAME = "e2e-snapshot"; + +const INFERENCE = { + apiKey: "nvapi-snapshot-commands-fixture-credential", + endpointUrl: "http://host.openshell.internal:31337/v1", + model: "snapshot-commands-model", +}; + +const HOSTED_CREDENTIAL_ENVS = ["NVIDIA_INFERENCE_API_KEY", "NVIDIA_API_KEY"] as const; + +afterEach(() => { + delete process.env[HOSTED_FLAG]; + for (const name of HOSTED_CREDENTIAL_ENVS) delete process.env[name]; +}); + +describe("snapshot commands live env helper", () => { + it("forwards an ambient hosted-inference flag through the shared probe env", () => { + process.env[HOSTED_FLAG] = "1"; + + // Negative control. The strip below only matters while the flag is + // allowlisted for forwarding; if this ever stops holding, the strip + // assertion would pass vacuously and the hermeticity guard would rot. + expect(buildAvailabilityProbeEnv()[HOSTED_FLAG]).toBe("1"); + }); + + it("strips an ambient hosted-inference flag so the target stays hermetic", () => { + process.env[HOSTED_FLAG] = "1"; + + const env = buildSnapshotCommandEnv(SANDBOX_NAME, INFERENCE); + + expect(env[HOSTED_FLAG]).toBeUndefined(); + expect(Object.hasOwn(env, HOSTED_FLAG)).toBe(false); + }); + + it("strips the hosted-inference flag even when no inference fixture is staged", () => { + process.env[HOSTED_FLAG] = "1"; + + expect(buildSnapshotCommandEnv(SANDBOX_NAME)[HOSTED_FLAG]).toBeUndefined(); + }); + + it("stages the compatible endpoint against the custom provider", () => { + const env = buildSnapshotCommandEnv(SANDBOX_NAME, INFERENCE); + + expect(env).toMatchObject({ + COMPATIBLE_API_KEY: INFERENCE.apiKey, + NEMOCLAW_COMPAT_MODEL: INFERENCE.model, + NEMOCLAW_ENDPOINT_URL: INFERENCE.endpointUrl, + NEMOCLAW_MODEL: INFERENCE.model, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + }); + }); + + it("leaves inference selection unset when no fixture is staged", () => { + const env = buildSnapshotCommandEnv(SANDBOX_NAME); + + expect(env.COMPATIBLE_API_KEY).toBeUndefined(); + expect(env.NEMOCLAW_ENDPOINT_URL).toBeUndefined(); + expect(env.NEMOCLAW_PROVIDER).toBeUndefined(); + }); + + it("never exposes an ambient hosted NVIDIA inference credential to the child env", () => { + process.env[HOSTED_FLAG] = "1"; + for (const name of HOSTED_CREDENTIAL_ENVS) { + process.env[name] = "nvapi-ambient-credential-that-must-not-leak"; + } + + const env = buildSnapshotCommandEnv(SANDBOX_NAME, INFERENCE); + + for (const name of HOSTED_CREDENTIAL_ENVS) { + // Guard against the assertion below going vacuous: the credential really + // is present in the ambient env this helper builds from. + expect(process.env[name]).toBe("nvapi-ambient-credential-that-must-not-leak"); + expect(env[name]).toBeUndefined(); + } + }); +}); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f2a021105e8..f8a2881d2ba 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -2468,7 +2468,11 @@ function validateSnapshotCommandsJob(errors: string[], jobs: WorkflowRecord): vo if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { errors.push("snapshot-commands job must force OPENSHELL_GATEWAY=nemoclaw"); } + if ("NEMOCLAW_E2E_USE_HOSTED_INFERENCE" in jobEnv) { + errors.push("snapshot-commands job must not enable hosted inference"); + } for (const secret of [ + "NVIDIA_API_KEY", "NVIDIA_INFERENCE_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", @@ -2482,9 +2486,9 @@ function validateSnapshotCommandsJob(errors: string[], jobs: WorkflowRecord): vo for (const step of steps) { const stepName = `snapshot-commands step '${step.name ?? step.uses ?? ""}'`; const stepEnv = asRecord(step.env); - if (step.name !== "Run snapshot commands live test") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); - } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NEMOCLAW_E2E_USE_HOSTED_INFERENCE"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); if (step.name !== "Authenticate to Docker Hub") { requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); @@ -2503,12 +2507,6 @@ function validateSnapshotCommandsJob(errors: string[], jobs: WorkflowRecord): vo } const runVitest = requireJobStep(errors, jobName, steps, "Run snapshot commands live test"); - const runVitestEnv = asRecord(runVitest?.env); - if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push( - "snapshot-commands live E2E step must receive NVIDIA_INFERENCE_API_KEY from secrets", - ); - } requireRunContains(errors, runVitest, "npx vitest run --project e2e-live"); requireRunContains(errors, runVitest, "test/e2e/live/snapshot-commands.test.ts"); }