test(e2e): make snapshot commands inference hermetic

.github/workflows/e2e.yaml

@@ -3566,7 +3566,6 @@ jobs:
       E2E_TARGET_ID: "snapshot-commands"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/snapshot-commands
       NEMOCLAW_RUN_LIVE_E2E: "1"
-      NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
       NEMOCLAW_NON_INTERACTIVE: "1"
       NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
       NEMOCLAW_SANDBOX_NAME: "e2e-snapshot"
@@ -3585,11 +3584,8 @@ jobs:
           build-cli: "false"
 
       - name: Run snapshot commands live test
-        # Preserves the
-        # original ubuntu-latest + Docker/OpenShell + NVIDIA_INFERENCE_API_KEY runner
-        # contract by driving install.sh and the real snapshot CLI lifecycle.
-        env:
-          NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
+        # Drives install.sh and the real snapshot CLI lifecycle while the test
+        # supplies a sandbox-reachable fake OpenAI-compatible endpoint.
         run: |
           set -euo pipefail
           npx vitest run --project e2e-live \

test/e2e/live/snapshot-commands-helpers.ts

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+
+export interface SnapshotInferenceFixture {
+  apiKey: string;
+  endpointUrl: string;
+  model: string;
+}
+
+/**
+ * Builds the child env for the snapshot-commands live target.
+ *
+ * `NEMOCLAW_E2E_USE_HOSTED_INFERENCE` is in the fixture env allowlist, so an
+ * ambient value is forwarded into the child. `stageHostedInferenceSourceSecretEnv`
+ * treats that flag alone as sufficient to force hosted-custom staging even when
+ * `COMPATIBLE_API_KEY` names an explicit endpoint, which would silently route
+ * this target back at hosted inference. Strip it so the target stays hermetic.
+ */
+export function buildSnapshotCommandEnv(
+  sandboxName: string,
+  inference?: SnapshotInferenceFixture,
+): NodeJS.ProcessEnv {
+  const env: NodeJS.ProcessEnv = {
+    ...buildAvailabilityProbeEnv(),
+    NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
+    NEMOCLAW_NON_INTERACTIVE: "1",
+    NEMOCLAW_RECREATE_SANDBOX: "1",
+    NEMOCLAW_SANDBOX_NAME: sandboxName,
+    OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw",
+  };
+  delete env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE;
+  if (inference) {
+    Object.assign(env, {
+      COMPATIBLE_API_KEY: inference.apiKey,
+      NEMOCLAW_COMPAT_MODEL: inference.model,
+      NEMOCLAW_ENDPOINT_URL: inference.endpointUrl,
+      NEMOCLAW_MODEL: inference.model,
+      NEMOCLAW_PREFERRED_API: "openai-completions",
+      NEMOCLAW_PROVIDER: "custom",
+    });
+  }
+  return env;
+}

test/e2e/live/snapshot-commands.test.ts

@@ -15,11 +15,18 @@ import path from "node:path";
 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
 import { resultText } from "../fixtures/clients/command.ts";
 import type { HostCliClient } from "../fixtures/clients/host.ts";
-import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts";
+import {
+  type SandboxClient,
+  trustedSandboxShellScript,
+  validateSandboxName,
+} from "../fixtures/clients/sandbox.ts";
 import { expect, test } from "../fixtures/e2e-test.ts";
+import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts";
 import { REPO_ROOT } from "../fixtures/paths.ts";
-import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
-import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts";
+import {
+  buildSnapshotCommandEnv,
+  type SnapshotInferenceFixture,
+} from "./snapshot-commands-helpers.ts";
 import { scanSnapshotCredentialLeaks } from "./snapshot-credential-scanner.ts";
 
 const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-snapshot";
@@ -32,19 +39,11 @@ if (!BACKUP_DIR.startsWith(`${path.resolve(BACKUP_ROOT)}${path.sep}`)) {
 const MARKER_FILE = "/sandbox/.openclaw/workspace/snapshot-marker.txt";
 const SECOND_MARKER = "/sandbox/.openclaw/workspace/snapshot-marker-2.txt";
 const LIVE_TIMEOUT_MS = 30 * 60_000;
-const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1;
+const INFERENCE_API_KEY = "nvapi-snapshot-commands-fixture-credential";
+const INFERENCE_MODEL = "snapshot-commands-model";
 
-function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
-  const env: NodeJS.ProcessEnv = {
-    ...buildAvailabilityProbeEnv(),
-    NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
-    NEMOCLAW_NON_INTERACTIVE: "1",
-    NEMOCLAW_RECREATE_SANDBOX: "1",
-    NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
-    OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw",
-  };
-  if (apiKey) env.NVIDIA_INFERENCE_API_KEY = apiKey;
-  return env;
+function commandEnv(inference?: SnapshotInferenceFixture): NodeJS.ProcessEnv {
+  return buildSnapshotCommandEnv(SANDBOX_NAME, inference);
 }
 
 async function bestEffort(run: () => Promise<unknown>): Promise<void> {
@@ -107,15 +106,15 @@ function firstSnapshotTimestamp(listOutput: string): string {
 
 test("snapshot commands preserve create/list/latest restore/targeted restore/no-leak lifecycle", {
   timeout: LIVE_TIMEOUT_MS,
-}, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => {
-  const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY");
+}, async ({ artifacts, cleanup, host, sandbox, skip }) => {
   await artifacts.target.declare({
     id: "snapshot-commands",
     boundary: "install.sh + nemoclaw snapshot commands + openshell sandbox exec",
     sandboxName: SANDBOX_NAME,
     backupDir: BACKUP_DIR,
     contracts: [
       "install.sh onboards a live OpenClaw sandbox",
+      "onboard authenticates to a hermetic compatible inference endpoint",
       "snapshot create reports Snapshot v<N> created",
       "snapshot list shows versioned snapshots and parseable timestamps",
       "latest snapshot restore recovers latest workspace state",
@@ -137,43 +136,68 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no-
     skip(`Docker is required for snapshot commands E2E: ${resultText(dockerInfo)}`);
   }
 
+  const inference = await startFakeOpenAiCompatibleServer({
+    apiKey: INFERENCE_API_KEY,
+    host: "0.0.0.0",
+    model: INFERENCE_MODEL,
+    publicHost: "host.openshell.internal",
+    requireAuth: true,
+    requireAuthModels: true,
+  });
+  cleanup.add("close snapshot commands compatible inference fixture", async () => {
+    await artifacts.writeJson("compatible-inference-requests.json", inference.requests());
+    await inference.close();
+  });
+  const inferenceConfig = {
+    apiKey: INFERENCE_API_KEY,
+    endpointUrl: inference.baseUrl,
+    model: INFERENCE_MODEL,
+  };
+
   cleanup.add(`destroy snapshot sandbox ${SANDBOX_NAME}`, () =>
     cleanupSnapshotSandbox(host, sandbox, "cleanup"),
   );
 
   await cleanupSnapshotSandbox(host, sandbox, "pre-cleanup");
   fs.rmSync(BACKUP_DIR, { recursive: true, force: true });
 
-  let install: ShellProbeResult | undefined;
-  for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) {
-    install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], {
-      artifactName:
-        attempt === 1 ? "phase-1-install-nemoclaw" : `phase-1-install-nemoclaw-attempt-${attempt}`,
-      cwd: REPO_ROOT,
-      env: commandEnv(apiKey),
-      redactionValues: [apiKey],
-      timeoutMs: 20 * 60_000,
-    });
-    if (install.exitCode === 0) break;
-    if (isTransientProviderValidationFailure(install) && attempt < INSTALL_ATTEMPTS) {
-      await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt));
-      continue;
-    }
-    if (isTransientProviderValidationFailure(install) && process.env.GITHUB_ACTIONS === "true") {
-      await artifacts.writeJson("transient-provider-validation.skip.json", {
-        reason: "transient NVIDIA Endpoints validation failure during install.sh onboard",
-        attempts: INSTALL_ATTEMPTS,
-        sourceBoundary: "external NVIDIA Endpoints provider availability",
-        removalCondition:
-          "remove once CI endpoint validation is stable for a release cycle or covered by a hermetic provider-validation fixture",
-      });
-      skip(
-        `NVIDIA Endpoints validation hit a transient upstream/rate-limit failure after ${INSTALL_ATTEMPTS} attempts`,
-      );
-    }
-    break;
-  }
-  expect(install?.exitCode, install ? resultText(install) : "install did not run").toBe(0);
+  const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], {
+    artifactName: "phase-1-install-nemoclaw",
+    cwd: REPO_ROOT,
+    env: commandEnv(inferenceConfig),
+    redactionValues: [INFERENCE_API_KEY],
+    timeoutMs: 20 * 60_000,
+  });
+  expect(install.exitCode, resultText(install)).toBe(0);
+
+  const authenticatedInference = await sandbox.execShell(
+    SANDBOX_NAME,
+    trustedSandboxShellScript(
+      `curl -fsS --max-time 60 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${JSON.stringify(
+        {
+          model: INFERENCE_MODEL,
+          messages: [{ role: "user", content: "reply with OK" }],
+          max_tokens: 8,
+        },
+      )}'`,
+    ),
+    {
+      artifactName: "phase-1-authenticated-inference-post",
+      env: commandEnv(),
+      timeoutMs: 90_000,
+    },
+  );
+  expect(
+    authenticatedInference.exitCode,
+    `${authenticatedInference.stdout}\n${authenticatedInference.stderr}`,
+  ).toBe(0);
+  expect(inference.requests()).toContainEqual(
+    expect.objectContaining({
+      auth: "ok",
+      model: INFERENCE_MODEL,
+      path: "/v1/chat/completions",
+    }),
+  );
 
   const cliProbe = await host.command(
     "bash",

test/e2e/support/e2e-workflow.test.ts

@@ -1076,11 +1076,17 @@ jobs:
     snapshotJob["timeout-minutes"] = 30;
     snapshotJob.env.DOCKER_CONFIG = "${{ github.workspace }}/.docker-config-shared";
     snapshotJob.env.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}";
+    snapshotJob.env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE = "1";
     for (const step of snapshotJob.steps) {
       if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) {
         step.with = { ...(step.with as Record<string, unknown>), "persist-credentials": true };
       }
       if (step.name === "Run snapshot commands live test") {
+        step.env = {
+          ...((step.env as Record<string, unknown> | undefined) ?? {}),
+          NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1",
+          NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}",
+        };
         step.run = String(step.run).replace(
           "test/e2e/live/snapshot-commands.test.ts",
           "test/e2e/live/registry-targets.test.ts",
@@ -1101,8 +1107,11 @@ jobs:
         expect.arrayContaining([
           "snapshot-commands job must keep a 40 minute timeout",
           "snapshot-commands job must not set DOCKER_CONFIG at job level",
+          "snapshot-commands job must not enable hosted inference",
           "snapshot-commands checkout step must set persist-credentials=false",
           "snapshot-commands job env must not include NVIDIA_INFERENCE_API_KEY",
+          "snapshot-commands step 'Run snapshot commands live test' env must not include NEMOCLAW_E2E_USE_HOSTED_INFERENCE",
+          "snapshot-commands step 'Run snapshot commands live test' env must not include NVIDIA_API_KEY",
           "snapshot-commands upload-e2e-artifacts invocation must not override its contract",
           "snapshot-commands upload-e2e-artifacts must use the action defaults",
           "step 'Run snapshot commands live test' run script must include test/e2e/live/snapshot-commands.test.ts",

test/e2e/support/snapshot-commands-helpers.test.ts

@@ -0,0 +1,87 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import { buildSnapshotCommandEnv } from "../live/snapshot-commands-helpers.ts";
+
+const HOSTED_FLAG = "NEMOCLAW_E2E_USE_HOSTED_INFERENCE";
+const SANDBOX_NAME = "e2e-snapshot";
+
+const INFERENCE = {
+  apiKey: "nvapi-snapshot-commands-fixture-credential",
+  endpointUrl: "http://host.openshell.internal:31337/v1",
+  model: "snapshot-commands-model",
+};
+
+const HOSTED_CREDENTIAL_ENVS = ["NVIDIA_INFERENCE_API_KEY", "NVIDIA_API_KEY"] as const;
+
+afterEach(() => {
+  delete process.env[HOSTED_FLAG];
+  for (const name of HOSTED_CREDENTIAL_ENVS) delete process.env[name];
+});
+
+describe("snapshot commands live env helper", () => {
+  it("forwards an ambient hosted-inference flag through the shared probe env", () => {
+    process.env[HOSTED_FLAG] = "1";
+
+    // Negative control. The strip below only matters while the flag is
+    // allowlisted for forwarding; if this ever stops holding, the strip
+    // assertion would pass vacuously and the hermeticity guard would rot.
+    expect(buildAvailabilityProbeEnv()[HOSTED_FLAG]).toBe("1");
+  });
+
+  it("strips an ambient hosted-inference flag so the target stays hermetic", () => {
+    process.env[HOSTED_FLAG] = "1";
+
+    const env = buildSnapshotCommandEnv(SANDBOX_NAME, INFERENCE);
+
+    expect(env[HOSTED_FLAG]).toBeUndefined();
+    expect(Object.hasOwn(env, HOSTED_FLAG)).toBe(false);
+  });
+
+  it("strips the hosted-inference flag even when no inference fixture is staged", () => {
+    process.env[HOSTED_FLAG] = "1";
+
+    expect(buildSnapshotCommandEnv(SANDBOX_NAME)[HOSTED_FLAG]).toBeUndefined();
+  });
+
+  it("stages the compatible endpoint against the custom provider", () => {
+    const env = buildSnapshotCommandEnv(SANDBOX_NAME, INFERENCE);
+
+    expect(env).toMatchObject({
+      COMPATIBLE_API_KEY: INFERENCE.apiKey,
+      NEMOCLAW_COMPAT_MODEL: INFERENCE.model,
+      NEMOCLAW_ENDPOINT_URL: INFERENCE.endpointUrl,
+      NEMOCLAW_MODEL: INFERENCE.model,
+      NEMOCLAW_PREFERRED_API: "openai-completions",
+      NEMOCLAW_PROVIDER: "custom",
+      NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
+    });
+  });
+
+  it("leaves inference selection unset when no fixture is staged", () => {
+    const env = buildSnapshotCommandEnv(SANDBOX_NAME);
+
+    expect(env.COMPATIBLE_API_KEY).toBeUndefined();
+    expect(env.NEMOCLAW_ENDPOINT_URL).toBeUndefined();
+    expect(env.NEMOCLAW_PROVIDER).toBeUndefined();
+  });
+
+  it("never exposes an ambient hosted NVIDIA inference credential to the child env", () => {
+    process.env[HOSTED_FLAG] = "1";
+    for (const name of HOSTED_CREDENTIAL_ENVS) {
+      process.env[name] = "nvapi-ambient-credential-that-must-not-leak";
+    }
+
+    const env = buildSnapshotCommandEnv(SANDBOX_NAME, INFERENCE);
+
+    for (const name of HOSTED_CREDENTIAL_ENVS) {
+      // Guard against the assertion below going vacuous: the credential really
+      // is present in the ambient env this helper builds from.
+      expect(process.env[name]).toBe("nvapi-ambient-credential-that-must-not-leak");
+      expect(env[name]).toBeUndefined();
+    }
+  });
+});

tools/e2e/workflow-boundary.mts

@@ -2468,7 +2468,11 @@ function validateSnapshotCommandsJob(errors: string[], jobs: WorkflowRecord): vo
   if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") {
     errors.push("snapshot-commands job must force OPENSHELL_GATEWAY=nemoclaw");
   }
+  if ("NEMOCLAW_E2E_USE_HOSTED_INFERENCE" in jobEnv) {
+    errors.push("snapshot-commands job must not enable hosted inference");
+  }
   for (const secret of [
+    "NVIDIA_API_KEY",
     "NVIDIA_INFERENCE_API_KEY",
     "DOCKERHUB_USERNAME",
     "DOCKERHUB_TOKEN",
@@ -2482,9 +2486,9 @@ function validateSnapshotCommandsJob(errors: string[], jobs: WorkflowRecord): vo
   for (const step of steps) {
     const stepName = `snapshot-commands step '${step.name ?? step.uses ?? "<unnamed>"}'`;
     const stepEnv = asRecord(step.env);
-    if (step.name !== "Run snapshot commands live test") {
-      requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY");
-    }
+    requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NEMOCLAW_E2E_USE_HOSTED_INFERENCE");
+    requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY");
+    requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY");
     if (step.name !== "Authenticate to Docker Hub") {
       requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME");
       requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN");
@@ -2503,12 +2507,6 @@ function validateSnapshotCommandsJob(errors: string[], jobs: WorkflowRecord): vo
   }
 
   const runVitest = requireJobStep(errors, jobName, steps, "Run snapshot commands live test");
-  const runVitestEnv = asRecord(runVitest?.env);
-  if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") {
-    errors.push(
-      "snapshot-commands live E2E step must receive NVIDIA_INFERENCE_API_KEY from secrets",
-    );
-  }
   requireRunContains(errors, runVitest, "npx vitest run --project e2e-live");
   requireRunContains(errors, runVitest, "test/e2e/live/snapshot-commands.test.ts");
 }