From d26dae618c782f3b9868f388f9547ba0bfa0ab24 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 12:50:40 -0400 Subject: [PATCH 1/7] test(e2e): restore recovery kimi scope parity --- .github/workflows/e2e-vitest-scenarios.yaml | 3 + .../live/kimi-inference-compat-helpers.ts | 47 +++++++++++-- .../live/kimi-inference-compat.test.ts | 67 +++++++++++++------ .../kimi-inference-compat-helpers.test.ts | 42 ++++++++++++ test/e2e-script-workflow.test.ts | 16 +++++ test/helpers/e2e-workflow-contract.ts | 1 + 6 files changed, 150 insertions(+), 26 deletions(-) create mode 100644 test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 2def39a7962..42fb248cd44 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -920,6 +920,7 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-kimi-compat" + NEMOCLAW_E2E_INFERENCE_MODE: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && 'public-nvidia' || 'mock' }} OPENSHELL_GATEWAY: "nemoclaw" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -937,6 +938,8 @@ jobs: - name: Install OpenShell CLI run: bash scripts/install-openshell.sh - name: Run Kimi compatibility live Vitest test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" diff --git a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts index cf93aeeb8b7..5dcd5ed35aa 100644 --- a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts +++ b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts @@ -21,6 +21,26 @@ export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-kimi-compa validateSandboxName(SANDBOX_NAME); export const KIMI_MODEL = process.env.NEMOCLAW_KIMI_MODEL ?? "moonshotai/kimi-k2.6"; +export type KimiInferenceMode = "mock" | "public-nvidia"; + +export interface KimiEnvOptions { + mode?: KimiInferenceMode; + apiKey?: string; +} + +export function resolveKimiInferenceMode(env: NodeJS.ProcessEnv = process.env): KimiInferenceMode { + if (env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase() === "public-nvidia") + return "public-nvidia"; + if (env.NEMOCLAW_KIMI_USE_MOCK === "0") return "public-nvidia"; + return "mock"; +} + +export function requirePublicNvidiaApiKey(value: string): string { + if (!value.startsWith("nvapi-")) + throw new Error("NVIDIA_API_KEY must be a public NVIDIA Endpoints nvapi-* key"); + return value; +} + export interface KimiRequest { path: string; model?: string; @@ -35,10 +55,13 @@ export interface KimiMock { close(): Promise; } -export function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { +export function env( + extra: NodeJS.ProcessEnv = {}, + options: KimiEnvOptions = {}, +): NodeJS.ProcessEnv { + const mode = options.mode ?? resolveKimiInferenceMode(); + const common: NodeJS.ProcessEnv = { ...buildAvailabilityProbeEnv(), - COMPATIBLE_API_KEY: "test-kimi-key", NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_MODEL: KIMI_MODEL, NEMOCLAW_NON_INTERACTIVE: "1", @@ -47,11 +70,27 @@ export function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { NEMOCLAW_POLICY_MODE: "skip", NEMOCLAW_POLICY_TIER: "restricted", NEMOCLAW_PREFERRED_API: "openai-completions", - NEMOCLAW_PROVIDER: "custom", NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_YES: "1", OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; + if (mode === "public-nvidia") { + return { + ...common, + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", + NEMOCLAW_PROVIDER: "cloud", + ...(options.apiKey + ? { NVIDIA_API_KEY: options.apiKey, NVIDIA_INFERENCE_API_KEY: options.apiKey } + : {}), + ...extra, + }; + } + return { + ...common, + COMPATIBLE_API_KEY: "test-kimi-key", + NEMOCLAW_E2E_INFERENCE_MODE: "mock", + NEMOCLAW_PROVIDER: "custom", ...extra, }; } diff --git a/test/e2e-scenario/live/kimi-inference-compat.test.ts b/test/e2e-scenario/live/kimi-inference-compat.test.ts index dbda023a005..11c15336e97 100644 --- a/test/e2e-scenario/live/kimi-inference-compat.test.ts +++ b/test/e2e-scenario/live/kimi-inference-compat.test.ts @@ -16,6 +16,8 @@ import { KIMI_MODEL, parseConfig, REPO_ROOT, + requirePublicNvidiaApiKey, + resolveKimiInferenceMode, SANDBOX_NAME, startKimiMock, } from "./kimi-inference-compat-helpers.ts"; @@ -25,16 +27,25 @@ const TIMEOUT_MS = 40 * 60_000; test.skipIf(!shouldRunLiveE2EScenarios())( "Kimi-compatible endpoint config enables plugin wiring and managed inference route", { timeout: TIMEOUT_MS }, - async ({ artifacts, cleanup, host, sandbox }) => { - const fake = await startKimiMock(); - cleanup.add("close fake Kimi endpoint", () => fake.close()); + async ({ artifacts, cleanup, host, sandbox, secrets }) => { + const mode = resolveKimiInferenceMode(); + const apiKey = + mode === "public-nvidia" + ? requirePublicNvidiaApiKey(secrets.required("NVIDIA_API_KEY")) + : undefined; + const fake = mode === "mock" ? await startKimiMock() : undefined; + if (fake) cleanup.add("close fake Kimi endpoint", () => fake.close()); cleanup.add("destroy Kimi sandbox", () => cleanupKimi(host, sandbox)); await artifacts.writeJson("scenario.json", { id: "kimi-inference-compat", legacySource: "test/e2e/test-kimi-inference-compat.sh", boundary: - "source CLI onboard + fake OpenAI-compatible Kimi endpoint + OpenClaw config/plugin/inference route", + mode === "public-nvidia" + ? "source CLI onboard + public NVIDIA Kimi endpoint + OpenClaw config/plugin/inference route" + : "source CLI onboard + fake OpenAI-compatible Kimi endpoint + OpenClaw config/plugin/inference route", + inferenceClassification: "public-nvidia required with mock/hermetic fallback", + inferenceMode: mode, sandboxName: SANDBOX_NAME, model: KIMI_MODEL, }); @@ -54,8 +65,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())( { artifactName: "onboard-kimi-compatible", cwd: REPO_ROOT, - env: env({ NEMOCLAW_ENDPOINT_URL: fake.baseUrl }), - redactionValues: ["test-kimi-key"], + env: env(fake ? { NEMOCLAW_ENDPOINT_URL: fake.baseUrl } : {}, { mode, apiKey }), + redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 20 * 60_000, }, ); @@ -63,7 +74,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { artifactName: "openclaw-config", - env: env(), + env: env({}, { mode, apiKey }), timeoutMs: 60_000, }); expect(config.exitCode, resultText(config)).toBe(0); @@ -88,7 +99,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const modelsRoute = await sandbox.exec( SANDBOX_NAME, ["curl", "-sk", "--max-time", "20", "https://inference.local/v1/models"], - { artifactName: "inference-local-models", env: env(), timeoutMs: 60_000 }, + { artifactName: "inference-local-models", env: env({}, { mode, apiKey }), timeoutMs: 60_000 }, ); expect(modelsRoute.exitCode, resultText(modelsRoute)).toBe(0); expect(resultText(modelsRoute)).toContain(KIMI_MODEL); @@ -100,8 +111,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ), { artifactName: "kimi-agent-smoke", - env: env(), - redactionValues: ["test-kimi-key"], + env: env({}, { mode, apiKey }), + redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 150_000, }, ); @@ -115,22 +126,34 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ), { artifactName: "kimi-agent-tool-splitting", - env: env(), - redactionValues: ["test-kimi-key"], + env: env({}, { mode, apiKey }), + redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 420_000, }, ); expect(toolAgent.exitCode, resultText(toolAgent)).toBe(0); await assertTrajectory(sandbox); - expect( - fake.requests.some( - (request) => - request.authOk && - request.path.includes("/chat/completions") && - request.model === KIMI_MODEL && - request.hasTools, - ), - ).toBe(true); - expect(fake.requests.some((request) => request.authOk && request.hasToolResult)).toBe(true); + if (fake) { + expect( + fake.requests.some( + (request) => + request.authOk && + request.path.includes("/chat/completions") && + request.model === KIMI_MODEL && + request.hasTools, + ), + ).toBe(true); + expect(fake.requests.some((request) => request.authOk && request.hasToolResult)).toBe(true); + } else { + const route = await host.command("openshell", ["inference", "get", "-g", "nemoclaw"], { + artifactName: "public-nvidia-kimi-route", + env: env({}, { mode, apiKey }), + redactionValues: [apiKey ?? ""], + timeoutMs: 60_000, + }); + expect(route.exitCode, resultText(route)).toBe(0); + expect(resultText(route)).toContain("nvidia-prod"); + expect(resultText(route)).toContain(KIMI_MODEL); + } }, ); diff --git a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts new file mode 100644 index 00000000000..15d26c5ef3d --- /dev/null +++ b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + env, + requirePublicNvidiaApiKey, + resolveKimiInferenceMode, +} from "../live/kimi-inference-compat-helpers.ts"; + +describe("Kimi inference compatibility mode selection", () => { + it("defaults to hermetic mock mode for local validation", () => { + const cfg = env({}, { mode: "mock" }); + expect(cfg.NEMOCLAW_E2E_INFERENCE_MODE).toBe("mock"); + expect(cfg.NEMOCLAW_PROVIDER).toBe("custom"); + expect(cfg.COMPATIBLE_API_KEY).toBe("test-kimi-key"); + expect(cfg.NVIDIA_API_KEY).toBeUndefined(); + }); + + it("preserves public NVIDIA mode using NVIDIA_API_KEY as the source secret", () => { + const cfg = env({}, { mode: "public-nvidia", apiKey: "nvapi-public-test-key" }); + expect(cfg.NEMOCLAW_E2E_INFERENCE_MODE).toBe("public-nvidia"); + expect(cfg.NEMOCLAW_PROVIDER).toBe("cloud"); + expect(cfg.NVIDIA_API_KEY).toBe("nvapi-public-test-key"); + expect(cfg.NVIDIA_INFERENCE_API_KEY).toBe("nvapi-public-test-key"); + expect(cfg.COMPATIBLE_API_KEY).toBeUndefined(); + }); + + it("rejects non-public NVIDIA keys for public Kimi validation", () => { + expect(() => requirePublicNvidiaApiKey("sk-compatible-key")).toThrow(/nvapi-\* key/); + expect(requirePublicNvidiaApiKey("nvapi-public-test-key")).toBe("nvapi-public-test-key"); + }); + + it("maps legacy and explicit env selectors to the expected mode", () => { + expect(resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" })).toBe( + "public-nvidia", + ); + expect(resolveKimiInferenceMode({ NEMOCLAW_KIMI_USE_MOCK: "0" })).toBe("public-nvidia"); + expect(resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "mock" })).toBe("mock"); + }); +}); diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index f966b53f988..0af46a5e0c5 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -584,6 +584,22 @@ describe("E2E reusable workflow contract", () => { ); }); + it("uses NVIDIA_API_KEY for the live Kimi Vitest lane", () => { + const vitestWorkflow = readYaml<{ jobs: Record }>( + ".github/workflows/e2e-vitest-scenarios.yaml", + ); + const job = vitestWorkflow.jobs["kimi-inference-compat-vitest"]; + const runStep = job.steps?.find( + (step) => step.name === "Run Kimi compatibility live Vitest test", + ); + + expect(job.env?.NEMOCLAW_E2E_INFERENCE_MODE).toBe( + `\${{ (${TRUSTED_REF_GUARD}) && 'public-nvidia' || 'mock' }}`, + ); + expect(runStep?.env?.NVIDIA_API_KEY).toBe(GUARDED_PUBLIC_NVIDIA_SECRET); + expect(runStep?.env?.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + }); + it("authenticates Docker Hub pulls in direct nightly E2E jobs", () => { const directE2eJobs = [ "openclaw-tui-chat-correlation-e2e", diff --git a/test/helpers/e2e-workflow-contract.ts b/test/helpers/e2e-workflow-contract.ts index f8ff5460850..5537f7e8800 100644 --- a/test/helpers/e2e-workflow-contract.ts +++ b/test/helpers/e2e-workflow-contract.ts @@ -12,6 +12,7 @@ export type WorkflowJob = { "runs-on"?: string; "timeout-minutes"?: number; uses?: string; + env?: Record; secrets?: Record; steps?: WorkflowStep[]; with?: Record; From 2af0cd4eebd22a1e76fb22f9b9b79656a92d4e14 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 12:54:32 -0400 Subject: [PATCH 2/7] fixup! test(e2e): restore recovery kimi scope parity --- .github/workflows/e2e-vitest-scenarios.yaml | 4 ++-- test/e2e-script-workflow.test.ts | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 42fb248cd44..d5d607cbcd7 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -920,7 +920,7 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-kimi-compat" - NEMOCLAW_E2E_INFERENCE_MODE: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && 'public-nvidia' || 'mock' }} + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" OPENSHELL_GATEWAY: "nemoclaw" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -939,7 +939,7 @@ jobs: run: bash scripts/install-openshell.sh - name: Run Kimi compatibility live Vitest test env: - NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index 0af46a5e0c5..d9cb24c3fe9 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -593,10 +593,8 @@ describe("E2E reusable workflow contract", () => { (step) => step.name === "Run Kimi compatibility live Vitest test", ); - expect(job.env?.NEMOCLAW_E2E_INFERENCE_MODE).toBe( - `\${{ (${TRUSTED_REF_GUARD}) && 'public-nvidia' || 'mock' }}`, - ); - expect(runStep?.env?.NVIDIA_API_KEY).toBe(GUARDED_PUBLIC_NVIDIA_SECRET); + expect(job.env?.NEMOCLAW_E2E_INFERENCE_MODE).toBe("public-nvidia"); + expect(runStep?.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); expect(runStep?.env?.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); }); From 4f9dc101f05b9949485d9cbd1992f40ed3f0e819 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 12:56:46 -0400 Subject: [PATCH 3/7] test(e2e): keep kimi live test linear --- .../live/kimi-inference-compat-helpers.ts | 58 +++++++++++++++++++ .../live/kimi-inference-compat.test.ts | 40 ++++--------- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts index 5dcd5ed35aa..f37650d60b6 100644 --- a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts +++ b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts @@ -116,6 +116,64 @@ export async function startKimiMock(): Promise { }; } +export function kimiBoundary(mode: KimiInferenceMode): string { + return mode === "public-nvidia" + ? "source CLI onboard + public NVIDIA Kimi endpoint + OpenClaw config/plugin/inference route" + : "source CLI onboard + fake OpenAI-compatible Kimi endpoint + OpenClaw config/plugin/inference route"; +} + +export async function startKimiUpstream(mode: KimiInferenceMode): Promise { + return mode === "mock" ? startKimiMock() : undefined; +} + +export function maybeRegisterKimiMockCleanup( + cleanup: { add(name: string, fn: () => Promise): void }, + fake: KimiMock | undefined, +): void { + if (fake) cleanup.add("close fake Kimi endpoint", () => fake.close()); +} + +export function kimiOnboardEnv( + fake: KimiMock | undefined, + mode: KimiInferenceMode, + apiKey: string | undefined, +): NodeJS.ProcessEnv { + return env(fake ? { NEMOCLAW_ENDPOINT_URL: fake.baseUrl } : {}, { mode, apiKey }); +} + +export async function assertKimiUpstreamTraffic(options: { + fake: KimiMock | undefined; + host: HostCliClient; + mode: KimiInferenceMode; + apiKey: string | undefined; +}): Promise { + if (options.fake) { + expect( + options.fake.requests.some( + (request) => + request.authOk && + request.path.includes("/chat/completions") && + request.model === KIMI_MODEL && + request.hasTools, + ), + ).toBe(true); + expect(options.fake.requests.some((request) => request.authOk && request.hasToolResult)).toBe( + true, + ); + return; + } + + const route = await options.host.command("openshell", ["inference", "get", "-g", "nemoclaw"], { + artifactName: "public-nvidia-kimi-route", + env: env({}, { mode: options.mode, apiKey: options.apiKey }), + redactionValues: [options.apiKey ?? ""], + timeoutMs: 60_000, + }); + expect(route.exitCode, resultText(route)).toBe(0); + expect(resultText(route)).toContain("nvidia-prod"); + expect(resultText(route)).toContain(KIMI_MODEL); +} + function handleKimiRequest( req: http.IncomingMessage, res: http.ServerResponse, diff --git a/test/e2e-scenario/live/kimi-inference-compat.test.ts b/test/e2e-scenario/live/kimi-inference-compat.test.ts index 11c15336e97..7eeb251a886 100644 --- a/test/e2e-scenario/live/kimi-inference-compat.test.ts +++ b/test/e2e-scenario/live/kimi-inference-compat.test.ts @@ -9,17 +9,21 @@ import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; import { + assertKimiUpstreamTraffic, assertTrajectory, CLI, cleanupKimi, env, KIMI_MODEL, + kimiBoundary, + kimiOnboardEnv, + maybeRegisterKimiMockCleanup, parseConfig, REPO_ROOT, requirePublicNvidiaApiKey, resolveKimiInferenceMode, SANDBOX_NAME, - startKimiMock, + startKimiUpstream, } from "./kimi-inference-compat-helpers.ts"; const TIMEOUT_MS = 40 * 60_000; @@ -33,17 +37,14 @@ test.skipIf(!shouldRunLiveE2EScenarios())( mode === "public-nvidia" ? requirePublicNvidiaApiKey(secrets.required("NVIDIA_API_KEY")) : undefined; - const fake = mode === "mock" ? await startKimiMock() : undefined; - if (fake) cleanup.add("close fake Kimi endpoint", () => fake.close()); + const fake = await startKimiUpstream(mode); + maybeRegisterKimiMockCleanup(cleanup, fake); cleanup.add("destroy Kimi sandbox", () => cleanupKimi(host, sandbox)); await artifacts.writeJson("scenario.json", { id: "kimi-inference-compat", legacySource: "test/e2e/test-kimi-inference-compat.sh", - boundary: - mode === "public-nvidia" - ? "source CLI onboard + public NVIDIA Kimi endpoint + OpenClaw config/plugin/inference route" - : "source CLI onboard + fake OpenAI-compatible Kimi endpoint + OpenClaw config/plugin/inference route", + boundary: kimiBoundary(mode), inferenceClassification: "public-nvidia required with mock/hermetic fallback", inferenceMode: mode, sandboxName: SANDBOX_NAME, @@ -65,7 +66,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( { artifactName: "onboard-kimi-compatible", cwd: REPO_ROOT, - env: env(fake ? { NEMOCLAW_ENDPOINT_URL: fake.baseUrl } : {}, { mode, apiKey }), + env: kimiOnboardEnv(fake, mode, apiKey), redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 20 * 60_000, }, @@ -133,27 +134,6 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ); expect(toolAgent.exitCode, resultText(toolAgent)).toBe(0); await assertTrajectory(sandbox); - if (fake) { - expect( - fake.requests.some( - (request) => - request.authOk && - request.path.includes("/chat/completions") && - request.model === KIMI_MODEL && - request.hasTools, - ), - ).toBe(true); - expect(fake.requests.some((request) => request.authOk && request.hasToolResult)).toBe(true); - } else { - const route = await host.command("openshell", ["inference", "get", "-g", "nemoclaw"], { - artifactName: "public-nvidia-kimi-route", - env: env({}, { mode, apiKey }), - redactionValues: [apiKey ?? ""], - timeoutMs: 60_000, - }); - expect(route.exitCode, resultText(route)).toBe(0); - expect(resultText(route)).toContain("nvidia-prod"); - expect(resultText(route)).toContain(KIMI_MODEL); - } + await assertKimiUpstreamTraffic({ fake, host, mode, apiKey }); }, ); From ea7464865248d63e1b55532f583ca1e9ee540a70 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 13:25:20 -0400 Subject: [PATCH 4/7] test(e2e): harden kimi public validation --- .../live/kimi-inference-compat-helpers.ts | 21 ++++++++++++------- .../live/kimi-inference-compat.test.ts | 21 +++---------------- .../kimi-inference-compat-helpers.test.ts | 18 ++++++++++++++-- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts index f37650d60b6..86ce07288ea 100644 --- a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts +++ b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts @@ -26,6 +26,7 @@ export type KimiInferenceMode = "mock" | "public-nvidia"; export interface KimiEnvOptions { mode?: KimiInferenceMode; apiKey?: string; + includeSecret?: boolean; } export function resolveKimiInferenceMode(env: NodeJS.ProcessEnv = process.env): KimiInferenceMode { @@ -80,7 +81,7 @@ export function env( ...common, NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", NEMOCLAW_PROVIDER: "cloud", - ...(options.apiKey + ...(options.includeSecret && options.apiKey ? { NVIDIA_API_KEY: options.apiKey, NVIDIA_INFERENCE_API_KEY: options.apiKey } : {}), ...extra, @@ -138,7 +139,11 @@ export function kimiOnboardEnv( mode: KimiInferenceMode, apiKey: string | undefined, ): NodeJS.ProcessEnv { - return env(fake ? { NEMOCLAW_ENDPOINT_URL: fake.baseUrl } : {}, { mode, apiKey }); + return env(fake ? { NEMOCLAW_ENDPOINT_URL: fake.baseUrl } : {}, { + mode, + apiKey, + includeSecret: true, + }); } export async function assertKimiUpstreamTraffic(options: { @@ -399,10 +404,7 @@ export function parseConfig(raw: string): { } export async function assertTrajectory(sandbox: SandboxClient): Promise { - const trajectory = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript(String.raw`python3 - <<'PY' -import json, pathlib, sys + const checkScript = String.raw`import json, pathlib, sys root=pathlib.Path('/sandbox/.openclaw') base=pathlib.Path('/sandbox/.openclaw/agents/main/sessions') session=base/'e2e-kimi-tools.jsonl' @@ -441,8 +443,11 @@ if not final_texts or normalize_final_text(final_texts[-1]) != 'hostname, date, roles=[m.get('role') for m in messages] if not ('toolResult' in roles and roles[-1]=='assistant'): errors.append('final assistant not after tool result') print(json.dumps({'errors':errors,'source':source,'toolMetas':metas,'roles':roles}, indent=2)) -sys.exit(1 if errors else 0) -PY`), +sys.exit(1 if errors else 0)`; + const encoded = Buffer.from(checkScript, "utf8").toString("base64"); + const trajectory = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript(`python3 -c "$(printf %s '${encoded}' | base64 -d)"`), { artifactName: "kimi-trajectory-tool-splitting-check", env: env(), timeoutMs: 60_000 }, ); expect(trajectory.exitCode, resultText(trajectory)).toBe(0); diff --git a/test/e2e-scenario/live/kimi-inference-compat.test.ts b/test/e2e-scenario/live/kimi-inference-compat.test.ts index 7eeb251a886..9649e026304 100644 --- a/test/e2e-scenario/live/kimi-inference-compat.test.ts +++ b/test/e2e-scenario/live/kimi-inference-compat.test.ts @@ -75,7 +75,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { artifactName: "openclaw-config", - env: env({}, { mode, apiKey }), + env: env({}, { mode }), timeoutMs: 60_000, }); expect(config.exitCode, resultText(config)).toBe(0); @@ -100,26 +100,11 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const modelsRoute = await sandbox.exec( SANDBOX_NAME, ["curl", "-sk", "--max-time", "20", "https://inference.local/v1/models"], - { artifactName: "inference-local-models", env: env({}, { mode, apiKey }), timeoutMs: 60_000 }, + { artifactName: "inference-local-models", env: env({}, { mode }), timeoutMs: 60_000 }, ); expect(modelsRoute.exitCode, resultText(modelsRoute)).toBe(0); expect(resultText(modelsRoute)).toContain(KIMI_MODEL); - const agent = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - "openclaw agent --agent main --json --session-id e2e-kimi-compat -m 'Reply with exactly: OK'", - ), - { - artifactName: "kimi-agent-smoke", - env: env({}, { mode, apiKey }), - redactionValues: ["test-kimi-key", apiKey ?? ""], - timeoutMs: 150_000, - }, - ); - expect(agent.exitCode, resultText(agent)).toBe(0); - expect(resultText(agent)).toMatch(/OK/i); - const toolAgent = await sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript( @@ -127,7 +112,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ), { artifactName: "kimi-agent-tool-splitting", - env: env({}, { mode, apiKey }), + env: env({}, { mode, apiKey, includeSecret: true }), redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 420_000, }, diff --git a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts index 15d26c5ef3d..09510772d1d 100644 --- a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts +++ b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { env, + kimiOnboardEnv, requirePublicNvidiaApiKey, resolveKimiInferenceMode, } from "../live/kimi-inference-compat-helpers.ts"; @@ -18,13 +19,26 @@ describe("Kimi inference compatibility mode selection", () => { expect(cfg.NVIDIA_API_KEY).toBeUndefined(); }); - it("preserves public NVIDIA mode using NVIDIA_API_KEY as the source secret", () => { + it("keeps public NVIDIA probe envs secret-free by default", () => { const cfg = env({}, { mode: "public-nvidia", apiKey: "nvapi-public-test-key" }); expect(cfg.NEMOCLAW_E2E_INFERENCE_MODE).toBe("public-nvidia"); expect(cfg.NEMOCLAW_PROVIDER).toBe("cloud"); + expect(cfg.NVIDIA_API_KEY).toBeUndefined(); + expect(cfg.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + expect(cfg.COMPATIBLE_API_KEY).toBeUndefined(); + }); + + it("limits the public NVIDIA source secret to onboard and agent envs", () => { + const cfg = env({}, { + mode: "public-nvidia", + apiKey: "nvapi-public-test-key", + includeSecret: true, + }); expect(cfg.NVIDIA_API_KEY).toBe("nvapi-public-test-key"); expect(cfg.NVIDIA_INFERENCE_API_KEY).toBe("nvapi-public-test-key"); - expect(cfg.COMPATIBLE_API_KEY).toBeUndefined(); + expect(kimiOnboardEnv(undefined, "public-nvidia", "nvapi-public-test-key").NVIDIA_API_KEY).toBe( + "nvapi-public-test-key", + ); }); it("rejects non-public NVIDIA keys for public Kimi validation", () => { From 5d68e8d8ea75c7588c20e9c7e2e876c3f5035c5c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 13:31:21 -0400 Subject: [PATCH 5/7] style(e2e): format kimi helper test --- .../kimi-inference-compat-helpers.test.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts index 09510772d1d..38514b7251f 100644 --- a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts +++ b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts @@ -29,11 +29,14 @@ describe("Kimi inference compatibility mode selection", () => { }); it("limits the public NVIDIA source secret to onboard and agent envs", () => { - const cfg = env({}, { - mode: "public-nvidia", - apiKey: "nvapi-public-test-key", - includeSecret: true, - }); + const cfg = env( + {}, + { + mode: "public-nvidia", + apiKey: "nvapi-public-test-key", + includeSecret: true, + }, + ); expect(cfg.NVIDIA_API_KEY).toBe("nvapi-public-test-key"); expect(cfg.NVIDIA_INFERENCE_API_KEY).toBe("nvapi-public-test-key"); expect(kimiOnboardEnv(undefined, "public-nvidia", "nvapi-public-test-key").NVIDIA_API_KEY).toBe( From 8b0ef1939e8444827b8592f5b5589290c532d81d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 14:01:19 -0400 Subject: [PATCH 6/7] test(e2e): honor explicit kimi mode --- test/e2e-scenario/live/kimi-inference-compat-helpers.ts | 4 ++-- .../support-tests/kimi-inference-compat-helpers.test.ts | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts index 86ce07288ea..bb23f7bd0de 100644 --- a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts +++ b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts @@ -30,8 +30,8 @@ export interface KimiEnvOptions { } export function resolveKimiInferenceMode(env: NodeJS.ProcessEnv = process.env): KimiInferenceMode { - if (env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase() === "public-nvidia") - return "public-nvidia"; + const explicitMode = env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase(); + if (explicitMode === "public-nvidia" || explicitMode === "mock") return explicitMode; if (env.NEMOCLAW_KIMI_USE_MOCK === "0") return "public-nvidia"; return "mock"; } diff --git a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts index 38514b7251f..57dbd365de1 100644 --- a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts +++ b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts @@ -54,6 +54,11 @@ describe("Kimi inference compatibility mode selection", () => { "public-nvidia", ); expect(resolveKimiInferenceMode({ NEMOCLAW_KIMI_USE_MOCK: "0" })).toBe("public-nvidia"); - expect(resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "mock" })).toBe("mock"); + expect( + resolveKimiInferenceMode({ + NEMOCLAW_E2E_INFERENCE_MODE: "mock", + NEMOCLAW_KIMI_USE_MOCK: "0", + }), + ).toBe("mock"); }); }); From f36fef6da16ce37a96c37115a9e767c691480d64 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 15:04:50 -0400 Subject: [PATCH 7/7] test(e2e): close kimi mode review gaps --- .../live/kimi-inference-compat-helpers.ts | 23 +++++++++++++++++-- .../live/kimi-inference-compat.test.ts | 3 ++- .../kimi-inference-compat-helpers.test.ts | 19 ++++++++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts index bb23f7bd0de..35b87b2651b 100644 --- a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts +++ b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts @@ -29,9 +29,21 @@ export interface KimiEnvOptions { includeSecret?: boolean; } +// Source-of-truth boundary for the Kimi public/mock split: trusted CI sets the +// canonical NEMOCLAW_E2E_INFERENCE_MODE selector, local runs default to mock only +// when the selector is absent, and unknown explicit values fail closed so a typo +// cannot downgrade public-NVIDIA validation into hermetic mock coverage. export function resolveKimiInferenceMode(env: NodeJS.ProcessEnv = process.env): KimiInferenceMode { - const explicitMode = env.NEMOCLAW_E2E_INFERENCE_MODE?.trim().toLowerCase(); - if (explicitMode === "public-nvidia" || explicitMode === "mock") return explicitMode; + if (env.NEMOCLAW_E2E_INFERENCE_MODE !== undefined) { + const explicitMode = env.NEMOCLAW_E2E_INFERENCE_MODE.trim().toLowerCase(); + if (explicitMode === "public-nvidia" || explicitMode === "mock") return explicitMode; + throw new Error( + `NEMOCLAW_E2E_INFERENCE_MODE must be one of: mock, public-nvidia; got ${env.NEMOCLAW_E2E_INFERENCE_MODE}`, + ); + } + // Temporary compatibility alias for legacy shell-lane invocations copied from + // test/e2e/test-kimi-inference-compat.sh. NEMOCLAW_E2E_INFERENCE_MODE is the + // canonical selector; remove this alias when the legacy shell lane retires. if (env.NEMOCLAW_KIMI_USE_MOCK === "0") return "public-nvidia"; return "mock"; } @@ -146,6 +158,13 @@ export function kimiOnboardEnv( }); } +export function kimiAgentEnv(mode: KimiInferenceMode): NodeJS.ProcessEnv { + // Onboard owns the only raw public NVIDIA key handoff. After that, the sandbox + // agent must use the configured nvidia-prod route rather than inheriting the + // repository secret in its process environment. + return env({}, { mode }); +} + export async function assertKimiUpstreamTraffic(options: { fake: KimiMock | undefined; host: HostCliClient; diff --git a/test/e2e-scenario/live/kimi-inference-compat.test.ts b/test/e2e-scenario/live/kimi-inference-compat.test.ts index 9649e026304..8b128e17ec6 100644 --- a/test/e2e-scenario/live/kimi-inference-compat.test.ts +++ b/test/e2e-scenario/live/kimi-inference-compat.test.ts @@ -15,6 +15,7 @@ import { cleanupKimi, env, KIMI_MODEL, + kimiAgentEnv, kimiBoundary, kimiOnboardEnv, maybeRegisterKimiMockCleanup, @@ -112,7 +113,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ), { artifactName: "kimi-agent-tool-splitting", - env: env({}, { mode, apiKey, includeSecret: true }), + env: kimiAgentEnv(mode), redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 420_000, }, diff --git a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts index 57dbd365de1..c06d9030e60 100644 --- a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts +++ b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { env, + kimiAgentEnv, kimiOnboardEnv, requirePublicNvidiaApiKey, resolveKimiInferenceMode, @@ -28,7 +29,7 @@ describe("Kimi inference compatibility mode selection", () => { expect(cfg.COMPATIBLE_API_KEY).toBeUndefined(); }); - it("limits the public NVIDIA source secret to onboard and agent envs", () => { + it("limits the public NVIDIA source secret to onboard envs only", () => { const cfg = env( {}, { @@ -42,6 +43,9 @@ describe("Kimi inference compatibility mode selection", () => { expect(kimiOnboardEnv(undefined, "public-nvidia", "nvapi-public-test-key").NVIDIA_API_KEY).toBe( "nvapi-public-test-key", ); + const agentCfg = kimiAgentEnv("public-nvidia"); + expect(agentCfg.NVIDIA_API_KEY).toBeUndefined(); + expect(agentCfg.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); }); it("rejects non-public NVIDIA keys for public Kimi validation", () => { @@ -49,11 +53,10 @@ describe("Kimi inference compatibility mode selection", () => { expect(requirePublicNvidiaApiKey("nvapi-public-test-key")).toBe("nvapi-public-test-key"); }); - it("maps legacy and explicit env selectors to the expected mode", () => { + it("maps canonical explicit env selectors to the expected mode", () => { expect(resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" })).toBe( "public-nvidia", ); - expect(resolveKimiInferenceMode({ NEMOCLAW_KIMI_USE_MOCK: "0" })).toBe("public-nvidia"); expect( resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "mock", @@ -61,4 +64,14 @@ describe("Kimi inference compatibility mode selection", () => { }), ).toBe("mock"); }); + + it("rejects unknown explicit modes instead of silently falling back to mock", () => { + expect(() => resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "public-nvida" })).toThrow( + /must be one of: mock, public-nvidia/, + ); + }); + + it("keeps legacy NEMOCLAW_KIMI_USE_MOCK=0 as temporary public-nvidia alias", () => { + expect(resolveKimiInferenceMode({ NEMOCLAW_KIMI_USE_MOCK: "0" })).toBe("public-nvidia"); + }); });