diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 2def39a7962..d5d607cbcd7 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -920,6 +920,7 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-kimi-compat" + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" OPENSHELL_GATEWAY: "nemoclaw" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -937,6 +938,8 @@ jobs: - name: Install OpenShell CLI run: bash scripts/install-openshell.sh - name: Run Kimi compatibility live Vitest test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" diff --git a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts index cf93aeeb8b7..35b87b2651b 100644 --- a/test/e2e-scenario/live/kimi-inference-compat-helpers.ts +++ b/test/e2e-scenario/live/kimi-inference-compat-helpers.ts @@ -21,6 +21,39 @@ export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-kimi-compa validateSandboxName(SANDBOX_NAME); export const KIMI_MODEL = process.env.NEMOCLAW_KIMI_MODEL ?? "moonshotai/kimi-k2.6"; +export type KimiInferenceMode = "mock" | "public-nvidia"; + +export interface KimiEnvOptions { + mode?: KimiInferenceMode; + apiKey?: string; + includeSecret?: boolean; +} + +// Source-of-truth boundary for the Kimi public/mock split: trusted CI sets the +// canonical NEMOCLAW_E2E_INFERENCE_MODE selector, local runs default to mock only +// when the selector is absent, and unknown explicit values fail closed so a typo +// cannot downgrade public-NVIDIA validation into hermetic mock coverage. +export function resolveKimiInferenceMode(env: NodeJS.ProcessEnv = process.env): KimiInferenceMode { + if (env.NEMOCLAW_E2E_INFERENCE_MODE !== undefined) { + const explicitMode = env.NEMOCLAW_E2E_INFERENCE_MODE.trim().toLowerCase(); + if (explicitMode === "public-nvidia" || explicitMode === "mock") return explicitMode; + throw new Error( + `NEMOCLAW_E2E_INFERENCE_MODE must be one of: mock, public-nvidia; got ${env.NEMOCLAW_E2E_INFERENCE_MODE}`, + ); + } + // Temporary compatibility alias for legacy shell-lane invocations copied from + // test/e2e/test-kimi-inference-compat.sh. NEMOCLAW_E2E_INFERENCE_MODE is the + // canonical selector; remove this alias when the legacy shell lane retires. + if (env.NEMOCLAW_KIMI_USE_MOCK === "0") return "public-nvidia"; + return "mock"; +} + +export function requirePublicNvidiaApiKey(value: string): string { + if (!value.startsWith("nvapi-")) + throw new Error("NVIDIA_API_KEY must be a public NVIDIA Endpoints nvapi-* key"); + return value; +} + export interface KimiRequest { path: string; model?: string; @@ -35,10 +68,13 @@ export interface KimiMock { close(): Promise; } -export function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { +export function env( + extra: NodeJS.ProcessEnv = {}, + options: KimiEnvOptions = {}, +): NodeJS.ProcessEnv { + const mode = options.mode ?? resolveKimiInferenceMode(); + const common: NodeJS.ProcessEnv = { ...buildAvailabilityProbeEnv(), - COMPATIBLE_API_KEY: "test-kimi-key", NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_MODEL: KIMI_MODEL, NEMOCLAW_NON_INTERACTIVE: "1", @@ -47,11 +83,27 @@ export function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { NEMOCLAW_POLICY_MODE: "skip", NEMOCLAW_POLICY_TIER: "restricted", NEMOCLAW_PREFERRED_API: "openai-completions", - NEMOCLAW_PROVIDER: "custom", NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_YES: "1", OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; + if (mode === "public-nvidia") { + return { + ...common, + NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia", + NEMOCLAW_PROVIDER: "cloud", + ...(options.includeSecret && options.apiKey + ? { NVIDIA_API_KEY: options.apiKey, NVIDIA_INFERENCE_API_KEY: options.apiKey } + : {}), + ...extra, + }; + } + return { + ...common, + COMPATIBLE_API_KEY: "test-kimi-key", + NEMOCLAW_E2E_INFERENCE_MODE: "mock", + NEMOCLAW_PROVIDER: "custom", ...extra, }; } @@ -77,6 +129,75 @@ export async function startKimiMock(): Promise { }; } +export function kimiBoundary(mode: KimiInferenceMode): string { + return mode === "public-nvidia" + ? "source CLI onboard + public NVIDIA Kimi endpoint + OpenClaw config/plugin/inference route" + : "source CLI onboard + fake OpenAI-compatible Kimi endpoint + OpenClaw config/plugin/inference route"; +} + +export async function startKimiUpstream(mode: KimiInferenceMode): Promise { + return mode === "mock" ? startKimiMock() : undefined; +} + +export function maybeRegisterKimiMockCleanup( + cleanup: { add(name: string, fn: () => Promise): void }, + fake: KimiMock | undefined, +): void { + if (fake) cleanup.add("close fake Kimi endpoint", () => fake.close()); +} + +export function kimiOnboardEnv( + fake: KimiMock | undefined, + mode: KimiInferenceMode, + apiKey: string | undefined, +): NodeJS.ProcessEnv { + return env(fake ? { NEMOCLAW_ENDPOINT_URL: fake.baseUrl } : {}, { + mode, + apiKey, + includeSecret: true, + }); +} + +export function kimiAgentEnv(mode: KimiInferenceMode): NodeJS.ProcessEnv { + // Onboard owns the only raw public NVIDIA key handoff. After that, the sandbox + // agent must use the configured nvidia-prod route rather than inheriting the + // repository secret in its process environment. + return env({}, { mode }); +} + +export async function assertKimiUpstreamTraffic(options: { + fake: KimiMock | undefined; + host: HostCliClient; + mode: KimiInferenceMode; + apiKey: string | undefined; +}): Promise { + if (options.fake) { + expect( + options.fake.requests.some( + (request) => + request.authOk && + request.path.includes("/chat/completions") && + request.model === KIMI_MODEL && + request.hasTools, + ), + ).toBe(true); + expect(options.fake.requests.some((request) => request.authOk && request.hasToolResult)).toBe( + true, + ); + return; + } + + const route = await options.host.command("openshell", ["inference", "get", "-g", "nemoclaw"], { + artifactName: "public-nvidia-kimi-route", + env: env({}, { mode: options.mode, apiKey: options.apiKey }), + redactionValues: [options.apiKey ?? ""], + timeoutMs: 60_000, + }); + expect(route.exitCode, resultText(route)).toBe(0); + expect(resultText(route)).toContain("nvidia-prod"); + expect(resultText(route)).toContain(KIMI_MODEL); +} + function handleKimiRequest( req: http.IncomingMessage, res: http.ServerResponse, @@ -302,10 +423,7 @@ export function parseConfig(raw: string): { } export async function assertTrajectory(sandbox: SandboxClient): Promise { - const trajectory = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript(String.raw`python3 - <<'PY' -import json, pathlib, sys + const checkScript = String.raw`import json, pathlib, sys root=pathlib.Path('/sandbox/.openclaw') base=pathlib.Path('/sandbox/.openclaw/agents/main/sessions') session=base/'e2e-kimi-tools.jsonl' @@ -344,8 +462,11 @@ if not final_texts or normalize_final_text(final_texts[-1]) != 'hostname, date, roles=[m.get('role') for m in messages] if not ('toolResult' in roles and roles[-1]=='assistant'): errors.append('final assistant not after tool result') print(json.dumps({'errors':errors,'source':source,'toolMetas':metas,'roles':roles}, indent=2)) -sys.exit(1 if errors else 0) -PY`), +sys.exit(1 if errors else 0)`; + const encoded = Buffer.from(checkScript, "utf8").toString("base64"); + const trajectory = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript(`python3 -c "$(printf %s '${encoded}' | base64 -d)"`), { artifactName: "kimi-trajectory-tool-splitting-check", env: env(), timeoutMs: 60_000 }, ); expect(trajectory.exitCode, resultText(trajectory)).toBe(0); diff --git a/test/e2e-scenario/live/kimi-inference-compat.test.ts b/test/e2e-scenario/live/kimi-inference-compat.test.ts index dbda023a005..8b128e17ec6 100644 --- a/test/e2e-scenario/live/kimi-inference-compat.test.ts +++ b/test/e2e-scenario/live/kimi-inference-compat.test.ts @@ -9,15 +9,22 @@ import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; import { + assertKimiUpstreamTraffic, assertTrajectory, CLI, cleanupKimi, env, KIMI_MODEL, + kimiAgentEnv, + kimiBoundary, + kimiOnboardEnv, + maybeRegisterKimiMockCleanup, parseConfig, REPO_ROOT, + requirePublicNvidiaApiKey, + resolveKimiInferenceMode, SANDBOX_NAME, - startKimiMock, + startKimiUpstream, } from "./kimi-inference-compat-helpers.ts"; const TIMEOUT_MS = 40 * 60_000; @@ -25,16 +32,22 @@ const TIMEOUT_MS = 40 * 60_000; test.skipIf(!shouldRunLiveE2EScenarios())( "Kimi-compatible endpoint config enables plugin wiring and managed inference route", { timeout: TIMEOUT_MS }, - async ({ artifacts, cleanup, host, sandbox }) => { - const fake = await startKimiMock(); - cleanup.add("close fake Kimi endpoint", () => fake.close()); + async ({ artifacts, cleanup, host, sandbox, secrets }) => { + const mode = resolveKimiInferenceMode(); + const apiKey = + mode === "public-nvidia" + ? requirePublicNvidiaApiKey(secrets.required("NVIDIA_API_KEY")) + : undefined; + const fake = await startKimiUpstream(mode); + maybeRegisterKimiMockCleanup(cleanup, fake); cleanup.add("destroy Kimi sandbox", () => cleanupKimi(host, sandbox)); await artifacts.writeJson("scenario.json", { id: "kimi-inference-compat", legacySource: "test/e2e/test-kimi-inference-compat.sh", - boundary: - "source CLI onboard + fake OpenAI-compatible Kimi endpoint + OpenClaw config/plugin/inference route", + boundary: kimiBoundary(mode), + inferenceClassification: "public-nvidia required with mock/hermetic fallback", + inferenceMode: mode, sandboxName: SANDBOX_NAME, model: KIMI_MODEL, }); @@ -54,8 +67,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())( { artifactName: "onboard-kimi-compatible", cwd: REPO_ROOT, - env: env({ NEMOCLAW_ENDPOINT_URL: fake.baseUrl }), - redactionValues: ["test-kimi-key"], + env: kimiOnboardEnv(fake, mode, apiKey), + redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 20 * 60_000, }, ); @@ -63,7 +76,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { artifactName: "openclaw-config", - env: env(), + env: env({}, { mode }), timeoutMs: 60_000, }); expect(config.exitCode, resultText(config)).toBe(0); @@ -88,26 +101,11 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const modelsRoute = await sandbox.exec( SANDBOX_NAME, ["curl", "-sk", "--max-time", "20", "https://inference.local/v1/models"], - { artifactName: "inference-local-models", env: env(), timeoutMs: 60_000 }, + { artifactName: "inference-local-models", env: env({}, { mode }), timeoutMs: 60_000 }, ); expect(modelsRoute.exitCode, resultText(modelsRoute)).toBe(0); expect(resultText(modelsRoute)).toContain(KIMI_MODEL); - const agent = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - "openclaw agent --agent main --json --session-id e2e-kimi-compat -m 'Reply with exactly: OK'", - ), - { - artifactName: "kimi-agent-smoke", - env: env(), - redactionValues: ["test-kimi-key"], - timeoutMs: 150_000, - }, - ); - expect(agent.exitCode, resultText(agent)).toBe(0); - expect(resultText(agent)).toMatch(/OK/i); - const toolAgent = await sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript( @@ -115,22 +113,13 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ), { artifactName: "kimi-agent-tool-splitting", - env: env(), - redactionValues: ["test-kimi-key"], + env: kimiAgentEnv(mode), + redactionValues: ["test-kimi-key", apiKey ?? ""], timeoutMs: 420_000, }, ); expect(toolAgent.exitCode, resultText(toolAgent)).toBe(0); await assertTrajectory(sandbox); - expect( - fake.requests.some( - (request) => - request.authOk && - request.path.includes("/chat/completions") && - request.model === KIMI_MODEL && - request.hasTools, - ), - ).toBe(true); - expect(fake.requests.some((request) => request.authOk && request.hasToolResult)).toBe(true); + await assertKimiUpstreamTraffic({ fake, host, mode, apiKey }); }, ); diff --git a/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts new file mode 100644 index 00000000000..c06d9030e60 --- /dev/null +++ b/test/e2e-scenario/support-tests/kimi-inference-compat-helpers.test.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + env, + kimiAgentEnv, + kimiOnboardEnv, + requirePublicNvidiaApiKey, + resolveKimiInferenceMode, +} from "../live/kimi-inference-compat-helpers.ts"; + +describe("Kimi inference compatibility mode selection", () => { + it("defaults to hermetic mock mode for local validation", () => { + const cfg = env({}, { mode: "mock" }); + expect(cfg.NEMOCLAW_E2E_INFERENCE_MODE).toBe("mock"); + expect(cfg.NEMOCLAW_PROVIDER).toBe("custom"); + expect(cfg.COMPATIBLE_API_KEY).toBe("test-kimi-key"); + expect(cfg.NVIDIA_API_KEY).toBeUndefined(); + }); + + it("keeps public NVIDIA probe envs secret-free by default", () => { + const cfg = env({}, { mode: "public-nvidia", apiKey: "nvapi-public-test-key" }); + expect(cfg.NEMOCLAW_E2E_INFERENCE_MODE).toBe("public-nvidia"); + expect(cfg.NEMOCLAW_PROVIDER).toBe("cloud"); + expect(cfg.NVIDIA_API_KEY).toBeUndefined(); + expect(cfg.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + expect(cfg.COMPATIBLE_API_KEY).toBeUndefined(); + }); + + it("limits the public NVIDIA source secret to onboard envs only", () => { + const cfg = env( + {}, + { + mode: "public-nvidia", + apiKey: "nvapi-public-test-key", + includeSecret: true, + }, + ); + expect(cfg.NVIDIA_API_KEY).toBe("nvapi-public-test-key"); + expect(cfg.NVIDIA_INFERENCE_API_KEY).toBe("nvapi-public-test-key"); + expect(kimiOnboardEnv(undefined, "public-nvidia", "nvapi-public-test-key").NVIDIA_API_KEY).toBe( + "nvapi-public-test-key", + ); + const agentCfg = kimiAgentEnv("public-nvidia"); + expect(agentCfg.NVIDIA_API_KEY).toBeUndefined(); + expect(agentCfg.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + }); + + it("rejects non-public NVIDIA keys for public Kimi validation", () => { + expect(() => requirePublicNvidiaApiKey("sk-compatible-key")).toThrow(/nvapi-\* key/); + expect(requirePublicNvidiaApiKey("nvapi-public-test-key")).toBe("nvapi-public-test-key"); + }); + + it("maps canonical explicit env selectors to the expected mode", () => { + expect(resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "public-nvidia" })).toBe( + "public-nvidia", + ); + expect( + resolveKimiInferenceMode({ + NEMOCLAW_E2E_INFERENCE_MODE: "mock", + NEMOCLAW_KIMI_USE_MOCK: "0", + }), + ).toBe("mock"); + }); + + it("rejects unknown explicit modes instead of silently falling back to mock", () => { + expect(() => resolveKimiInferenceMode({ NEMOCLAW_E2E_INFERENCE_MODE: "public-nvida" })).toThrow( + /must be one of: mock, public-nvidia/, + ); + }); + + it("keeps legacy NEMOCLAW_KIMI_USE_MOCK=0 as temporary public-nvidia alias", () => { + expect(resolveKimiInferenceMode({ NEMOCLAW_KIMI_USE_MOCK: "0" })).toBe("public-nvidia"); + }); +}); diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index f966b53f988..d9cb24c3fe9 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -584,6 +584,20 @@ describe("E2E reusable workflow contract", () => { ); }); + it("uses NVIDIA_API_KEY for the live Kimi Vitest lane", () => { + const vitestWorkflow = readYaml<{ jobs: Record }>( + ".github/workflows/e2e-vitest-scenarios.yaml", + ); + const job = vitestWorkflow.jobs["kimi-inference-compat-vitest"]; + const runStep = job.steps?.find( + (step) => step.name === "Run Kimi compatibility live Vitest test", + ); + + expect(job.env?.NEMOCLAW_E2E_INFERENCE_MODE).toBe("public-nvidia"); + expect(runStep?.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); + expect(runStep?.env?.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + }); + it("authenticates Docker Hub pulls in direct nightly E2E jobs", () => { const directE2eJobs = [ "openclaw-tui-chat-correlation-e2e", diff --git a/test/helpers/e2e-workflow-contract.ts b/test/helpers/e2e-workflow-contract.ts index f8ff5460850..5537f7e8800 100644 --- a/test/helpers/e2e-workflow-contract.ts +++ b/test/helpers/e2e-workflow-contract.ts @@ -12,6 +12,7 @@ export type WorkflowJob = { "runs-on"?: string; "timeout-minutes"?: number; uses?: string; + env?: Record; secrets?: Record; steps?: WorkflowStep[]; with?: Record;