diff --git a/ci/e2e-assertion-budget.json b/ci/e2e-assertion-budget.json index 749d971ec9e..1cce311ab2b 100644 --- a/ci/e2e-assertion-budget.json +++ b/ci/e2e-assertion-budget.json @@ -15,26 +15,26 @@ "testFileCount": 86, "liveFileCount": 222, "direct": { - "expectCalls": 1797, - "matcherAssertions": 1766, + "expectCalls": 1796, + "matcherAssertions": 1765, "nodeAssertions": 105, "namedAssertionHelpers": 601, "failCalls": 8, "throwGuards": 87, "objectFieldAssertions": 243, - "assertionPoints": 2810, + "assertionPoints": 2809, "generatedProbeBlocks": 122, "generatedProbeConditions": 322 }, "unique": { - "expectCalls": 2289, - "matcherAssertions": 2253, + "expectCalls": 2288, + "matcherAssertions": 2252, "nodeAssertions": 126, "namedAssertionHelpers": 905, "failCalls": 38, "throwGuards": 633, "objectFieldAssertions": 342, - "assertionPoints": 4297, + "assertionPoints": 4296, "generatedProbeBlocks": 276, "generatedProbeConditions": 943 }, @@ -65,7 +65,7 @@ "test/e2e/live/full-e2e.test.ts": [30,34,39,72,7], "test/e2e/live/gateway-guard-recovery.test.ts": [48,54,48,57,3], "test/e2e/live/gpu-double-onboard.test.ts": [21,24,21,24,0], - "test/e2e/live/gpu-e2e.test.ts": [40,45,73,91,3], + "test/e2e/live/gpu-e2e.test.ts": [39,44,72,90,3], "test/e2e/live/hermes-discord.test.ts": [10,25,17,64,14], "test/e2e/live/hermes-e2e.test.ts": [82,93,112,127,3], "test/e2e/live/hermes-gpu-startup.test.ts": [19,27,63,106,12], diff --git a/schemas/nemoclaw-config-v1.schema.json b/schemas/nemoclaw-config-v1.schema.json index ee20d93fc0c..87c5c2861b8 100644 --- a/schemas/nemoclaw-config-v1.schema.json +++ b/schemas/nemoclaw-config-v1.schema.json @@ -218,6 +218,85 @@ } }, "additionalProperties": false + }, + { + "type": "object", + "required": ["name", "provider", "api", "serving"], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$" + }, + "provider": { + "type": "string", + "const": "ollama-local" + }, + "api": { + "type": "string", + "const": "openai-completions" + }, + "serving": { + "type": "object", + "required": ["backend", "daemon", "proxy", "model"], + "properties": { + "backend": { + "type": "string", + "const": "ollama" + }, + "daemon": { + "type": "object", + "required": ["management", "hostPort"], + "properties": { + "management": { + "type": "string", + "const": "external" + }, + "hostPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "additionalProperties": false + }, + "proxy": { + "type": "object", + "required": ["management", "hostPort"], + "properties": { + "management": { + "type": "string", + "const": "nemoclaw" + }, + "hostPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "additionalProperties": false + }, + "model": { + "type": "object", + "required": ["servedName", "digest"], + "properties": { + "servedName": { + "type": "string", + "const": "qwen3.5:9b" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } ] }, diff --git a/scripts/ollama-auth-proxy.mts b/scripts/ollama-auth-proxy.mts index 513292806a2..d22b96e3738 100755 --- a/scripts/ollama-auth-proxy.mts +++ b/scripts/ollama-auth-proxy.mts @@ -342,7 +342,7 @@ function assertBackendBoundToLoopback(port: number): void { function buildProxyServer(token: string, backendUrl: URL): http.Server { const expectedBuf = Buffer.from(`Bearer ${token}`); - return http.createServer((clientReq, clientRes) => { + const server = http.createServer((clientReq, clientRes) => { // Every request must present a valid Bearer token. The proxy binds 0.0.0.0 // so the OpenShell sandbox container can reach it via the docker bridge — // which also means anything else with network reach to the host could, @@ -366,6 +366,33 @@ function buildProxyServer(token: string, backendUrl: URL): http.Server { return; } + if (clientReq.url === "/_nemoclaw/proxy-config") { + if (clientReq.method !== "GET") { + clientRes.writeHead(405, { Allow: "GET" }); + clientRes.end(); + return; + } + const listener = server.address(); + if (Buffer.byteLength(backendUrl.origin) > 512) { + clientRes.writeHead(503); + clientRes.end(); + return; + } + clientRes.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" }); + clientRes.end( + JSON.stringify({ + schemaVersion: 1, + pid: process.pid, + listener: + typeof listener === "object" && listener !== null + ? { address: listener.address, port: listener.port } + : null, + backendOrigin: backendUrl.origin, + }), + ); + return; + } + // Strip the auth header before forwarding to Ollama const headers = { ...clientReq.headers }; delete headers.authorization; @@ -407,6 +434,7 @@ function buildProxyServer(token: string, backendUrl: URL): http.Server { clientReq.pipe(proxyReq); }); + return server; } function shouldProbeBackendHostname(hostname: string): boolean { diff --git a/src/lib/actions/config/observe-export-source.ts b/src/lib/actions/config/observe-export-source.ts index 1a76b25431b..9aec67b4fe0 100644 --- a/src/lib/actions/config/observe-export-source.ts +++ b/src/lib/actions/config/observe-export-source.ts @@ -86,6 +86,7 @@ const LIVE_READ_SOURCE_LABELS = { "provider-metadata": "live inference provider metadata", "web-search-provider": "live web-search provider metadata", "managed-serving": "managed serving runtime", + "ollama-serving": "live Ollama daemon and proxy mapping", "effective-policy": "effective OpenShell policy", } satisfies Readonly>; diff --git a/src/lib/adapters/config/live-export-source-test-fixture.ts b/src/lib/adapters/config/live-export-source-test-fixture.ts index 63d23c194ec..7bd2324e3d4 100644 --- a/src/lib/adapters/config/live-export-source-test-fixture.ts +++ b/src/lib/adapters/config/live-export-source-test-fixture.ts @@ -1,6 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; + +import { EXPORTED_OLLAMA_MODEL } from "../../config/model"; +import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "../../inference/ollama/contract"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; +import { resolveManagedStartupInferenceRoute } from "../../inference/gateway/route-contract"; import { buildManagedStartupProfile } from "../../onboard/managed-startup/profile-builder"; import type { ManagedStartupProfileBuilderInput } from "../../onboard/managed-startup/profile-builder"; import type { SandboxEntry } from "../../state/registry/types"; @@ -126,3 +132,118 @@ export function configuration(revision = 3) { globalPolicyVersion: 0, }; } + +export function ollamaSource(model = EXPORTED_OLLAMA_MODEL) { + const route = resolveManagedStartupInferenceRoute( + "openclaw", + "ollama-local", + model, + "openai-completions", + ); + const built = buildManagedStartupProfile({ + ...startupInput, + inference: { + routeProvider: route.providerKey, + upstreamProvider: "ollama-local", + model, + routedBaseUrl: route.inferenceBaseUrl, + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: route.primaryModelRef, + compatibility: route.inferenceCompat ?? {}, + }, + }); + const source: SandboxEntry = { + ...entry, + provider: "ollama-local", + model, + endpointUrl: "http://host.openshell.internal:11440/v1", + credentialEnv: OLLAMA_LOCAL_CREDENTIAL_ENV, + workload: { + ...entry.workload, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + }, + }; + const observed: ObservedOllamaProxy = { + pid: 1234, + listenerAddress: "0.0.0.0", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { servedName: model, digest: `sha256:${"a".repeat(64)}` }, + }, + }; + return { source, observed }; +} + +export function telemetryEntry( + telemetry: Readonly> = {}, + agentSettings: Readonly> = {}, +) { + const profile = JSON.parse(Buffer.from(startup.encodedProfile, "base64url").toString("utf8")) as { + agentConfig: { otel: Record }; + }; + Object.assign(profile.agentConfig.otel, { + enabled: true, + serviceName: "research-assistant", + sampleRate: 0.5, + ...telemetry, + }); + Object.assign(profile.agentConfig, agentSettings); + const encodedProfile = Buffer.from(JSON.stringify(profile)).toString("base64url"); + return { + ...entry, + workload: { + ...entry.workload, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile).digest("hex"), + }, + }; +} + +export function dashboardSource() { + const built = buildManagedStartupProfile({ + ...startupInput, + toolDisclosure: "direct", + dashboard: { + agent: "openclaw" as const, + mode: "remote" as const, + url: "http://127.0.0.1:19000", + port: 19000, + bindAddress: "0.0.0.0" as const, + wslExposure: false, + }, + }); + return { + ...entry, + toolDisclosure: "direct" as const, + dashboardPort: 19000, + dashboardRemoteBindPrepared: true, + workload: { + ...entry.workload, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + }, + }; +} + +export function openAiProviderProfile() { + return { + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: 4n, + credentials: [], + endpoints: [], + binaries: [], + inferenceCapable: true, + }, + }; +} + +export function nativeNvidiaProvider() { + return { ...provider().provider, type: "nvidia", profileWorkspace: "", config: {} }; +} diff --git a/src/lib/adapters/config/live-export-source.test.ts b/src/lib/adapters/config/live-export-source.test.ts index fc6666e00f7..0534cbff9d1 100644 --- a/src/lib/adapters/config/live-export-source.test.ts +++ b/src/lib/adapters/config/live-export-source.test.ts @@ -1,12 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; +import os from "node:os"; import { describe, expect, it, vi } from "vitest"; import YAML from "yaml"; import { managedBraveProfile } from "../../../../test/fixtures/openshell-provider-profile"; import { runConfigExport } from "../../actions/config/export"; import { + EXPORTED_OLLAMA_MODEL, parseNemoClawConfigDocumentName, EXPORTED_VLLM_PROFILE_ID, EXPORTED_VLLM_RECIPE_ID, @@ -18,6 +19,11 @@ import { validateNemoClawConfig } from "../../config/schema"; vi.mock("../../inference/serving/vllm-export-runtime", () => ({ observeManagedVllmForExport: vi.fn(), })); +vi.mock("../../inference/ollama/proxy", () => ({ createOllamaExportProbe: vi.fn() })); +vi.mock("../../platform", async (importOriginal) => ({ + ...(await importOriginal()), + isWsl: vi.fn(() => false), +})); vi.mock("../../state/registry/persistence", () => ({ load: vi.fn() })); vi.mock("../../state/registry-entry-view", () => ({ getSandboxEntryInference: vi.fn() })); vi.mock("../../inference/live", () => ({ getLiveGatewayInference: vi.fn() })); @@ -39,6 +45,9 @@ vi.mock("../../onboard/gateway/state-dir", () => ({ })); import { observeManagedVllmForExport } from "../../inference/serving/vllm-export-runtime"; +import { createOllamaExportProbe } from "../../inference/ollama/proxy"; +import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "../../inference/ollama/contract"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; import { loadServingCatalog } from "../../inference/serving/catalog-loader"; import { servingProfileProvenance } from "../../inference/serving/profile-provenance"; import { applyVllmRuntimeContextWindow } from "../../inference/vllm-runtime-context"; @@ -48,7 +57,6 @@ import { getLiveGatewayInference } from "../../inference/live"; import { resolveGatewayStateDirForPort } from "../../onboard/gateway/state-dir"; import { buildManagedStartupProfile } from "../../onboard/managed-startup/profile-builder"; import type { ManagedStartupProfileBuilderInput } from "../../onboard/managed-startup/profile-builder"; -import { encodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; import { getSandboxEntryInference } from "../../state/registry-entry-view"; import { load as loadRegistry } from "../../state/registry/persistence"; import type { SandboxEntry } from "../../state/registry/types"; @@ -56,44 +64,22 @@ import { connectManagedOpenShellSdk } from "../openshell/sdk"; import { observeStableExportSource } from "../../actions/config/observe-export-source"; import { captureSanitizedResolvedOpenshell } from "../openshell/sanitized-capture"; import { createLiveExportSnapshotReader } from "./live-export-source"; - import { endpoint, readFailureCanary, imageRef, startupInput, - startup, entry, inventory, provider, configuration, + openAiProviderProfile, + nativeNvidiaProvider, + ollamaSource, + telemetryEntry, + dashboardSource, } from "./live-export-source-test-fixture"; -function telemetryEntry( - telemetry: Readonly> = {}, - agentSettings: Readonly> = {}, -) { - const profile = JSON.parse(Buffer.from(startup.encodedProfile, "base64url").toString("utf8")) as { - agentConfig: { otel: Record }; - }; - Object.assign(profile.agentConfig.otel, { - enabled: true, - serviceName: "research-assistant", - sampleRate: 0.5, - ...telemetry, - }); - Object.assign(profile.agentConfig, agentSettings); - const encodedProfile = Buffer.from(JSON.stringify(profile)).toString("base64url"); - return { - ...entry, - workload: { - ...entry.workload, - encodedProfile, - startupProfileSha256: createHash("sha256").update(encodedProfile).digest("hex"), - }, - }; -} - const raw = { getProvider: vi.fn(), getProviderProfile: vi.fn(), @@ -203,10 +189,20 @@ async function exportLiveSource() { return { result, writeStdout, publish }; } -function nativeNvidiaProvider() { - return { ...provider().provider, type: "nvidia", profileWorkspace: "", config: {} }; +function expectExportRefusal( + exported: Awaited>, + finding: Readonly<{ field?: string; category: string }>, +) { + expect(exported.result).toMatchObject({ + ok: false, + failure: { + kind: "observation", + findings: expect.arrayContaining([expect.objectContaining(finding)]), + }, + }); + expect(exported.writeStdout).not.toHaveBeenCalled(); + expect(exported.publish).not.toHaveBeenCalled(); } - function mockNativeNvidiaSource() { mockSupportedLiveSource(); raw.getProvider.mockResolvedValue({ provider: nativeNvidiaProvider() }); @@ -975,7 +971,7 @@ function mockManagedVllmSource( }); Object.assign(environment, environmentOverrides); const built = buildManagedStartupProfile({ - agent: "openclaw", + ...startupInput, inference: { routeProvider: inference.providerKey, upstreamProvider: "vllm-local", @@ -986,21 +982,8 @@ function mockManagedVllmSource( primaryModelRef: inference.primaryModelRef, compatibility: inference.inferenceCompat ?? {}, }, - dashboard: { - agent: "openclaw", - mode: "loopback", - url: "http://127.0.0.1:18789", - port: 18789, - bindAddress: "127.0.0.1", - wslExposure: false, - }, webSearch, toolDisclosure, - hermesToolGateways: [], - messagingPlan: null, - dcodeAutoApprovalMode: null, - observabilityEnabled: null, - corporateCa: null, environment, }); const source: SandboxEntry = { @@ -1065,18 +1048,7 @@ function mockManagedVllmSource( config: { OPENAI_BASE_URL: source.endpointUrl }, }, }); - raw.getProviderProfile.mockResolvedValue({ - profile: { - id: "openai", - source: "user", - scope: "workspace", - resourceVersion: 4n, - credentials: [], - endpoints: [], - binaries: [], - inferenceCapable: true, - }, - }); + raw.getProviderProfile.mockResolvedValue(openAiProviderProfile()); return { source, observed }; } @@ -1299,37 +1271,147 @@ describe("managed vLLM export pipeline", () => { }); }); +function ollamaProbe(observed: ObservedOllamaProxy) { + const models = JSON.stringify({ + models: [{ name: observed.serving.model.servedName, digest: observed.serving.model.digest }], + }); + return { + backend: { + kind: "ollama" as const, + url: `http://127.0.0.1:${observed.serving.daemon.hostPort}`, + }, + proxyPort: String(observed.serving.proxy.hostPort), + pid: String(observed.pid), + processMatches: vi.fn(() => true), + readActiveConfig: vi.fn(() => + JSON.stringify({ + schemaVersion: 1, + pid: observed.pid, + listener: { address: observed.listenerAddress, port: observed.serving.proxy.hostPort }, + backendOrigin: `http://127.0.0.1:${observed.serving.daemon.hostPort}`, + }), + ), + readProxyModels: vi.fn(() => models), + readDaemonModels: vi.fn(() => models), + }; +} + +function mockOllamaSource() { + vi.spyOn(os, "platform").mockReturnValue("linux"); + const model = EXPORTED_OLLAMA_MODEL; + const { source, observed } = ollamaSource(model); + mockSupportedLiveSource(3, 3, source); + const probe = ollamaProbe(observed); + vi.mocked(createOllamaExportProbe).mockReturnValue(probe); + vi.mocked(getSandboxEntryInference).mockReturnValue({ + kind: "configured", + provider: "ollama-local", + model, + }); + vi.mocked(getLiveGatewayInference).mockReturnValue({ + failure: null, + inference: { provider: "ollama-local", model }, + output: "", + status: 0, + }); + const liveSandbox = inventory(); + Object.assign(liveSandbox.sandbox.spec, { providers: ["ollama-local"] }); + raw.getSandbox.mockResolvedValue(liveSandbox); + const readCredential = vi.fn(() => { + throw new Error(readFailureCanary); + }); + const credentials = Object.defineProperty({}, OLLAMA_LOCAL_CREDENTIAL_ENV, { + enumerable: true, + get: readCredential, + }); + const localProvider = { + ...provider().provider, + metadata: { ...provider().provider.metadata, name: "ollama-local" }, + profileWorkspace: "default", + credentials, + config: { OPENAI_BASE_URL: source.endpointUrl }, + }; + raw.getProvider.mockResolvedValue({ provider: localProvider }); + raw.getProviderProfile.mockResolvedValue(openAiProviderProfile()); + return { source, observed, probe, readCredential, localProvider }; +} + +describe("attached Ollama export pipeline", () => { + it("publishes the complete managed document without reading gateway credentials (#11435)", async () => { + const { observed, probe, readCredential } = mockOllamaSource(); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); + const yaml = writeStdout.mock.calls[0]![0]; + const document = validateNemoClawConfig(YAML.parse(yaml)); + expect(document.spec.inferenceProviders).toEqual([ + { + name: "local-ollama", + provider: "ollama-local", + api: "openai-completions", + serving: observed.serving, + }, + ]); + expect(document.spec.sandboxes[0]!.agents[0]!.inference.routes).toEqual([ + { name: "primary", providerRef: "local-ollama", overrides: { model: EXPORTED_OLLAMA_MODEL } }, + ]); + expect(probe.readActiveConfig).toHaveBeenCalledWith(11440); + expect(probe.readDaemonModels).toHaveBeenCalledWith(11439); + expect(readCredential).not.toHaveBeenCalled(); + expect(yaml).not.toMatch( + /NEMOCLAW_OLLAMA_PROXY_TOKEN|credential-canary-value|host\.openshell\.internal/u, + ); + expect(publish).not.toHaveBeenCalled(); + }); + it("sanitizes a failed or legacy proxy observation and publishes nothing (#11435)", async () => { + mockOllamaSource(); + vi.mocked(createOllamaExportProbe).mockImplementation(() => { + throw new Error(readFailureCanary); + }); + const { result, writeStdout } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(JSON.stringify(result)).not.toContain(readFailureCanary); + expect(writeStdout).not.toHaveBeenCalled(); + }); + + it.each([ + [ + { endpointUrl: "http://host.openshell.internal:11435/v1" }, + { field: "spec.inferenceProviders[].endpoint", category: "drifted" }, + ], + [ + { credentialEnv: "OTHER_TOKEN" }, + { field: "source.live", category: "live-verification-failed" }, + ], + [{ agent: "hermes" }, { field: "spec.inferenceProviders[].serving", category: "drifted" }], + [ + { sandboxGpuEnabled: true, sandboxGpuDevice: "nvidia.com/gpu=all" }, + { field: "spec.sandboxes[].runtime.gpu", category: "unsupported" }, + ], + ])("refuses unsupported or drifted local route intent %# (#11435)", async (change, finding) => { + const { source } = mockOllamaSource(); + Object.assign(source, change); + expectExportRefusal(await exportLiveSource(), finding); + }); + it("refuses an absent provider attachment (#11435)", async () => { + mockOllamaSource(); + raw.getSandbox.mockResolvedValue(inventory()); + expectExportRefusal(await exportLiveSource(), { + field: "spec.inferenceProviders[].serving", + category: "drifted", + }); + }); + it("refuses a continuously changing active proxy identity (#11435)", async () => { + const { observed } = mockOllamaSource(); + let pid = observed.pid; + vi.mocked(createOllamaExportProbe).mockImplementation(() => + ollamaProbe({ ...observed, pid: ++pid }), + ); + expectExportRefusal(await exportLiveSource(), { category: "unstable-source" }); + }); +}); describe("dashboard export observation", () => { it("projects registered dashboard and direct tools through complete live observation (#10904)", async () => { - const workload = entry.workload as Extract< - NonNullable, - { kind: "managed-image" } - >; - expect(workload?.kind).toBe("managed-image"); - const profile = { - ...startup.profile, - tools: { ...startup.profile.tools, disclosure: "direct" as const }, - dashboard: { - agent: "openclaw" as const, - mode: "remote" as const, - url: "http://127.0.0.1:19000", - port: 19000, - bindAddress: "0.0.0.0" as const, - wslExposure: false, - }, - }; - const encodedProfile = encodeManagedStartupProfile(profile); - const sourceEntry = { - ...entry, - toolDisclosure: "direct" as const, - dashboardPort: 19000, - dashboardRemoteBindPrepared: true, - workload: { - ...workload, - encodedProfile, - startupProfileSha256: createHash("sha256").update(encodedProfile).digest("hex"), - }, - }; + const sourceEntry = dashboardSource(); mockSupportedLiveSource(3, 3, sourceEntry); const reader = createLiveExportSnapshotReader(); const observed = await reader.read("alpha"); @@ -1357,7 +1439,6 @@ describe("dashboard export observation", () => { }, ); expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); - expect(raw.getSandbox).toHaveBeenCalledTimes(3); expect(JSON.stringify(result)).not.toContain(readFailureCanary); const yaml = writeStdout.mock.calls[0]?.[0] ?? ""; expect(yaml).not.toContain(readFailureCanary); @@ -1369,20 +1450,15 @@ describe("dashboard export observation", () => { }); }); - it("retains dashboard registry changes in both complete snapshots (#10904)", async () => { + it("refuses dashboard registry changes without publishing (#10904)", async () => { mockSupportedLiveSource(); let reads = 0; vi.mocked(loadRegistry).mockImplementation(() => ({ sandboxes: { alpha: { ...entry, dashboardPort: reads++ % 2 === 0 ? 18789 : 19000 } }, defaultSandbox: null, })); - const result = await observeStableExportSource("alpha", createLiveExportSnapshotReader()); - expect(result).toMatchObject({ - ok: false, - attempts: 2, - findings: [expect.objectContaining({ category: "unstable-source" })], - }); - expect(loadRegistry).toHaveBeenCalledTimes(4); + const exported = await exportLiveSource(); + expectExportRefusal(exported, { category: "unstable-source" }); }); }); diff --git a/src/lib/adapters/config/live-export-source.ts b/src/lib/adapters/config/live-export-source.ts index a52c07192ea..e3aad21d2ac 100644 --- a/src/lib/adapters/config/live-export-source.ts +++ b/src/lib/adapters/config/live-export-source.ts @@ -27,6 +27,8 @@ import type { import { getLiveGatewayInference } from "../../inference/live"; import { VLLM_LOCAL_CREDENTIAL_ENV } from "../../inference/serving/vllm-credential-contract"; import { observeManagedVllmForExport } from "../../inference/serving/vllm-export-runtime"; +import { createOllamaExportProbe } from "../../inference/ollama/proxy"; +import { observeOllamaProxy } from "../../inference/ollama/proxy-observation"; import { normalizeInferenceSelection } from "../../inference/selection"; import { resolveGatewayName } from "../../onboard/gateway-binding/identity"; import { @@ -149,11 +151,34 @@ function inferenceTopology( managed: boolean, ): ObservedExportInference["topology"] { if (managed) return "managed"; + if (entry.provider === "ollama-local") return "local"; return entry.hostLocalInferenceReceipt || entry.hostLocalInferenceProvenance || entry.nimContainer ? "local" : "hosted"; } +function matchesProviderMetadata( + provider: Provider, + normalized: ReturnType, + routeProvider: string, + managed: boolean, +): boolean { + const { type, configKey } = providerContract(normalized.preferredInferenceApi); + const builtin = provider.builtinInferenceEndpoint !== undefined; + return ( + type !== null && + isDeepStrictEqual( + [provider.name, provider.type, provider.credentialKeys, provider.configKeys], + [ + routeProvider, + builtin ? "nvidia" : type, + expectedCredentialKeys(normalized.credentialEnv, managed), + builtin ? [] : [configKey], + ], + ) + ); +} + async function readProviderEvidence( normalized: ReturnType, routeProvider: string, @@ -161,29 +186,23 @@ async function readProviderEvidence( signal: AbortSignal, managedServing?: ObservedManagedVllmRuntime, ): Promise { - const { type, configKey } = providerContract(normalized.preferredInferenceApi); + const { configKey } = providerContract(normalized.preferredInferenceApi); + const managedProfile = !!managedServing || routeProvider === "ollama-local"; const provider = await createProviders().get({ target: namedOpenShellGateway(gatewayName), workspace: "default", name: routeProvider, - ...(managedServing ? { profileContract: "openai" as const } : {}), + ...(managedProfile ? { profileContract: "openai" as const } : {}), configKeys: [configKey], signal, }); if (!provider) throw new Error("The live inference provider is missing."); - const credentialKeys = expectedCredentialKeys(normalized.credentialEnv, !!managedServing); const builtin = provider.builtinInferenceEndpoint !== undefined; - if ( - type === null || - !isDeepStrictEqual( - [provider.name, provider.type, provider.credentialKeys, provider.configKeys], - [routeProvider, builtin ? "nvidia" : type, credentialKeys, builtin ? [] : [configKey]], - ) - ) { + if (!matchesProviderMetadata(provider, normalized, routeProvider, !!managedServing)) { throw new Error("The live inference provider metadata does not match the registry."); } return { - provider: providerIdentity(provider, gatewayName, !!managedServing), + provider: providerIdentity(provider, gatewayName, managedProfile), endpoint: provider.builtinInferenceEndpoint ?? provider.config[configKey] ?? "", source: builtin ? { kind: "builtin-profile", profileId: "nvidia" } @@ -193,14 +212,14 @@ async function readProviderEvidence( async function inferenceFor( entry: Readonly, - beforeProviderRead: () => void, + beforeRead: (stage: ExportSnapshotReadStage) => void, signal: AbortSignal, managedServing?: ObservedManagedVllmRuntime, ): Promise { const normalized = normalizeInferenceSelection(entry); const gateway = resolveGatewayBinding(entry); const live = readInferenceRoute(entry, gateway.name); - beforeProviderRead(); + beforeRead("provider-metadata"); const endpointEvidence = await readProviderEvidence( normalized, live.provider, @@ -208,6 +227,11 @@ async function inferenceFor( signal, managedServing, ); + let ollamaServing: ObservedExportInference["ollamaServing"]; + if (entry.provider === "ollama-local") { + beforeRead("ollama-serving"); + ollamaServing = observeOllamaProxy({ model: live.model, ...createOllamaExportProbe() }); + } return { topology: inferenceTopology(entry, !!managedServing), provider: live.provider, @@ -217,6 +241,7 @@ async function inferenceFor( endpointEvidence, credentialEnv: normalized.credentialEnv, ...(managedServing ? { managedServing } : {}), + ...(ollamaServing ? { ollamaServing } : {}), }; } @@ -305,8 +330,8 @@ async function readSnapshot(sandboxName: string): Promise { stage = "inference-route"; const inference = await inferenceFor( entry, - () => { - stage = "provider-metadata"; + (nextStage) => { + stage = nextStage; }, signal, managedServing, diff --git a/src/lib/config/config.test.ts b/src/lib/config/config.test.ts index 392b5fe40fa..031a0d07271 100644 --- a/src/lib/config/config.test.ts +++ b/src/lib/config/config.test.ts @@ -6,6 +6,7 @@ import Ajv from "ajv/dist/2020.js"; import { describe, expect, it } from "vitest"; import { renderCanonicalNemoClawConfig, validateNemoClawConfig } from "./index"; import { + EXPORTED_OLLAMA_MODEL, EXPORTED_VLLM_PROFILE_ID, EXPORTED_VLLM_RECIPE_ID, isCredentialEnvironmentReferenceName, @@ -86,6 +87,12 @@ function renderInput(value: unknown) { } describe("NemoClawConfig v1", () => { + it("preserves an earlier valid hosted document using the ollama-local label (#11435)", () => { + const value = config(); + value.spec.inferenceProviders[0]!.provider = "ollama-local"; + expect(validateNemoClawConfig(value)).toEqual(value); + }); + it.each(["progressive", "direct"])("validates tool disclosure %s", (disclosure) => { const value = config(); Object.assign(value.spec.sandboxes[0]!.agents[0]!, { tools: { disclosure } }); @@ -858,6 +865,77 @@ describe("fixed managed serving public contract", () => { }); }); +function ollamaConfig() { + const value = config(); + value.spec.sandboxes[0]!.agents[0]!.inference.routes[0]!.overrides.model = EXPORTED_OLLAMA_MODEL; + return { + ...value, + spec: { + ...value.spec, + inferenceProviders: [ + { + name: "hosted-openai", + provider: "ollama-local", + api: "openai-completions", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { servedName: EXPORTED_OLLAMA_MODEL, digest: `sha256:${"a".repeat(64)}` }, + }, + }, + ], + }, + }; +} + +describe("attached Ollama serving public contract", () => { + it("round trips the external daemon separately from its managed proxy (#11435)", () => { + const value = ollamaConfig(); + const rendered = renderCanonicalNemoClawConfig(validateNemoClawConfig(value)); + expect(validateNemoClawConfig(YAML.parse(rendered.yaml))).toEqual(value); + }); + + it.each([ + { daemon: { management: "nemoclaw", hostPort: 11439 } }, + { daemon: { management: "external", hostPort: 11440 } }, + { proxy: { management: "external", hostPort: 11440 } }, + { proxy: { management: "nemoclaw", hostPort: 65536 } }, + { model: { servedName: "other:tag", digest: `sha256:${"a".repeat(64)}` } }, + { model: { servedName: EXPORTED_OLLAMA_MODEL, digest: "not-a-digest" } }, + { runtime: { image: { ref: "ollama:latest" } } }, + ])("rejects unsupported lifecycle or model declarations %# (#11435)", (change) => { + const value = ollamaConfig(); + Object.assign(value.spec.inferenceProviders[0]!.serving, change); + expect(() => validateNemoClawConfig(value)).toThrow(); + }); + + it.each([ + { endpoint: "http://127.0.0.1:11439/v1" }, + { credential: { env: "NEMOCLAW_OLLAMA_PROXY_TOKEN" } }, + ])( + "rejects internal transport or credential fields on the local branch %# (#11435)", + (change) => { + const value = ollamaConfig(); + Object.assign(value.spec.inferenceProviders[0]!, change); + expect(() => validateNemoClawConfig(value)).toThrow(); + }, + ); + + it.each([ + { agent: "hermes", runtime: "docker", model: EXPORTED_OLLAMA_MODEL }, + { agent: "openclaw", runtime: "remote", model: EXPORTED_OLLAMA_MODEL }, + { agent: "openclaw", runtime: "docker", model: "different-model" }, + ])("rejects an unsupported local consumer %s (#11435)", (change) => { + const value = ollamaConfig(); + const sandbox = value.spec.sandboxes[0]!; + sandbox.agents[0]!.type = change.agent; + sandbox.runtime.provider = change.runtime; + sandbox.agents[0]!.inference.routes[0]!.overrides.model = change.model; + expect(() => validateNemoClawConfig(value)).toThrow(); + }); +}); + describe("OpenClaw dashboard configuration", () => { it("rejects OpenClaw dashboard fields on Hermes (#10904)", () => { const value = config(); diff --git a/src/lib/config/model.ts b/src/lib/config/model.ts index 0cb070b09e0..04152110bee 100644 --- a/src/lib/config/model.ts +++ b/src/lib/config/model.ts @@ -312,9 +312,41 @@ const NemoClawManagedInferenceProviderConfigSchema = Type.Object( { additionalProperties: false }, ); +export const EXPORTED_OLLAMA_MODEL = "qwen3.5:9b" as const; +export const NemoClawOllamaServingSchema = Type.Object( + { + backend: Type.Literal("ollama"), + daemon: Type.Object( + { management: Type.Literal("external"), hostPort: TcpPortSchema }, + { additionalProperties: false }, + ), + proxy: Type.Object( + { management: Type.Literal("nemoclaw"), hostPort: TcpPortSchema }, + { additionalProperties: false }, + ), + model: Type.Object( + { servedName: Type.Literal(EXPORTED_OLLAMA_MODEL), digest: ServingDigestSchema }, + { additionalProperties: false }, + ), + }, + { additionalProperties: false }, +); +export type NemoClawOllamaServing = TypeBoxModule.Type.Static; + +const NemoClawOllamaInferenceProviderConfigSchema = Type.Object( + { + name: LocalResourceNameSchema, + provider: Type.Literal("ollama-local"), + api: Type.Literal("openai-completions"), + serving: NemoClawOllamaServingSchema, + }, + { additionalProperties: false }, +); + const NemoClawInferenceProviderConfigSchema = Type.Union([ NemoClawHostedInferenceProviderConfigSchema, NemoClawManagedInferenceProviderConfigSchema, + NemoClawOllamaInferenceProviderConfigSchema, ]); const NemoClawRouteOverridesSchema = Type.Object( diff --git a/src/lib/config/schema.ts b/src/lib/config/schema.ts index 26c458088da..3549f21ef98 100644 --- a/src/lib/config/schema.ts +++ b/src/lib/config/schema.ts @@ -171,6 +171,27 @@ function managedProviderProblems( provider: Extract, providerIndex: number, ): string[] { + if (provider.serving.backend === "ollama") { + const serving = provider.serving; + const matches = + serving.daemon.hostPort !== serving.proxy.hostPort && + config.spec.sandboxes.every((sandbox) => + sandbox.agents.every((agent) => + agent.inference.routes.every( + (route) => + route.providerRef !== provider.name || + (sandbox.runtime.provider === "docker" && + agent.type === "openclaw" && + route.overrides.model === serving.model.servedName), + ), + ), + ); + return matches + ? [] + : [ + `/spec/inferenceProviders/${providerIndex}/serving requires distinct Ollama ports and a matching Docker OpenClaw route`, + ]; + } const matches = config.spec.sandboxes.every((sandbox) => sandbox.agents.every((agent) => agent.inference.routes.every( diff --git a/src/lib/domain/config/export-document.ts b/src/lib/domain/config/export-document.ts index 3d5308639aa..f24fb584206 100644 --- a/src/lib/domain/config/export-document.ts +++ b/src/lib/domain/config/export-document.ts @@ -19,15 +19,29 @@ function providerLocalName(provider: string): string { return `hosted-${normalized || "provider"}`.slice(0, 63).replace(/[^a-z0-9]+$/gu, ""); } +function exportedProviderName(inference: VerifiedExportSource["inference"]): string { + if ("serving" in inference) + return inference.serving.backend === "vllm" ? "managed-vllm" : "local-ollama"; + return providerLocalName(inference.provider); +} + function inferenceProvider( source: VerifiedExportSource, name: string, ): NemoClawInferenceProviderConfig { if ("serving" in source.inference) { + if (source.inference.serving.backend === "ollama") { + return { + name, + provider: "ollama-local", + api: "openai-completions", + serving: source.inference.serving, + }; + } return { name, - provider: source.inference.provider, - api: source.inference.api, + provider: "vllm-local", + api: "openai-completions", serving: source.inference.serving, }; } @@ -73,7 +87,7 @@ function exportAgent(source: VerifiedExportSource, providerName: string): NemoCl providerRef: providerName, overrides: { model: source.inference.model, - ...("serving" in source.inference + ...(source.inference.provider === "vllm-local" && "serving" in source.inference ? { contextWindow: EXPORTED_VLLM_CONTEXT_WINDOW } : {}), ...("overrides" in source.inference ? source.inference.overrides : {}), @@ -94,8 +108,7 @@ export function buildExportConfig( source: VerifiedExportSource, identity: ExportConfigBuildIdentity, ): NemoClawConfig { - const providerName = - "serving" in source.inference ? "managed-vllm" : providerLocalName(source.inference.provider); + const providerName = exportedProviderName(source.inference); const candidate = { apiVersion: "nemoclaw.nvidia.com/v1", kind: "NemoClawConfig", diff --git a/src/lib/domain/config/export-evidence.ts b/src/lib/domain/config/export-evidence.ts index e4c3aea5d66..8867c709e6e 100644 --- a/src/lib/domain/config/export-evidence.ts +++ b/src/lib/domain/config/export-evidence.ts @@ -5,6 +5,7 @@ import type * as TypeBoxModule from "typebox" with { "resolution-mode": "import" import { BoundedTextSchema, NemoClawManagedVllmServingSchema, + NemoClawOllamaServingSchema, CredentialEnvironmentReferenceNameSchema, ImmutableImageReferenceSchema, InferenceEndpointSchema, @@ -28,6 +29,7 @@ import { } from "../../config/model"; import type { SandboxConfiguration } from "../sandbox/configuration"; import type { SandboxEntry } from "../../state/registry/types"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; const { Type } = require("typebox") as typeof TypeBoxModule; @@ -161,6 +163,7 @@ export interface ObservedExportInference { readonly endpointEvidence: ObservedExportEndpointEvidence | null; readonly credentialEnv: string | null; readonly managedServing?: ObservedManagedVllmRuntime; + readonly ollamaServing?: ObservedOllamaProxy; } export interface ObservedExportPolicy { @@ -188,6 +191,7 @@ export type ExportSnapshotReadStage = | "provider-metadata" | "web-search-provider" | "managed-serving" + | "ollama-serving" | "effective-policy"; /** One complete, untrusted read from all export evidence owners. */ @@ -256,6 +260,16 @@ const HostedExportInferenceSchema = Type.Object({ const ExportInferenceSchema = Type.Union([ HostedExportInferenceSchema, + Type.Object( + { + provider: Type.Literal("ollama-local"), + model: Type.Refine(BoundedTextSchema, isValidNemoClawBoundedText), + api: Type.Literal("openai-completions"), + serving: NemoClawOllamaServingSchema, + overrides: Type.Optional(NemoClawInferenceTuningSchema), + }, + { additionalProperties: false }, + ), Type.Object( { provider: Type.Literal("vllm-local"), diff --git a/src/lib/domain/config/verify-export-source.ts b/src/lib/domain/config/verify-export-source.ts index 938ced37dcc..9f425d1891a 100644 --- a/src/lib/domain/config/verify-export-source.ts +++ b/src/lib/domain/config/verify-export-source.ts @@ -35,6 +35,7 @@ import { fingerprintOpenShellSandboxId } from "../sandbox/openshell-identity"; import { HERMES_PROVIDER_NAME } from "../../onboard/inference-providers/hermes-provider-identity"; import { ExportSourceValuesSchema } from "./export-evidence"; import { validateManagedServing } from "./verify-managed-serving"; +import { validateOllamaServing } from "./verify-ollama-serving"; import { inspectAgentInterfaces } from "./verify-agent-interfaces"; import type { CanonicalExportPolicy, @@ -874,7 +875,11 @@ function validateGateway(snapshot: QualifiedExportSnapshot): ExportFinding[] { function validateInferenceSelection(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { registry: entry, inference } = snapshot; const findings: ExportFinding[] = []; - if (inference.topology !== "hosted" && inference.topology !== "managed") + if ( + inference.topology !== "hosted" && + inference.topology !== "managed" && + !(inference.topology === "local" && inference.provider === "ollama-local") + ) findings.push( finding( "spec.inferenceProviders", @@ -925,6 +930,8 @@ function validateInferenceSelection(snapshot: QualifiedExportSnapshot): ExportFi function validateInferenceRepresentation(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { inference } = snapshot; + if (inference.provider === "ollama-local" || inference.ollamaServing) + return validateOllamaServing(snapshot); if (inference.topology === "managed") return validateManagedServing(snapshot); const findings: ExportFinding[] = []; if ( @@ -975,7 +982,11 @@ function validateEndpointEvidence(snapshot: QualifiedExportSnapshot): ExportFind } const findings: ExportFinding[] = []; - if (inference.topology !== "managed" && !isValidNemoClawInferenceEndpoint(evidence.endpoint)) + if ( + inference.topology !== "managed" && + inference.provider !== "ollama-local" && + !isValidNemoClawInferenceEndpoint(evidence.endpoint) + ) findings.push( finding( "source.inference.endpoint", @@ -1012,7 +1023,7 @@ function validateEndpointEvidence(snapshot: QualifiedExportSnapshot): ExportFind function validateCredentialReference(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { inference } = snapshot; - if (inference.topology === "managed") return []; + if (inference.topology === "managed" || inference.provider === "ollama-local") return []; const findings: ExportFinding[] = []; if ( inference.credentialEnv !== null && @@ -1113,6 +1124,9 @@ function projectVerifiedInference( model: selected.model, api: selected.preferredInferenceApi, }; + if (snapshot.inference.provider === "ollama-local") { + return { ...common, serving: snapshot.inference.ollamaServing?.serving }; + } return snapshot.inference.topology === "managed" ? { ...common, serving: snapshot.inference.managedServing?.serving } : { diff --git a/src/lib/domain/config/verify-managed-serving.ts b/src/lib/domain/config/verify-managed-serving.ts index 9752156b9f0..e454c4e8ea5 100644 --- a/src/lib/domain/config/verify-managed-serving.ts +++ b/src/lib/domain/config/verify-managed-serving.ts @@ -64,7 +64,7 @@ function validProfileVersion(version: string): boolean { return /^(0|[1-9][0-9]{0,19})$/.test(version) && BigInt(version) <= 18446744073709551615n; } -function hasManagedOpenAiProfile(evidence: ObservedExportEndpointEvidence | null): boolean { +export function hasManagedOpenAiProfile(evidence: ObservedExportEndpointEvidence | null): boolean { if (!evidence) return false; const { provider } = evidence; const profile = provider.managedProfile; diff --git a/src/lib/domain/config/verify-ollama-serving.ts b/src/lib/domain/config/verify-ollama-serving.ts new file mode 100644 index 00000000000..4b2ce4195a9 --- /dev/null +++ b/src/lib/domain/config/verify-ollama-serving.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import type * as TypeBoxValueModule from "typebox/value" with { "resolution-mode": "import" }; +import { NemoClawOllamaServingSchema } from "../../config/model"; +import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "../../inference/ollama/contract"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; +import type { ExportFinding, QualifiedExportSnapshot } from "./export-evidence"; +import { hasManagedOpenAiProfile } from "./verify-managed-serving"; + +const { Check } = require("typebox/value") as typeof TypeBoxValueModule; + +function validObservation( + observed: ObservedOllamaProxy | undefined, +): observed is ObservedOllamaProxy { + if (!observed) return false; + const { serving, pid } = observed; + return ( + Check(NemoClawOllamaServingSchema, serving) && + serving.daemon.hostPort !== serving.proxy.hostPort && + Number.isInteger(pid) && + pid > 0 && + pid <= 2_147_483_647 && + observed.listenerAddress === "0.0.0.0" + ); +} + +export function validateOllamaServing(snapshot: QualifiedExportSnapshot): ExportFinding[] { + const { registry: entry, inference } = snapshot; + const observed = inference.ollamaServing; + if ( + validObservation(observed) && + hasManagedOpenAiProfile(inference.endpointEvidence) && + entry.workload?.kind === "managed-image" && + /^linux\/(?:amd64|arm64)$/u.test(entry.workload.platform ?? "") && + isDeepStrictEqual( + [ + entry.agent, + entry.openshellDriver, + inference.topology, + inference.provider, + inference.api, + inference.credentialEnv, + inference.model, + inference.endpoint, + snapshot.sandbox.providerNames.filter((name) => name === inference.provider), + ], + [ + "openclaw", + "docker", + "local", + "ollama-local", + "openai-completions", + OLLAMA_LOCAL_CREDENTIAL_ENV, + observed.serving.model.servedName, + `http://host.openshell.internal:${observed.serving.proxy.hostPort}/v1`, + ["ollama-local"], + ], + ) + ) + return []; + return [ + { + field: "spec.inferenceProviders[].serving", + category: "drifted", + diagnostic: + "The attached Ollama daemon, managed proxy, model, or sandbox route could not be verified.", + }, + ]; +} diff --git a/src/lib/inference/README.md b/src/lib/inference/README.md index e0c8e52a344..e0577ae8469 100644 --- a/src/lib/inference/README.md +++ b/src/lib/inference/README.md @@ -24,3 +24,36 @@ onboard-probes.ts onboarding-time inference validation probes ``` Longer term, pure inference decisions should move under `src/lib/domain/inference/**`, and HTTP/process boundaries should move under `src/lib/adapters/**`. + +## Ollama configuration export + +The native Linux Docker export slice composes the current proxy owner and nonsecret observer in +`adapters/config/live-export-source.ts`. Each snapshot asks `ollama/proxy.ts` for a fresh probe of +the retained descriptor, port, PID, and three fixed endpoint reads. `ollama/proxy-observation.ts` +observes the host platform and validates retained intent before requesting an authenticated read. +Only then does the proxy owner read its token, capturing it for that snapshot's proxy requests. +The token stays inside this owner and enters curl through stdin. Fixed loopback reads bypass ambient +HTTP proxies and have bounded responses. These reads must not acquire a lifecycle lock, migrate +state, restart a process, or call the token getters that can adopt legacy state. + +```mermaid +flowchart LR + retained[Retained proxy intent] --> owner[Existing proxy owner] + active[Authenticated active proxy configuration] --> owner + models[Daemon and proxy model inventories] --> owner + host[Native host observation] --> observation[Nonsecret observation validation] + owner --> observation + observation --> export[Shared export snapshot and verifier] + export --> document[External daemon and managed proxy configuration] +``` + +The proxy's authenticated `GET /_nemoclaw/proxy-config` reports its current PID, actual listener, +and backend origin. It never forwards this request or returns backend userinfo, paths, queries, or +credentials. `ollama/proxy-observation.ts` compares that response with retained intent and the selected +model digest from both native model inventories. Older running proxies without this response cannot +be exported; upgrading or restarting them remains an operator lifecycle action. + +The public `serving.backend: ollama` branch records the daemon as external and the auth proxy as +NemoClaw-managed. It does not claim ownership of the daemon process, software installation, or model +cache. The initial contract covers `qwen3.5:9b` on native Linux Docker with managed OpenClaw and no +direct sandbox GPU. Hosted inference and managed vLLM retain their existing schema variants. diff --git a/src/lib/inference/ollama/proxy-observation.test.ts b/src/lib/inference/ollama/proxy-observation.test.ts new file mode 100644 index 00000000000..25e3611b2f5 --- /dev/null +++ b/src/lib/inference/ollama/proxy-observation.test.ts @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import os from "node:os"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { isWsl } from "../../platform"; +import { observeOllamaProxy, type OllamaProxyObservationInput } from "./proxy-observation"; + +vi.mock("../../platform", () => ({ isWsl: vi.fn(() => false) })); + +const digest = "a".repeat(64); +const active = { + schemaVersion: 1, + pid: 1234, + listener: { address: "0.0.0.0", port: 11440 }, + backendOrigin: "http://127.0.0.1:11439", +}; +const models = JSON.stringify({ models: [{ name: "qwen3.5:9b", digest, size: 6000000000 }] }); +function observation(): OllamaProxyObservationInput { + return { + model: "qwen3.5:9b", + backend: { kind: "ollama", url: active.backendOrigin }, + proxyPort: "11440", + pid: "1234", + processMatches: vi.fn(() => true), + readActiveConfig: vi.fn(() => JSON.stringify(active)), + readProxyModels: vi.fn(() => models), + readDaemonModels: vi.fn(() => models), + }; +} + +describe("read-only Ollama export observation", () => { + beforeEach(() => { + vi.spyOn(os, "platform").mockReturnValue("linux"); + vi.mocked(isWsl).mockReturnValue(false); + }); + + it.each([ + { platform: "darwin", wsl: false }, + { platform: "linux", wsl: true }, + ] as const)("refuses unsupported host facts before protected reads %# (#11435)", (host) => { + vi.spyOn(os, "platform").mockReturnValue(host.platform); + vi.mocked(isWsl).mockReturnValue(host.wsl); + const input = observation(); + expect(() => observeOllamaProxy(input)).toThrow("requires a native Linux host"); + expect(input.readActiveConfig).not.toHaveBeenCalled(); + expect(input.readProxyModels).not.toHaveBeenCalled(); + expect(input.readDaemonModels).not.toHaveBeenCalled(); + }); + + it("exports the active nondefault mapping and attached model identity (#11435)", () => { + const input = observation(); + expect(observeOllamaProxy(input)).toEqual({ + pid: 1234, + listenerAddress: "0.0.0.0", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { servedName: "qwen3.5:9b", digest: `sha256:${digest}` }, + }, + }); + expect(input.readActiveConfig).toHaveBeenCalledWith(11440); + expect(input.readProxyModels).toHaveBeenCalledWith(11440); + expect(input.readDaemonModels).toHaveBeenCalledWith(11439); + }); + + it.each([ + { backend: { kind: "compatible", url: active.backendOrigin } }, + { backend: { kind: "ollama", url: "http://127.0.0.1:11439/private?secret" } }, + { backend: { kind: "ollama", url: "http://remote.example:11439" } }, + { proxyPort: "11439" }, + { proxyPort: null }, + { proxyPort: "011440" }, + { pid: null }, + { pid: "-1" }, + { pid: "1234junk" }, + { model: "different:tag" }, + { processMatches: () => false }, + ])( + "refuses unsupported or incomplete retained intent before network reads %# (#11435)", + (change) => { + const input = { ...observation(), ...change }; + expect(() => observeOllamaProxy(input)).toThrow("could not be verified"); + expect(input.readActiveConfig).not.toHaveBeenCalled(); + expect(input.readProxyModels).not.toHaveBeenCalled(); + expect(input.readDaemonModels).not.toHaveBeenCalled(); + }, + ); + + it.each([ + "", + "legacy proxy response", + JSON.stringify({ error: "not found" }), + "a".repeat(65537), + JSON.stringify({ ...active, pid: 1235 }), + JSON.stringify({ ...active, listener: { address: "127.0.0.1", port: 11440 } }), + JSON.stringify({ ...active, listener: { address: "0.0.0.0", port: 11435 } }), + JSON.stringify({ ...active, backendOrigin: "http://127.0.0.1:11434" }), + JSON.stringify({ ...active, backendOrigin: "http://user:secret@127.0.0.1:11439/private" }), + ])("refuses a missing or mismatched active response without guessing %# (#11435)", (body) => { + const input = { ...observation(), readActiveConfig: () => body }; + expect(() => observeOllamaProxy(input)).toThrow("could not be verified"); + expect(input.readProxyModels).not.toHaveBeenCalled(); + }); + + it.each([ + { models: [] }, + { models: [{ name: "qwen3.5:9b", digest: "invalid" }] }, + { models: [{ name: "other:tag", digest }] }, + { + models: [ + { name: "qwen3.5:9b", digest }, + { name: "qwen3.5:9b", digest }, + ], + }, + { models: [{ name: "qwen3.5:9b", digest: "b".repeat(64) }] }, + ])("refuses a missing, ambiguous, or changed daemon model %# (#11435)", (value) => { + expect(() => + observeOllamaProxy({ ...observation(), readDaemonModels: () => JSON.stringify(value) }), + ).toThrow("could not be verified"); + }); +}); diff --git a/src/lib/inference/ollama/proxy-observation.ts b/src/lib/inference/ollama/proxy-observation.ts new file mode 100644 index 00000000000..b682a551708 --- /dev/null +++ b/src/lib/inference/ollama/proxy-observation.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import os from "node:os"; +import type * as TypeBoxModule from "typebox" with { "resolution-mode": "import" }; +import type * as TypeBoxValueModule from "typebox/value" with { "resolution-mode": "import" }; +import { isWsl } from "../../platform"; +import { + BoundedTextSchema, + EXPORTED_OLLAMA_MODEL, + TcpPortSchema, + type NemoClawOllamaServing, +} from "../../config/model"; + +const { Type } = require("typebox") as typeof TypeBoxModule; +const { Check } = require("typebox/value") as typeof TypeBoxValueModule; +const ProcessId = Type.Integer({ minimum: 1, maximum: 2_147_483_647 }); +const ActiveConfig = Type.Object( + { + schemaVersion: Type.Literal(1), + pid: ProcessId, + listener: Type.Object( + { address: Type.Literal("0.0.0.0"), port: TcpPortSchema }, + { additionalProperties: false }, + ), + backendOrigin: Type.String({ maxLength: 64 }), + }, + { additionalProperties: false }, +); +const Models = Type.Object({ + models: Type.Array( + Type.Object({ + name: BoundedTextSchema, + digest: Type.String({ pattern: "^(?:sha256:)?[a-f0-9]{64}$(?![\\s\\S])" }), + }), + { maxItems: 512 }, + ), +}); + +export interface ObservedOllamaProxy { + readonly serving: NemoClawOllamaServing; + readonly pid: number; + readonly listenerAddress: "0.0.0.0"; +} + +export interface OllamaProxyObservationInput { + readonly model: string; + readonly backend: { readonly kind: string; readonly url: string | null }; + readonly proxyPort: string | null; + readonly pid: string | null; + readonly processMatches: (pid: number) => boolean; + readonly readActiveConfig: (port: number) => string; + readonly readProxyModels: (port: number) => string; + readonly readDaemonModels: (port: number) => string; +} + +function fail(): never { + throw new Error( + "The live Ollama daemon, proxy mapping, or model could not be verified for export.", + ); +} + +function parse( + schema: Schema, + body: string, +): TypeBoxModule.Type.Static { + if (Buffer.byteLength(body) > 64 * 1024) fail(); + let value: unknown; + try { + value = JSON.parse(body); + } catch { + fail(); + } + if (!Check(schema, value)) fail(); + return value; +} + +function selectedDigest(body: string, model: string): string { + const matches = parse(Models, body).models.filter((entry) => entry.name === model); + if (matches.length !== 1) fail(); + const digest = matches[0].digest; + return digest.startsWith("sha256:") ? digest : `sha256:${digest}`; +} + +/** Validate retained intent against current proxy and model reads; receives no credentials. */ +export function observeOllamaProxy(input: OllamaProxyObservationInput): ObservedOllamaProxy { + if (os.platform() !== "linux" || isWsl()) { + throw new Error("Ollama export requires a native Linux host."); + } + const proxyPort = Number(input.proxyPort); + const pid = Number(input.pid); + const backend = input.backend.url?.match(/^http:\/\/127\.0\.0\.1:([1-9][0-9]{0,4})$/u); + const daemonPort = Number(backend?.[1]); + if ( + input.backend.kind !== "ollama" || + input.model !== EXPORTED_OLLAMA_MODEL || + !Check(TcpPortSchema, daemonPort) || + !Check(TcpPortSchema, proxyPort) || + input.proxyPort !== String(proxyPort) || + !Check(ProcessId, pid) || + input.pid !== String(pid) || + daemonPort === proxyPort || + !input.processMatches(pid) + ) + fail(); + + const active = parse(ActiveConfig, input.readActiveConfig(proxyPort)); + if ( + !isDeepStrictEqual( + [active.pid, active.listener.port, active.backendOrigin], + [pid, proxyPort, input.backend.url], + ) + ) + fail(); + const digest = selectedDigest(input.readProxyModels(proxyPort), input.model); + if (selectedDigest(input.readDaemonModels(daemonPort), input.model) !== digest) fail(); + return { + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: daemonPort }, + proxy: { management: "nemoclaw", hostPort: proxyPort }, + model: { servedName: EXPORTED_OLLAMA_MODEL, digest }, + }, + pid, + listenerAddress: active.listener.address, + }; +} diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 0017c692b20..ac06112f2e4 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -355,6 +355,38 @@ function runCurlCaptureWithAuthConfig( return result.status === 0 ? String(result.stdout || "") : ""; } +/** Fixed export reads capture one credential only after the observer admits retained intent. */ +export function createOllamaExportProbe() { + let token: string | null = null; + const args = [ + "-q", + "--noproxy", + "*", + "-fsS", + "--connect-timeout", + "3", + "--max-time", + "5", + "--max-filesize", + "65536", + ]; + const readProxy = (port: number, route: string) => { + token ??= readProxyStateFile(PROXY_TOKEN_PATH); + if (!token) throw new Error("The existing Ollama proxy credential is unavailable."); + return runCurlCaptureWithAuthConfig(args, `http://127.0.0.1:${port}${route}`, token); + }; + return { + backend: readProxyBackendIdentity(), + proxyPort: readProxyStateFile(PROXY_PORT_PATH), + pid: readProxyStateFile(PROXY_PID_PATH), + processMatches: isOllamaProxyProcess, + readActiveConfig: (port: number) => readProxy(port, "/_nemoclaw/proxy-config"), + readProxyModels: (port: number) => readProxy(port, "/api/tags"), + readDaemonModels: (port: number) => + runCurlCaptureWithAuthConfig(args, `http://127.0.0.1:${port}/api/tags`), + }; +} + // ── PID persistence ────────────────────────────────────────────── function persistProxyPid(pid: number | null | undefined): void { diff --git a/test/e2e/README.md b/test/e2e/README.md index 6c867bb6a2d..2eb2c5121e7 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -461,6 +461,15 @@ The standard layout writes product evidence and `evidence-manifest.json` under ` When `shard` is not `default`, the standard layout adds the shard directory. The security-posture matrix uses the reviewed flat-shard layout to preserve its existing artifact names. The `gpu-double-onboard`, `gpu-e2e`, and `llama-cpp-generic-gpu` targets keep the standard layout and select `linux-amd64-gpu-rtxpro6000-latest-1` through the catalogue. +The `gpu-e2e` target also qualifies configuration export for an attached native Linux Ollama daemon. +A separate OpenClaw scenario disables direct sandbox GPU, starts a fixture-owned daemon on port +11439, and uses normal onboarding to create the managed proxy on port 11440. It exports twice through +the candidate CLI and real SDK, validates both documents, compares their specs and model digest, +checks credential omission, and requires a stopped daemon to prevent publication. Private YAML is +removed through the cleanup registry; retained evidence contains only the selected model, ports, +managed image, and result booleans. The existing CUDA, authentication, and inference lifecycle +scenarios remain separate. Daemon readiness polls only connection refusal, for at most 20 reads, +and records each attempt; onboarding and export mutations are not retried. Retained workflow jobs are exceptions to the catalogue shape. Keep one only for a multi-job handoff, an unrepresented credential boundary, or an execution contract the reusable profile cannot represent. diff --git a/test/e2e/live/gpu-e2e-helpers.ts b/test/e2e/live/gpu-e2e-helpers.ts index 2d6a7f2dd51..e21ac560e76 100644 --- a/test/e2e/live/gpu-e2e-helpers.ts +++ b/test/e2e/live/gpu-e2e-helpers.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; +import { ownChildProcess } from "../../helpers/child-process-lifecycle.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; @@ -10,11 +11,23 @@ import { resultText } from "../fixtures/clients/index.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { stripAnsi } from "./json-envelope.ts"; export { REPO_ROOT }; +/** The export fixture owns only this attached daemon; onboarding owns the auth proxy. */ +export function startAttachedOllama(progress: TestProgress, environment: NodeJS.ProcessEnv) { + const child = spawnObservedChild("ollama", ["serve"], { + activityLabel: "attached Ollama daemon", + progress, + spawn: { cwd: REPO_ROOT, env: environment, stdio: "ignore" }, + }); + return ownChildProcess(child); +} + export const CLI = CLI_ENTRYPOINT; export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-gpu-ollama"; const DEFAULT_GPU_E2E_MODEL = "qwen3.5:9b"; diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index c95578a2371..b6474487a89 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -1,11 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { validateNemoClawConfig } from "../../../src/lib/config/schema.ts"; +import { load as loadRegistry } from "../../../src/lib/state/registry/persistence.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { pollUntil } from "../fixtures/polling.ts"; import { assertAgentExecutionSucceeded, assertGpuInstallProofs, @@ -25,6 +32,7 @@ import { readTokenFileChecked, restartProxy, SANDBOX_NAME, + startAttachedOllama, } from "./gpu-e2e-helpers.ts"; import { assertHermesFollowUpReplies } from "./hermes-cli-adapter-live.ts"; @@ -111,8 +119,7 @@ test( assertNvidiaAvailable(nvidia, skip); await ensureOllama(host); - const ollamaCleanup = await cleanupOllama(host, "pre-cleanup-ollama"); - expect(ollamaCleanup.exitCode, resultText(ollamaCleanup)).toBe(0); + await cleanupOllama(host, "pre-cleanup-ollama"); progress.phase("install Ollama and GPU sandbox"); const install = await host.command("bash", ["install.sh", "--non-interactive"], { @@ -130,20 +137,17 @@ test( env: env(), timeoutMs: 120_000, }); - expect(status.exitCode, resultText(status)).toBe(0); expect(resultText(status)).toContain("Sandbox GPU: enabled"); - expect(resultText(status)).toMatch(/CUDA verified|CUDA unverified|last CUDA proof failed/i); - expect(resultText(status)).not.toMatch(/last CUDA proof failed|CUDA unverified/i); + expect(resultText(status)).toContain("CUDA verified"); const installLog = resultText(install); assertGpuInstallProofs(installLog); expect(installLog).toContain( "Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.", ); - expect(installLog).not.toContain( - "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", + expect(installLog).not.toMatch( + /Recreating OpenShell Docker sandbox container with NVIDIA GPU access|Docker GPU mode selected/u, ); - expect(installLog).not.toContain("Docker GPU mode selected"); const sandboxContainers = await runtimeProvider.command( [ @@ -161,7 +165,6 @@ test( timeoutMs: 30_000, }, ); - expect(sandboxContainers.exitCode, resultText(sandboxContainers)).toBe(0); const sandboxContainerInventory = sandboxContainers.stdout .split(/\r?\n/) .map((line) => line.trim()) @@ -184,7 +187,6 @@ test( env: env(), timeoutMs: 30_000, }); - expect(route.exitCode, resultText(route)).toBe(0); expect(resultText(route)).toMatch(/ollama/i); progress.phase("validate Ollama proxy credential boundary"); @@ -198,7 +200,6 @@ test( ["-sS", "-o", "/dev/null", "-w", "%{http_code}", `http://127.0.0.1:${PROXY_PORT}/api/tags`], { artifactName: "ollama-proxy-unauthorized", env: env(), timeoutMs: 30_000 }, ); - expect(proxyUnauth.exitCode, resultText(proxyUnauth)).toBe(0); expect(proxyUnauth.stdout).toBe("401"); const proxyAuth = await host.command( @@ -211,11 +212,8 @@ test( timeoutMs: 30_000, }, ); - expect(proxyAuth.exitCode, resultText(proxyAuth)).toBe(0); expect(proxyAuth.stdout).toMatch(/models|name/i); - const proxyBefore = await proxyStatus(host, token, "proxy-status-before-restart"); - expect(proxyBefore.exitCode, resultText(proxyBefore)).toBe(0); await restartProxy(host, token); const proxyAfter = await proxyStatus(host, token, "proxy-status-after-restart"); expect(proxyAfter.exitCode, resultText(proxyAfter)).toBe(0); @@ -250,7 +248,6 @@ test( ), { artifactName: "sandbox-inference-local-chat", env: env(), timeoutMs: 150_000 }, ); - expect(chat.exitCode, resultText(chat)).toBe(0); expect(chatContent(chat.stdout)).toMatch(/pong/i); const readySandbox = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { @@ -258,7 +255,6 @@ test( env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); - expect(readySandbox.exitCode, resultText(readySandbox)).toBe(0); expect( hasExactReadyPhase(readySandbox.stdout), `OpenShell sandbox must be exactly Ready after routed inference; got ${resultText(readySandbox)}`, @@ -295,7 +291,6 @@ exit 1`, ], { artifactName: "ollama-daemon-restart-unloaded", env: env(), timeoutMs: 90_000 }, ); - expect(restart.exitCode, resultText(restart)).toBe(0); const restartLines = restart.stdout.trim().split("\n"); expect(restartLines[0]).toMatch(/^restart_mode=(system|user|manual)$/u); expect(loadedOllamaModels(restartLines.slice(1).join("\n"))).toEqual([]); @@ -318,7 +313,6 @@ exit 1`, timeoutMs: 12 * 60_000, }, ); - expect(recovered.exitCode, resultText(recovered)).toBe(0); assertAgentExecutionSucceeded(recovered.stdout, "inference", model); const loaded = await host.command("curl", ["-fsS", "http://127.0.0.1:11434/api/ps"], { @@ -326,7 +320,6 @@ exit 1`, env: env(), timeoutMs: 30_000, }); - expect(loaded.exitCode, resultText(loaded)).toBe(0); expect(loadedOllamaModels(loaded.stdout)).toContain(model); }, ); @@ -413,3 +406,172 @@ test( }); }, ); + +test( + "OpenClaw exports an attached Ollama daemon and refuses a stopped backend (#11435)", + { + timeout: TIMEOUT_MS, + meta: { + e2ePhases: [ + "prepare an attached Ollama daemon", + "onboard OpenClaw without sandbox GPU", + "export and compare the active Ollama configuration", + "refuse export after the attached daemon stops", + ], + }, + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { + await artifacts.target.declare({ + id: "gpu-e2e", + boundary: + "native Linux Docker + attached Ollama daemon + NemoClaw proxy + managed OpenClaw + SDK configuration export", + credentialBoundary: + "The existing proxy owner authenticates observation; exported YAML contains no token or internal endpoint.", + }); + const exportEnv = env({ + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_SANDBOX_GPU: "0", + NEMOCLAW_SANDBOX_GPU_DEVICE: "", + NEMOCLAW_OLLAMA_PORT: "11439", + NEMOCLAW_OLLAMA_PROXY_PORT: "11440", + NEMOCLAW_MODEL: "qwen3.5:9b", + NEMOCLAW_WEB_SEARCH_PROVIDER: "none", + OLLAMA_HOST: "127.0.0.1:11439", + OLLAMA_CONTEXT_LENGTH: "32768", + }); + cleanup.trackDisposable("stop Ollama processes after export qualification", async () => { + const result = await cleanupOllama(host, "export-cleanup-ollama-processes"); + expect(result.exitCode, resultText(result)).toBe(0); + }); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-export-")); + cleanup.trackDisposable("remove private Ollama export documents", () => + fs.rmSync(directory, { recursive: true, force: true }), + ); + await cleanupGpu(host, sandbox); + await runtimeProvider.requireAvailable({ + artifactName: "export-runtime-info", + scenarioLabel: "attached Ollama export", + }); + await ensureOllama(host); + await cleanupOllama(host, "export-stop-default-ollama"); + const daemonOwner = startAttachedOllama(progress, exportEnv); + cleanup.trackDisposable("stop the fixture-owned Ollama daemon", () => daemonOwner.terminate()); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "export-cleanup-gateway", + env: exportEnv, + timeoutMs: 60000, + }); + cleanup.trackDisposable("delete the export sandbox", () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "export-cleanup-openshell", + env: exportEnv, + timeoutMs: 60000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "export-cleanup-sandbox", + env: exportEnv, + timeoutMs: 120000, + }); + // Only connection refusal is transient while this fixture's child starts. Every read is recorded. + await pollUntil({ + artifactPrefix: "export-daemon-ready", + attempts: 20, + delayMs: 500, + probe: (_attempt, artifactName) => + host.command( + "curl", + ["-q", "--noproxy", "*", "-fsS", "--max-time", "2", "http://127.0.0.1:11439/api/tags"], + { artifactName, env: exportEnv, timeoutMs: 5000 }, + ), + accept: (result) => result.exitCode === 0, + terminal: (result) => + result.exitCode !== 0 && result.exitCode !== 7 + ? "The attached daemon readiness read failed." + : undefined, + }); + + progress.phase("onboard OpenClaw without sandbox GPU"); + const onboard = await host.command( + "node", + [CLI, "onboard", "--fresh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "export-onboard-ollama", + cwd: REPO_ROOT, + env: exportEnv, + timeoutMs: execTimeout(55 * 60000), + }, + ); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + progress.phase("export and compare the active Ollama configuration"); + const firstPath = path.join(directory, "first.yaml"); + const exported = await host.command( + "node", + [CLI, "config", "export", SANDBOX_NAME, "--output", firstPath, "--json"], + { artifactName: "export-ollama-first", cwd: REPO_ROOT, env: exportEnv, timeoutMs: 60000 }, + ); + expect(exported.exitCode, resultText(exported)).toBe(0); + const raw = fs.readFileSync(firstPath, "utf8"); + const document = validateNemoClawConfig(YAML.parse(raw)); + const token = readTokenFileChecked(ollamaProxyTokenFile()).token; + artifacts.addRedactionValues([token]); + expect(raw.includes(token), "Export must omit the proxy credential").toBe(false); + expect(raw).not.toMatch(/NEMOCLAW_OLLAMA_PROXY_TOKEN|host\.openshell\.internal/u); + const tags = await host.command( + "curl", + ["-q", "--noproxy", "*", "-fsS", "--max-time", "5", "http://127.0.0.1:11439/api/tags"], + { artifactName: "export-attached-model-identity", env: exportEnv, timeoutMs: 10000 }, + ); + const model = ( + JSON.parse(tags.stdout) as { models: Array<{ name: string; digest: string }> } + ).models.find(({ name }) => name === "qwen3.5:9b"); + const exportedProvider = document.spec.inferenceProviders[0]; + const serving = + "serving" in exportedProvider && exportedProvider.serving.backend === "ollama" + ? exportedProvider.serving + : undefined; + expect(exportedProvider.provider).toBe("ollama-local"); + expect(serving?.daemon.hostPort).toBe(11439); + expect(serving?.proxy.hostPort).toBe(11440); + expect(serving?.model.digest).toBe(`sha256:${model?.digest.replace(/^sha256:/u, "")}`); + const entry = loadRegistry().sandboxes[SANDBOX_NAME]; + expect(document.spec.sandboxes[0].runtime.image.ref).toBe( + entry.workload?.kind === "managed-image" ? entry.workload.reference : null, + ); + const repeatPath = path.join(directory, "repeat.yaml"); + await host.command( + "node", + [CLI, "config", "export", SANDBOX_NAME, "--output", repeatPath, "--json"], + { artifactName: "export-ollama-repeat", cwd: REPO_ROOT, env: exportEnv, timeoutMs: 60000 }, + ); + expect(validateNemoClawConfig(YAML.parse(fs.readFileSync(repeatPath, "utf8"))).spec).toEqual( + document.spec, + ); + + progress.phase("refuse export after the attached daemon stops"); + await daemonOwner.terminate(); + const rejectedPath = path.join(directory, "must-not-exist.yaml"); + const rejected = await host.command( + "node", + [CLI, "config", "export", SANDBOX_NAME, "--output", rejectedPath, "--json"], + { + artifactName: "export-ollama-stopped-daemon", + cwd: REPO_ROOT, + env: exportEnv, + timeoutMs: 60000, + }, + ); + expect(rejected.exitCode, resultText(rejected)).not.toBe(0); + expect(fs.existsSync(rejectedPath), "A stopped daemon must prevent publication").toBe(false); + await artifacts.writeJson("ollama-config-export-evidence.json", { + sandboxName: SANDBOX_NAME, + daemonPort: 11439, + proxyPort: 11440, + model: "qwen3.5:9b", + image: document.spec.sandboxes[0].runtime.image.ref, + repeatedSpecMatches: true, + stoppedDaemonPreventedPublication: true, + }); + }, +); diff --git a/test/e2e/support/gpu-e2e-helpers.test.ts b/test/e2e/support/gpu-e2e-helpers.test.ts index b1cfafaa90a..cdcdab04fd6 100644 --- a/test/e2e/support/gpu-e2e-helpers.test.ts +++ b/test/e2e/support/gpu-e2e-helpers.test.ts @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; +import { ChildProcess, execFileSync } from "node:child_process"; import { once } from "node:events"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; @@ -20,7 +20,11 @@ import { hasExactReadyPhase, ollamaCleanupScript, openClawModelConfigProjectionScript, + REPO_ROOT, + startAttachedOllama, } from "../live/gpu-e2e-helpers.ts"; +import * as observedChild from "../fixtures/observed-child-process.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; import { PROTECTED_OLLAMA_CURL_MAX_SECONDS, PROTECTED_OLLAMA_READY_ATTEMPTS, @@ -160,6 +164,52 @@ const invalidExecutionProofs: Array<{ ]; describe("GPU E2E helpers", () => { + it("owns the attached Ollama daemon with the supplied listener and cleanup (#11435)", async () => { + const progress = startTestProgress( + "attached Ollama helper", + ["start the attached daemon", "close the owned daemon"], + { + logLine: () => undefined, + }, + ); + const child = new ChildProcess(); + const spawn = vi.spyOn(observedChild, "spawnObservedChild").mockReturnValue(child); + const kill = vi.spyOn(child, "kill").mockReturnValue(true); + const environment = { OLLAMA_HOST: "127.0.0.1:11444", OLLAMA_MODELS: "/tmp/owned-models" }; + try { + const owner = startAttachedOllama(progress, environment); + expect(spawn).toHaveBeenCalledExactlyOnceWith( + "ollama", + ["serve"], + expect.objectContaining({ + progress, + spawn: { cwd: REPO_ROOT, env: environment, stdio: "ignore" }, + }), + ); + expect(owner.child).toBe(child); + let closed = false; + void owner.closed.then(() => { + closed = true; + }); + let terminated = false; + const termination = owner.terminate().then(() => { + terminated = true; + }); + expect(kill).toHaveBeenCalledExactlyOnceWith("SIGTERM"); + await Promise.resolve(); + expect([closed, terminated]).toEqual([false, false]); + child.emit("close", null, "SIGTERM"); + await termination; + await owner.closed; + expect(closed).toBe(true); + await owner.terminate(); + expect(kill).toHaveBeenCalledTimes(1); + } finally { + child.emit("close", null, "SIGTERM"); + progress.stop(); + } + }); + it("stops the Ollama system service before cleanup completes", async () => { const root = mkdtempSync(path.join(tmpdir(), "nemoclaw-ollama-cleanup-")); const listenerPort = await unusedLoopbackPort(); diff --git a/test/inference/ollama/ollama-auth-proxy-handler.test.ts b/test/inference/ollama/ollama-auth-proxy-handler.test.ts index 4cb03fc3fdd..85a8d3ce4c6 100644 --- a/test/inference/ollama/ollama-auth-proxy-handler.test.ts +++ b/test/inference/ollama/ollama-auth-proxy-handler.test.ts @@ -68,6 +68,60 @@ describe("ollama-auth-proxy request handler", () => { expect(backend?.captured).toHaveLength(0); }); + it.each([undefined, "Bearer wrong-token"])( + "protects active configuration with the proxy token (#11435)", + async (auth) => { + const response = await request(proxyPort, { path: "/_nemoclaw/proxy-config", auth }); + expect(response.status).toBe(401); + expect(backend?.captured).toHaveLength(0); + }, + ); + + it("reports the active listener and backend without forwarding or credentials (#11435)", async () => { + const response = await request(proxyPort, { + path: "/_nemoclaw/proxy-config", + auth: `Bearer ${TOKEN}`, + }); + expect(response.status).toBe(200); + expect(JSON.parse(response.body)).toEqual({ + schemaVersion: 1, + pid: proxy?.pid, + listener: { address: "0.0.0.0", port: proxyPort }, + backendOrigin: `http://localhost:${backend?.port}`, + }); + expect(response.body).not.toContain(TOKEN); + expect(backend?.captured).toHaveLength(0); + }); + + it.each(["POST", "PUT", "DELETE"])( + "rejects %s for active configuration without forwarding (#11435)", + async (method) => { + const response = await request(proxyPort, { + path: "/_nemoclaw/proxy-config", + method, + auth: `Bearer ${TOKEN}`, + body: "{}", + }); + expect(response.status).toBe(405); + expect(backend?.captured).toHaveLength(0); + }, + ); + + it("omits backend userinfo, path, and query from active configuration (#11435)", async () => { + await terminate(proxy); + proxy = await startProxy(proxyPort, backend!.port, TOKEN, { + backendUrl: `http://user:secret@localhost:${backend!.port}/private?canary=value`, + }); + const response = await request(proxyPort, { + path: "/_nemoclaw/proxy-config", + auth: `Bearer ${TOKEN}`, + }); + expect(response.status).toBe(200); + expect(JSON.parse(response.body).backendOrigin).toBe(`http://localhost:${backend!.port}`); + expect(response.body).not.toMatch(/user|secret|private|canary|value/u); + expect(backend?.captured).toHaveLength(0); + }); + it("returns 401 for unauthenticated /api/tags — no health-check bypass (#3338)", async () => { const res = await request(proxyPort, { path: "/api/tags" }); expect(res.status).toBe(401); diff --git a/test/inference/ollama/ollama-proxy-export.test.ts b/test/inference/ollama/ollama-proxy-export.test.ts new file mode 100644 index 00000000000..428055122e1 --- /dev/null +++ b/test/inference/ollama/ollama-proxy-export.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { expect } from "vitest"; +import { test } from "../../helpers/owned-test-resources"; +import { + closeServer, + freePort, + startBackend, + startProxy, + terminate, +} from "../../ollama-auth-proxy-handler-helpers"; + +const execFileAsync = promisify(execFile); + +test.skipIf(process.platform !== "linux")( + "observes existing proxy state without migration or credential exposure (#11435)", + async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-proxy-export-")); + const state = path.join(directory, ".nemoclaw"); + const token = "export-credential-canary"; + const digest = "a".repeat(64); + const backend = await startBackend({ ok: true, models: [{ name: "qwen3.5:9b", digest }] }); + let proxy; + try { + const port = await freePort(); + proxy = await startProxy(port, backend.port, token); + fs.mkdirSync(state); + const retained = { + "ollama-proxy-token": token, + "ollama-auth-proxy.pid": String(proxy.pid), + "ollama-proxy-port": String(port), + "ollama-backend": `http://127.0.0.1:${backend.port}`, + "ollama-backend.json": JSON.stringify({ + schemaVersion: 1, + kind: "ollama", + url: `http://127.0.0.1:${backend.port}`, + }), + }; + fs.writeFileSync(path.join(state, "ollama-proxy-token"), retained["ollama-proxy-token"], { + mode: 0o600, + }); + fs.writeFileSync( + path.join(state, "ollama-auth-proxy.pid"), + retained["ollama-auth-proxy.pid"], + { + mode: 0o600, + }, + ); + fs.writeFileSync(path.join(state, "ollama-proxy-port"), retained["ollama-proxy-port"], { + mode: 0o600, + }); + fs.writeFileSync(path.join(state, "ollama-backend"), retained["ollama-backend"], { + mode: 0o600, + }); + fs.writeFileSync(path.join(state, "ollama-backend.json"), retained["ollama-backend.json"], { + mode: 0o600, + }); + const script = ` +const fs = require("node:fs"); +const {createOllamaExportProbe} = require(${JSON.stringify(path.resolve(import.meta.dirname, "../../../src/lib/inference/ollama/proxy.ts"))}); +const {observeOllamaProxy} = require(${JSON.stringify(path.resolve(import.meta.dirname, "../../../src/lib/inference/ollama/proxy-observation.ts"))}); +const originalOpen = fs.openSync; +let credentialReads = 0; +fs.openSync = (...args) => { + credentialReads += Number(args[0] === ${JSON.stringify(path.join(state, "ollama-proxy-token"))}); + return originalOpen(...args); +}; +const rejection = (input) => { + try { observeOllamaProxy(input); return "unexpected success"; } + catch (error) { return error.message; } +}; +process.env.WSL_DISTRO_NAME = "unsupported-export-host"; +const unsupportedHost = rejection({model: "qwen3.5:9b", ...createOllamaExportProbe()}); +delete process.env.WSL_DISTRO_NAME; +const invalidMetadata = rejection({model: "qwen3.5:9b", ...createOllamaExportProbe(), proxyPort: "invalid"}); +const credentialReadsBeforeValid = credentialReads; +const observed = observeOllamaProxy({model: "qwen3.5:9b", ...createOllamaExportProbe()}); +process.stdout.write(JSON.stringify({observed, unsupportedHost, invalidMetadata, credentialReadsBeforeValid, credentialReads})); +`; + const { stdout, stderr } = await execFileAsync(process.execPath, ["-e", script], { + env: { + ...process.env, + HOME: directory, + NEMOCLAW_OLLAMA_PORT: "11434", + NEMOCLAW_OLLAMA_PROXY_PORT: "11435", + WSL_DISTRO_NAME: "", + WSL_INTEROP: "", + HTTP_PROXY: "http://127.0.0.1:1", + http_proxy: "http://127.0.0.1:1", + ALL_PROXY: "http://127.0.0.1:1", + all_proxy: "http://127.0.0.1:1", + NO_PROXY: "", + no_proxy: "", + }, + timeout: 30000, + }); + expect(JSON.parse(stdout)).toMatchObject({ + observed: { + pid: proxy.pid, + serving: { + daemon: { management: "external", hostPort: backend.port }, + proxy: { management: "nemoclaw", hostPort: port }, + model: { digest: `sha256:${digest}` }, + }, + }, + unsupportedHost: "Ollama export requires a native Linux host.", + invalidMetadata: + "The live Ollama daemon, proxy mapping, or model could not be verified for export.", + credentialReadsBeforeValid: 0, + credentialReads: 1, + }); + expect(stdout + stderr).not.toContain(token); + expect( + Object.fromEntries( + fs + .readdirSync(state) + .map((name) => [name, fs.readFileSync(path.join(state, name), "utf8")]), + ), + ).toEqual(retained); + expect(backend.captured).toHaveLength(2); + expect(backend.captured.every(({ headers }) => headers.authorization === undefined)).toBe( + true, + ); + } finally { + await terminate(proxy); + await closeServer(backend.server); + fs.rmSync(directory, { recursive: true, force: true }); + } + }, +); diff --git a/test/ollama-auth-proxy-handler-helpers.ts b/test/ollama-auth-proxy-handler-helpers.ts index a8ee0ffbcbf..350835c6aba 100644 --- a/test/ollama-auth-proxy-handler-helpers.ts +++ b/test/ollama-auth-proxy-handler-helpers.ts @@ -35,7 +35,7 @@ export type StartProxyOptions = { }; /** Start a loopback stub backend that records the request it received. */ -export function startBackend(): Promise<{ +export function startBackend(response = { ok: true, models: [] as unknown[] }): Promise<{ server: http.Server; port: number; captured: BackendCapture[]; @@ -51,7 +51,7 @@ export function startBackend(): Promise<{ req.resume(); req.on("end", () => { res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ ok: true, models: [] })); + res.end(JSON.stringify(response)); }); }); return new Promise((resolve) => { diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 98ceac67a7e..b76a896ead3 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -392,6 +392,7 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map