From 78866df3c207d3502ed3e6f88ff2c12dd90bef2b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 10 Sep 2026 09:12:14 -0700 Subject: [PATCH 1/6] feat(config): export attached Ollama daemon and managed proxy ports Observe retained native Ollama intent through fixed authenticated proxy reads, and preserve daemon and proxy ownership in the shared configuration export. Extend the existing managed Ollama qualification and approved assertion budget. Fixes #11435 Refs #10904 Signed-off-by: Carlos Villela --- ci/e2e-assertion-budget.json | 18 +- schemas/nemoclaw-config-v1.schema.json | 79 +++++++ scripts/ollama-auth-proxy.mts | 30 ++- .../actions/config/observe-export-source.ts | 1 + .../config/live-export-source.test.ts | 197 ++++++++++++++++++ src/lib/adapters/config/live-export-source.ts | 55 +++-- src/lib/config/config.test.ts | 77 +++++++ src/lib/config/model.ts | 32 +++ src/lib/config/schema.ts | 21 ++ src/lib/domain/config/export-document.ts | 24 ++- src/lib/domain/config/export-evidence.ts | 14 ++ src/lib/domain/config/verify-export-source.ts | 20 +- .../domain/config/verify-managed-serving.ts | 2 +- .../domain/config/verify-ollama-serving.ts | 71 +++++++ src/lib/inference/README.md | 33 +++ .../ollama/proxy-observation.test.ts | 123 +++++++++++ src/lib/inference/ollama/proxy-observation.ts | 128 ++++++++++++ src/lib/inference/ollama/proxy.ts | 32 +++ test/e2e/README.md | 9 + test/e2e/live/gpu-e2e-helpers.ts | 13 ++ test/e2e/live/gpu-e2e.test.ts | 185 ++++++++++++++++ .../ollama/ollama-auth-proxy-handler.test.ts | 54 +++++ .../ollama/ollama-proxy-export.test.ts | 136 ++++++++++++ test/ollama-auth-proxy-handler-helpers.ts | 4 +- tools/e2e/check-semantic-phases.mts | 1 + tools/e2e/target-catalogue.mts | 11 +- 26 files changed, 1333 insertions(+), 37 deletions(-) create mode 100644 src/lib/domain/config/verify-ollama-serving.ts create mode 100644 src/lib/inference/ollama/proxy-observation.test.ts create mode 100644 src/lib/inference/ollama/proxy-observation.ts create mode 100644 test/inference/ollama/ollama-proxy-export.test.ts diff --git a/ci/e2e-assertion-budget.json b/ci/e2e-assertion-budget.json index 17a045104e2..c805609e014 100644 --- a/ci/e2e-assertion-budget.json +++ b/ci/e2e-assertion-budget.json @@ -15,26 +15,26 @@ "testFileCount": 86, "liveFileCount": 222, "direct": { - "expectCalls": 1798, - "matcherAssertions": 1767, + "expectCalls": 1809, + "matcherAssertions": 1778, "nodeAssertions": 105, "namedAssertionHelpers": 601, "failCalls": 8, "throwGuards": 87, - "objectFieldAssertions": 243, - "assertionPoints": 2811, + "objectFieldAssertions": 252, + "assertionPoints": 2831, "generatedProbeBlocks": 122, "generatedProbeConditions": 322 }, "unique": { - "expectCalls": 2290, - "matcherAssertions": 2254, + "expectCalls": 2301, + "matcherAssertions": 2265, "nodeAssertions": 126, "namedAssertionHelpers": 905, "failCalls": 38, "throwGuards": 633, - "objectFieldAssertions": 342, - "assertionPoints": 4298, + "objectFieldAssertions": 351, + "assertionPoints": 4318, "generatedProbeBlocks": 276, "generatedProbeConditions": 943 }, @@ -65,7 +65,7 @@ "test/e2e/live/full-e2e.test.ts": [30,34,39,72,7], "test/e2e/live/gateway-guard-recovery.test.ts": [48,54,48,57,3], "test/e2e/live/gpu-double-onboard.test.ts": [21,24,21,24,0], - "test/e2e/live/gpu-e2e.test.ts": [40,45,73,91,3], + "test/e2e/live/gpu-e2e.test.ts": [51,65,84,111,3], "test/e2e/live/hermes-discord.test.ts": [10,25,17,64,14], "test/e2e/live/hermes-e2e.test.ts": [82,93,112,127,3], "test/e2e/live/hermes-gpu-startup.test.ts": [19,27,63,106,12], diff --git a/schemas/nemoclaw-config-v1.schema.json b/schemas/nemoclaw-config-v1.schema.json index d293ce3ad72..5ca3950e4c0 100644 --- a/schemas/nemoclaw-config-v1.schema.json +++ b/schemas/nemoclaw-config-v1.schema.json @@ -218,6 +218,85 @@ } }, "additionalProperties": false + }, + { + "type": "object", + "required": ["name", "provider", "api", "serving"], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$" + }, + "provider": { + "type": "string", + "const": "ollama-local" + }, + "api": { + "type": "string", + "const": "openai-completions" + }, + "serving": { + "type": "object", + "required": ["backend", "daemon", "proxy", "model"], + "properties": { + "backend": { + "type": "string", + "const": "ollama" + }, + "daemon": { + "type": "object", + "required": ["management", "hostPort"], + "properties": { + "management": { + "type": "string", + "const": "external" + }, + "hostPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "additionalProperties": false + }, + "proxy": { + "type": "object", + "required": ["management", "hostPort"], + "properties": { + "management": { + "type": "string", + "const": "nemoclaw" + }, + "hostPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + }, + "additionalProperties": false + }, + "model": { + "type": "object", + "required": ["servedName", "digest"], + "properties": { + "servedName": { + "type": "string", + "const": "qwen3.5:9b" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } ] }, diff --git a/scripts/ollama-auth-proxy.mts b/scripts/ollama-auth-proxy.mts index 513292806a2..d22b96e3738 100755 --- a/scripts/ollama-auth-proxy.mts +++ b/scripts/ollama-auth-proxy.mts @@ -342,7 +342,7 @@ function assertBackendBoundToLoopback(port: number): void { function buildProxyServer(token: string, backendUrl: URL): http.Server { const expectedBuf = Buffer.from(`Bearer ${token}`); - return http.createServer((clientReq, clientRes) => { + const server = http.createServer((clientReq, clientRes) => { // Every request must present a valid Bearer token. The proxy binds 0.0.0.0 // so the OpenShell sandbox container can reach it via the docker bridge — // which also means anything else with network reach to the host could, @@ -366,6 +366,33 @@ function buildProxyServer(token: string, backendUrl: URL): http.Server { return; } + if (clientReq.url === "/_nemoclaw/proxy-config") { + if (clientReq.method !== "GET") { + clientRes.writeHead(405, { Allow: "GET" }); + clientRes.end(); + return; + } + const listener = server.address(); + if (Buffer.byteLength(backendUrl.origin) > 512) { + clientRes.writeHead(503); + clientRes.end(); + return; + } + clientRes.writeHead(200, { "Content-Type": "application/json", "Cache-Control": "no-store" }); + clientRes.end( + JSON.stringify({ + schemaVersion: 1, + pid: process.pid, + listener: + typeof listener === "object" && listener !== null + ? { address: listener.address, port: listener.port } + : null, + backendOrigin: backendUrl.origin, + }), + ); + return; + } + // Strip the auth header before forwarding to Ollama const headers = { ...clientReq.headers }; delete headers.authorization; @@ -407,6 +434,7 @@ function buildProxyServer(token: string, backendUrl: URL): http.Server { clientReq.pipe(proxyReq); }); + return server; } function shouldProbeBackendHostname(hostname: string): boolean { diff --git a/src/lib/actions/config/observe-export-source.ts b/src/lib/actions/config/observe-export-source.ts index 1a76b25431b..9aec67b4fe0 100644 --- a/src/lib/actions/config/observe-export-source.ts +++ b/src/lib/actions/config/observe-export-source.ts @@ -86,6 +86,7 @@ const LIVE_READ_SOURCE_LABELS = { "provider-metadata": "live inference provider metadata", "web-search-provider": "live web-search provider metadata", "managed-serving": "managed serving runtime", + "ollama-serving": "live Ollama daemon and proxy mapping", "effective-policy": "effective OpenShell policy", } satisfies Readonly>; diff --git a/src/lib/adapters/config/live-export-source.test.ts b/src/lib/adapters/config/live-export-source.test.ts index a92cf53a523..0c5ccc3fc83 100644 --- a/src/lib/adapters/config/live-export-source.test.ts +++ b/src/lib/adapters/config/live-export-source.test.ts @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import os from "node:os"; import { describe, expect, it, vi } from "vitest"; import YAML from "yaml"; import { managedBraveProfile } from "../../../../test/fixtures/openshell-provider-profile"; import { runConfigExport } from "../../actions/config/export"; import { + EXPORTED_OLLAMA_MODEL, parseNemoClawConfigDocumentName, EXPORTED_VLLM_PROFILE_ID, EXPORTED_VLLM_RECIPE_ID, @@ -17,6 +19,11 @@ import { validateNemoClawConfig } from "../../config/schema"; vi.mock("../../inference/serving/vllm-export-runtime", () => ({ observeManagedVllmForExport: vi.fn(), })); +vi.mock("../../inference/ollama/proxy", () => ({ createOllamaExportProbe: vi.fn() })); +vi.mock("../../platform", async (importOriginal) => ({ + ...(await importOriginal()), + isWsl: vi.fn(() => false), +})); vi.mock("../../state/registry/persistence", () => ({ load: vi.fn() })); vi.mock("../../state/registry-entry-view", () => ({ getSandboxEntryInference: vi.fn() })); vi.mock("../../inference/live", () => ({ getLiveGatewayInference: vi.fn() })); @@ -38,6 +45,9 @@ vi.mock("../../onboard/gateway/state-dir", () => ({ })); import { observeManagedVllmForExport } from "../../inference/serving/vllm-export-runtime"; +import { createOllamaExportProbe } from "../../inference/ollama/proxy"; +import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "../../inference/ollama/contract"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; import { loadServingCatalog } from "../../inference/serving/catalog-loader"; import { servingProfileProvenance } from "../../inference/serving/profile-provenance"; import { applyVllmRuntimeContextWindow } from "../../inference/vllm-runtime-context"; @@ -1167,3 +1177,190 @@ describe("managed vLLM export pipeline", () => { expect(raw.getProvider).not.toHaveBeenCalled(); }); }); + +function ollamaProbe(observed: ObservedOllamaProxy) { + const models = JSON.stringify({ + models: [{ name: observed.serving.model.servedName, digest: observed.serving.model.digest }], + }); + return { + backend: { + kind: "ollama" as const, + url: `http://127.0.0.1:${observed.serving.daemon.hostPort}`, + }, + proxyPort: String(observed.serving.proxy.hostPort), + pid: String(observed.pid), + processMatches: vi.fn(() => true), + readActiveConfig: vi.fn(() => + JSON.stringify({ + schemaVersion: 1, + pid: observed.pid, + listener: { address: observed.listenerAddress, port: observed.serving.proxy.hostPort }, + backendOrigin: `http://127.0.0.1:${observed.serving.daemon.hostPort}`, + }), + ), + readProxyModels: vi.fn(() => models), + readDaemonModels: vi.fn(() => models), + }; +} + +function mockOllamaSource() { + vi.spyOn(os, "platform").mockReturnValue("linux"); + const model = EXPORTED_OLLAMA_MODEL; + const route = resolveManagedStartupInferenceRoute( + "openclaw", + "ollama-local", + model, + "openai-completions", + ); + const built = buildManagedStartupProfile({ + ...startupInput, + inference: { + routeProvider: route.providerKey, + upstreamProvider: "ollama-local", + model, + routedBaseUrl: route.inferenceBaseUrl, + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: route.primaryModelRef, + compatibility: route.inferenceCompat ?? {}, + }, + }); + const source: SandboxEntry = { + ...entry, + provider: "ollama-local", + model, + endpointUrl: "http://host.openshell.internal:11440/v1", + credentialEnv: OLLAMA_LOCAL_CREDENTIAL_ENV, + workload: { + ...entry.workload!, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + } as SandboxEntry["workload"], + }; + const observed: ObservedOllamaProxy = { + pid: 1234, + listenerAddress: "0.0.0.0", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { servedName: model, digest: `sha256:${"a".repeat(64)}` }, + }, + }; + mockSupportedLiveSource(3, 3, source); + const probe = ollamaProbe(observed); + vi.mocked(createOllamaExportProbe).mockReturnValue(probe); + vi.mocked(getSandboxEntryInference).mockReturnValue({ + kind: "configured", + provider: "ollama-local", + model, + }); + vi.mocked(getLiveGatewayInference).mockReturnValue({ + failure: null, + inference: { provider: "ollama-local", model }, + output: "", + status: 0, + }); + const liveSandbox = inventory(); + Object.assign(liveSandbox.sandbox.spec, { providers: ["ollama-local"] }); + raw.getSandbox.mockResolvedValue(liveSandbox); + const readCredential = vi.fn(() => { + throw new Error(readFailureCanary); + }); + const credentials = Object.defineProperty({}, OLLAMA_LOCAL_CREDENTIAL_ENV, { + enumerable: true, + get: readCredential, + }); + const localProvider = { + ...provider().provider, + metadata: { ...provider().provider.metadata, name: "ollama-local" }, + profileWorkspace: "default", + credentials, + config: { OPENAI_BASE_URL: source.endpointUrl }, + }; + raw.getProvider.mockResolvedValue({ provider: localProvider }); + raw.getProviderProfile.mockResolvedValue({ + profile: { + id: "openai", + source: "user", + scope: "workspace", + resourceVersion: 4n, + credentials: [], + endpoints: [], + binaries: [], + inferenceCapable: true, + }, + }); + return { source, observed, probe, readCredential, localProvider }; +} + +describe("attached Ollama export pipeline", () => { + it("publishes the complete managed document without reading gateway credentials (#11435)", async () => { + const { observed, probe, readCredential } = mockOllamaSource(); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); + const yaml = writeStdout.mock.calls[0]![0]; + const document = validateNemoClawConfig(YAML.parse(yaml)); + expect(document.spec.inferenceProviders).toEqual([ + { + name: "local-ollama", + provider: "ollama-local", + api: "openai-completions", + serving: observed.serving, + }, + ]); + expect(document.spec.sandboxes[0]!.agents[0]!.inference.routes).toEqual([ + { name: "primary", providerRef: "local-ollama", overrides: { model: EXPORTED_OLLAMA_MODEL } }, + ]); + expect(probe.readActiveConfig).toHaveBeenCalledWith(11440); + expect(probe.readDaemonModels).toHaveBeenCalledWith(11439); + expect(readCredential).not.toHaveBeenCalled(); + expect(yaml).not.toMatch( + /NEMOCLAW_OLLAMA_PROXY_TOKEN|credential-canary-value|host\.openshell\.internal/u, + ); + expect(publish).not.toHaveBeenCalled(); + }); + + it("sanitizes a failed or legacy proxy observation and publishes nothing (#11435)", async () => { + mockOllamaSource(); + vi.mocked(createOllamaExportProbe).mockImplementation(() => { + throw new Error(readFailureCanary); + }); + const { result, writeStdout } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(JSON.stringify(result)).not.toContain(readFailureCanary); + expect(writeStdout).not.toHaveBeenCalled(); + }); + + it.each([ + { endpointUrl: "http://host.openshell.internal:11435/v1" }, + { credentialEnv: "OTHER_TOKEN" }, + { agent: "hermes" }, + { sandboxGpuEnabled: true, sandboxGpuDevice: "nvidia.com/gpu=all" }, + ])("refuses unsupported or drifted local route intent %# (#11435)", async (change) => { + const { source } = mockOllamaSource(); + Object.assign(source, change); + const { result, writeStdout } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(writeStdout).not.toHaveBeenCalled(); + }); + + it("refuses an absent provider attachment (#11435)", async () => { + mockOllamaSource(); + raw.getSandbox.mockResolvedValue(inventory()); + const { result, writeStdout } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(writeStdout).not.toHaveBeenCalled(); + }); + + it("refuses a continuously changing active proxy identity (#11435)", async () => { + const { observed } = mockOllamaSource(); + let pid = observed.pid; + vi.mocked(createOllamaExportProbe).mockImplementation(() => + ollamaProbe({ ...observed, pid: ++pid }), + ); + const { result, writeStdout } = await exportLiveSource(); + expect(result).toMatchObject({ ok: false }); + expect(writeStdout).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/adapters/config/live-export-source.ts b/src/lib/adapters/config/live-export-source.ts index a52c07192ea..e3aad21d2ac 100644 --- a/src/lib/adapters/config/live-export-source.ts +++ b/src/lib/adapters/config/live-export-source.ts @@ -27,6 +27,8 @@ import type { import { getLiveGatewayInference } from "../../inference/live"; import { VLLM_LOCAL_CREDENTIAL_ENV } from "../../inference/serving/vllm-credential-contract"; import { observeManagedVllmForExport } from "../../inference/serving/vllm-export-runtime"; +import { createOllamaExportProbe } from "../../inference/ollama/proxy"; +import { observeOllamaProxy } from "../../inference/ollama/proxy-observation"; import { normalizeInferenceSelection } from "../../inference/selection"; import { resolveGatewayName } from "../../onboard/gateway-binding/identity"; import { @@ -149,11 +151,34 @@ function inferenceTopology( managed: boolean, ): ObservedExportInference["topology"] { if (managed) return "managed"; + if (entry.provider === "ollama-local") return "local"; return entry.hostLocalInferenceReceipt || entry.hostLocalInferenceProvenance || entry.nimContainer ? "local" : "hosted"; } +function matchesProviderMetadata( + provider: Provider, + normalized: ReturnType, + routeProvider: string, + managed: boolean, +): boolean { + const { type, configKey } = providerContract(normalized.preferredInferenceApi); + const builtin = provider.builtinInferenceEndpoint !== undefined; + return ( + type !== null && + isDeepStrictEqual( + [provider.name, provider.type, provider.credentialKeys, provider.configKeys], + [ + routeProvider, + builtin ? "nvidia" : type, + expectedCredentialKeys(normalized.credentialEnv, managed), + builtin ? [] : [configKey], + ], + ) + ); +} + async function readProviderEvidence( normalized: ReturnType, routeProvider: string, @@ -161,29 +186,23 @@ async function readProviderEvidence( signal: AbortSignal, managedServing?: ObservedManagedVllmRuntime, ): Promise { - const { type, configKey } = providerContract(normalized.preferredInferenceApi); + const { configKey } = providerContract(normalized.preferredInferenceApi); + const managedProfile = !!managedServing || routeProvider === "ollama-local"; const provider = await createProviders().get({ target: namedOpenShellGateway(gatewayName), workspace: "default", name: routeProvider, - ...(managedServing ? { profileContract: "openai" as const } : {}), + ...(managedProfile ? { profileContract: "openai" as const } : {}), configKeys: [configKey], signal, }); if (!provider) throw new Error("The live inference provider is missing."); - const credentialKeys = expectedCredentialKeys(normalized.credentialEnv, !!managedServing); const builtin = provider.builtinInferenceEndpoint !== undefined; - if ( - type === null || - !isDeepStrictEqual( - [provider.name, provider.type, provider.credentialKeys, provider.configKeys], - [routeProvider, builtin ? "nvidia" : type, credentialKeys, builtin ? [] : [configKey]], - ) - ) { + if (!matchesProviderMetadata(provider, normalized, routeProvider, !!managedServing)) { throw new Error("The live inference provider metadata does not match the registry."); } return { - provider: providerIdentity(provider, gatewayName, !!managedServing), + provider: providerIdentity(provider, gatewayName, managedProfile), endpoint: provider.builtinInferenceEndpoint ?? provider.config[configKey] ?? "", source: builtin ? { kind: "builtin-profile", profileId: "nvidia" } @@ -193,14 +212,14 @@ async function readProviderEvidence( async function inferenceFor( entry: Readonly, - beforeProviderRead: () => void, + beforeRead: (stage: ExportSnapshotReadStage) => void, signal: AbortSignal, managedServing?: ObservedManagedVllmRuntime, ): Promise { const normalized = normalizeInferenceSelection(entry); const gateway = resolveGatewayBinding(entry); const live = readInferenceRoute(entry, gateway.name); - beforeProviderRead(); + beforeRead("provider-metadata"); const endpointEvidence = await readProviderEvidence( normalized, live.provider, @@ -208,6 +227,11 @@ async function inferenceFor( signal, managedServing, ); + let ollamaServing: ObservedExportInference["ollamaServing"]; + if (entry.provider === "ollama-local") { + beforeRead("ollama-serving"); + ollamaServing = observeOllamaProxy({ model: live.model, ...createOllamaExportProbe() }); + } return { topology: inferenceTopology(entry, !!managedServing), provider: live.provider, @@ -217,6 +241,7 @@ async function inferenceFor( endpointEvidence, credentialEnv: normalized.credentialEnv, ...(managedServing ? { managedServing } : {}), + ...(ollamaServing ? { ollamaServing } : {}), }; } @@ -305,8 +330,8 @@ async function readSnapshot(sandboxName: string): Promise { stage = "inference-route"; const inference = await inferenceFor( entry, - () => { - stage = "provider-metadata"; + (nextStage) => { + stage = nextStage; }, signal, managedServing, diff --git a/src/lib/config/config.test.ts b/src/lib/config/config.test.ts index f5e4a9e3b2e..ff518edd139 100644 --- a/src/lib/config/config.test.ts +++ b/src/lib/config/config.test.ts @@ -5,6 +5,7 @@ import YAML from "yaml"; import { describe, expect, it } from "vitest"; import { renderCanonicalNemoClawConfig, validateNemoClawConfig } from "./index"; import { + EXPORTED_OLLAMA_MODEL, EXPORTED_VLLM_PROFILE_ID, EXPORTED_VLLM_RECIPE_ID, isCredentialEnvironmentReferenceName, @@ -81,6 +82,11 @@ function renderInput(value: unknown) { } describe("NemoClawConfig v1", () => { + it("preserves an earlier valid hosted document using the ollama-local label (#11435)", () => { + const value = config(); + value.spec.inferenceProviders[0]!.provider = "ollama-local"; + expect(validateNemoClawConfig(value)).toEqual(value); + }); it("validates one aggregate config with explicit effective policy (#10938)", () => { expect(validateNemoClawConfig(config())).toEqual(config()); }); @@ -642,3 +648,74 @@ describe("fixed managed serving public contract", () => { expect(() => validateNemoClawConfig(f.value)).toThrow(); }); }); + +function ollamaConfig() { + const value = config(); + value.spec.sandboxes[0]!.agents[0]!.inference.routes[0]!.overrides.model = EXPORTED_OLLAMA_MODEL; + return { + ...value, + spec: { + ...value.spec, + inferenceProviders: [ + { + name: "hosted-openai", + provider: "ollama-local", + api: "openai-completions", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { servedName: EXPORTED_OLLAMA_MODEL, digest: `sha256:${"a".repeat(64)}` }, + }, + }, + ], + }, + }; +} + +describe("attached Ollama serving public contract", () => { + it("round trips the external daemon separately from its managed proxy (#11435)", () => { + const value = ollamaConfig(); + const rendered = renderCanonicalNemoClawConfig(validateNemoClawConfig(value)); + expect(validateNemoClawConfig(YAML.parse(rendered.yaml))).toEqual(value); + }); + + it.each([ + { daemon: { management: "nemoclaw", hostPort: 11439 } }, + { daemon: { management: "external", hostPort: 11440 } }, + { proxy: { management: "external", hostPort: 11440 } }, + { proxy: { management: "nemoclaw", hostPort: 65536 } }, + { model: { servedName: "other:tag", digest: `sha256:${"a".repeat(64)}` } }, + { model: { servedName: EXPORTED_OLLAMA_MODEL, digest: "not-a-digest" } }, + { runtime: { image: { ref: "ollama:latest" } } }, + ])("rejects unsupported lifecycle or model declarations %# (#11435)", (change) => { + const value = ollamaConfig(); + Object.assign(value.spec.inferenceProviders[0]!.serving, change); + expect(() => validateNemoClawConfig(value)).toThrow(); + }); + + it.each([ + { endpoint: "http://127.0.0.1:11439/v1" }, + { credential: { env: "NEMOCLAW_OLLAMA_PROXY_TOKEN" } }, + ])( + "rejects internal transport or credential fields on the local branch %# (#11435)", + (change) => { + const value = ollamaConfig(); + Object.assign(value.spec.inferenceProviders[0]!, change); + expect(() => validateNemoClawConfig(value)).toThrow(); + }, + ); + + it.each([ + { agent: "hermes", runtime: "docker", model: EXPORTED_OLLAMA_MODEL }, + { agent: "openclaw", runtime: "remote", model: EXPORTED_OLLAMA_MODEL }, + { agent: "openclaw", runtime: "docker", model: "different-model" }, + ])("rejects an unsupported local consumer %s (#11435)", (change) => { + const value = ollamaConfig(); + const sandbox = value.spec.sandboxes[0]!; + sandbox.agents[0]!.type = change.agent; + sandbox.runtime.provider = change.runtime; + sandbox.agents[0]!.inference.routes[0]!.overrides.model = change.model; + expect(() => validateNemoClawConfig(value)).toThrow(); + }); +}); diff --git a/src/lib/config/model.ts b/src/lib/config/model.ts index 7da4ab70d41..48782dcba46 100644 --- a/src/lib/config/model.ts +++ b/src/lib/config/model.ts @@ -312,9 +312,41 @@ const NemoClawManagedInferenceProviderConfigSchema = Type.Object( { additionalProperties: false }, ); +export const EXPORTED_OLLAMA_MODEL = "qwen3.5:9b" as const; +export const NemoClawOllamaServingSchema = Type.Object( + { + backend: Type.Literal("ollama"), + daemon: Type.Object( + { management: Type.Literal("external"), hostPort: TcpPortSchema }, + { additionalProperties: false }, + ), + proxy: Type.Object( + { management: Type.Literal("nemoclaw"), hostPort: TcpPortSchema }, + { additionalProperties: false }, + ), + model: Type.Object( + { servedName: Type.Literal(EXPORTED_OLLAMA_MODEL), digest: ServingDigestSchema }, + { additionalProperties: false }, + ), + }, + { additionalProperties: false }, +); +export type NemoClawOllamaServing = TypeBoxModule.Type.Static; + +const NemoClawOllamaInferenceProviderConfigSchema = Type.Object( + { + name: LocalResourceNameSchema, + provider: Type.Literal("ollama-local"), + api: Type.Literal("openai-completions"), + serving: NemoClawOllamaServingSchema, + }, + { additionalProperties: false }, +); + const NemoClawInferenceProviderConfigSchema = Type.Union([ NemoClawHostedInferenceProviderConfigSchema, NemoClawManagedInferenceProviderConfigSchema, + NemoClawOllamaInferenceProviderConfigSchema, ]); const NemoClawRouteOverridesSchema = Type.Object( diff --git a/src/lib/config/schema.ts b/src/lib/config/schema.ts index aaabed3142d..9a9dafc0159 100644 --- a/src/lib/config/schema.ts +++ b/src/lib/config/schema.ts @@ -131,6 +131,27 @@ function managedProviderProblems( provider: Extract, providerIndex: number, ): string[] { + if (provider.serving.backend === "ollama") { + const serving = provider.serving; + const matches = + serving.daemon.hostPort !== serving.proxy.hostPort && + config.spec.sandboxes.every((sandbox) => + sandbox.agents.every((agent) => + agent.inference.routes.every( + (route) => + route.providerRef !== provider.name || + (sandbox.runtime.provider === "docker" && + agent.type === "openclaw" && + route.overrides.model === serving.model.servedName), + ), + ), + ); + return matches + ? [] + : [ + `/spec/inferenceProviders/${providerIndex}/serving requires distinct Ollama ports and a matching Docker OpenClaw route`, + ]; + } const matches = config.spec.sandboxes.every((sandbox) => sandbox.agents.every((agent) => agent.inference.routes.every( diff --git a/src/lib/domain/config/export-document.ts b/src/lib/domain/config/export-document.ts index 7b20ae83599..8d0bd68ee5b 100644 --- a/src/lib/domain/config/export-document.ts +++ b/src/lib/domain/config/export-document.ts @@ -18,15 +18,29 @@ function providerLocalName(provider: string): string { return `hosted-${normalized || "provider"}`.slice(0, 63).replace(/[^a-z0-9]+$/gu, ""); } +function exportedProviderName(inference: VerifiedExportSource["inference"]): string { + if ("serving" in inference) + return inference.serving.backend === "vllm" ? "managed-vllm" : "local-ollama"; + return providerLocalName(inference.provider); +} + function inferenceProvider( source: VerifiedExportSource, name: string, ): NemoClawInferenceProviderConfig { if ("serving" in source.inference) { + if (source.inference.serving.backend === "ollama") { + return { + name, + provider: "ollama-local", + api: "openai-completions", + serving: source.inference.serving, + }; + } return { name, - provider: source.inference.provider, - api: source.inference.api, + provider: "vllm-local", + api: "openai-completions", serving: source.inference.serving, }; } @@ -51,8 +65,7 @@ export function buildExportConfig( source: VerifiedExportSource, identity: ExportConfigBuildIdentity, ): NemoClawConfig { - const providerName = - "serving" in source.inference ? "managed-vllm" : providerLocalName(source.inference.provider); + const providerName = exportedProviderName(source.inference); const candidate = { apiVersion: "nemoclaw.nvidia.com/v1", kind: "NemoClawConfig", @@ -90,7 +103,8 @@ export function buildExportConfig( providerRef: providerName, overrides: { model: source.inference.model, - ...("serving" in source.inference + ...(source.inference.provider === "vllm-local" && + "serving" in source.inference ? { contextWindow: EXPORTED_VLLM_CONTEXT_WINDOW } : {}), ...("overrides" in source.inference ? source.inference.overrides : {}), diff --git a/src/lib/domain/config/export-evidence.ts b/src/lib/domain/config/export-evidence.ts index 734586b1e42..6b9099a27d5 100644 --- a/src/lib/domain/config/export-evidence.ts +++ b/src/lib/domain/config/export-evidence.ts @@ -5,6 +5,7 @@ import type * as TypeBoxModule from "typebox" with { "resolution-mode": "import" import { BoundedTextSchema, NemoClawManagedVllmServingSchema, + NemoClawOllamaServingSchema, CredentialEnvironmentReferenceNameSchema, ImmutableImageReferenceSchema, InferenceEndpointSchema, @@ -25,6 +26,7 @@ import { } from "../../config/model"; import type { SandboxConfiguration } from "../sandbox/configuration"; import type { SandboxEntry } from "../../state/registry/types"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; const { Type } = require("typebox") as typeof TypeBoxModule; @@ -157,6 +159,7 @@ export interface ObservedExportInference { readonly endpointEvidence: ObservedExportEndpointEvidence | null; readonly credentialEnv: string | null; readonly managedServing?: ObservedManagedVllmRuntime; + readonly ollamaServing?: ObservedOllamaProxy; } export interface ObservedExportPolicy { @@ -184,6 +187,7 @@ export type ExportSnapshotReadStage = | "provider-metadata" | "web-search-provider" | "managed-serving" + | "ollama-serving" | "effective-policy"; /** One complete, untrusted read from all export evidence owners. */ @@ -252,6 +256,16 @@ const HostedExportInferenceSchema = Type.Object({ const ExportInferenceSchema = Type.Union([ HostedExportInferenceSchema, + Type.Object( + { + provider: Type.Literal("ollama-local"), + model: Type.Refine(BoundedTextSchema, isValidNemoClawBoundedText), + api: Type.Literal("openai-completions"), + serving: NemoClawOllamaServingSchema, + overrides: Type.Optional(NemoClawInferenceTuningSchema), + }, + { additionalProperties: false }, + ), Type.Object( { provider: Type.Literal("vllm-local"), diff --git a/src/lib/domain/config/verify-export-source.ts b/src/lib/domain/config/verify-export-source.ts index bcfdd67514a..a2aae8018c9 100644 --- a/src/lib/domain/config/verify-export-source.ts +++ b/src/lib/domain/config/verify-export-source.ts @@ -32,6 +32,7 @@ import { import { fingerprintOpenShellSandboxId } from "../sandbox/openshell-identity"; import { ExportSourceValuesSchema } from "./export-evidence"; import { validateManagedServing } from "./verify-managed-serving"; +import { validateOllamaServing } from "./verify-ollama-serving"; import type { CanonicalExportPolicy, ExportFinding, @@ -755,7 +756,11 @@ function validateGateway(snapshot: QualifiedExportSnapshot): ExportFinding[] { function validateInferenceSelection(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { registry: entry, inference } = snapshot; const findings: ExportFinding[] = []; - if (inference.topology !== "hosted" && inference.topology !== "managed") + if ( + inference.topology !== "hosted" && + inference.topology !== "managed" && + !(inference.topology === "local" && inference.provider === "ollama-local") + ) findings.push( finding( "spec.inferenceProviders", @@ -806,6 +811,8 @@ function validateInferenceSelection(snapshot: QualifiedExportSnapshot): ExportFi function validateInferenceRepresentation(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { inference } = snapshot; + if (inference.provider === "ollama-local" || inference.ollamaServing) + return validateOllamaServing(snapshot); if (inference.topology === "managed") return validateManagedServing(snapshot); const findings: ExportFinding[] = []; if ( @@ -856,7 +863,11 @@ function validateEndpointEvidence(snapshot: QualifiedExportSnapshot): ExportFind } const findings: ExportFinding[] = []; - if (inference.topology !== "managed" && !isValidNemoClawInferenceEndpoint(evidence.endpoint)) + if ( + inference.topology !== "managed" && + inference.provider !== "ollama-local" && + !isValidNemoClawInferenceEndpoint(evidence.endpoint) + ) findings.push( finding( "source.inference.endpoint", @@ -893,7 +904,7 @@ function validateEndpointEvidence(snapshot: QualifiedExportSnapshot): ExportFind function validateCredentialReference(snapshot: QualifiedExportSnapshot): ExportFinding[] { const { inference } = snapshot; - if (inference.topology === "managed") return []; + if (inference.topology === "managed" || inference.provider === "ollama-local") return []; const findings: ExportFinding[] = []; if ( inference.credentialEnv !== null && @@ -993,6 +1004,9 @@ function projectVerifiedInference( model: selected.model, api: selected.preferredInferenceApi, }; + if (snapshot.inference.provider === "ollama-local") { + return { ...common, serving: snapshot.inference.ollamaServing?.serving }; + } return snapshot.inference.topology === "managed" ? { ...common, serving: snapshot.inference.managedServing?.serving } : { diff --git a/src/lib/domain/config/verify-managed-serving.ts b/src/lib/domain/config/verify-managed-serving.ts index 9752156b9f0..e454c4e8ea5 100644 --- a/src/lib/domain/config/verify-managed-serving.ts +++ b/src/lib/domain/config/verify-managed-serving.ts @@ -64,7 +64,7 @@ function validProfileVersion(version: string): boolean { return /^(0|[1-9][0-9]{0,19})$/.test(version) && BigInt(version) <= 18446744073709551615n; } -function hasManagedOpenAiProfile(evidence: ObservedExportEndpointEvidence | null): boolean { +export function hasManagedOpenAiProfile(evidence: ObservedExportEndpointEvidence | null): boolean { if (!evidence) return false; const { provider } = evidence; const profile = provider.managedProfile; diff --git a/src/lib/domain/config/verify-ollama-serving.ts b/src/lib/domain/config/verify-ollama-serving.ts new file mode 100644 index 00000000000..4b2ce4195a9 --- /dev/null +++ b/src/lib/domain/config/verify-ollama-serving.ts @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import type * as TypeBoxValueModule from "typebox/value" with { "resolution-mode": "import" }; +import { NemoClawOllamaServingSchema } from "../../config/model"; +import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "../../inference/ollama/contract"; +import type { ObservedOllamaProxy } from "../../inference/ollama/proxy-observation"; +import type { ExportFinding, QualifiedExportSnapshot } from "./export-evidence"; +import { hasManagedOpenAiProfile } from "./verify-managed-serving"; + +const { Check } = require("typebox/value") as typeof TypeBoxValueModule; + +function validObservation( + observed: ObservedOllamaProxy | undefined, +): observed is ObservedOllamaProxy { + if (!observed) return false; + const { serving, pid } = observed; + return ( + Check(NemoClawOllamaServingSchema, serving) && + serving.daemon.hostPort !== serving.proxy.hostPort && + Number.isInteger(pid) && + pid > 0 && + pid <= 2_147_483_647 && + observed.listenerAddress === "0.0.0.0" + ); +} + +export function validateOllamaServing(snapshot: QualifiedExportSnapshot): ExportFinding[] { + const { registry: entry, inference } = snapshot; + const observed = inference.ollamaServing; + if ( + validObservation(observed) && + hasManagedOpenAiProfile(inference.endpointEvidence) && + entry.workload?.kind === "managed-image" && + /^linux\/(?:amd64|arm64)$/u.test(entry.workload.platform ?? "") && + isDeepStrictEqual( + [ + entry.agent, + entry.openshellDriver, + inference.topology, + inference.provider, + inference.api, + inference.credentialEnv, + inference.model, + inference.endpoint, + snapshot.sandbox.providerNames.filter((name) => name === inference.provider), + ], + [ + "openclaw", + "docker", + "local", + "ollama-local", + "openai-completions", + OLLAMA_LOCAL_CREDENTIAL_ENV, + observed.serving.model.servedName, + `http://host.openshell.internal:${observed.serving.proxy.hostPort}/v1`, + ["ollama-local"], + ], + ) + ) + return []; + return [ + { + field: "spec.inferenceProviders[].serving", + category: "drifted", + diagnostic: + "The attached Ollama daemon, managed proxy, model, or sandbox route could not be verified.", + }, + ]; +} diff --git a/src/lib/inference/README.md b/src/lib/inference/README.md index e0c8e52a344..e0577ae8469 100644 --- a/src/lib/inference/README.md +++ b/src/lib/inference/README.md @@ -24,3 +24,36 @@ onboard-probes.ts onboarding-time inference validation probes ``` Longer term, pure inference decisions should move under `src/lib/domain/inference/**`, and HTTP/process boundaries should move under `src/lib/adapters/**`. + +## Ollama configuration export + +The native Linux Docker export slice composes the current proxy owner and nonsecret observer in +`adapters/config/live-export-source.ts`. Each snapshot asks `ollama/proxy.ts` for a fresh probe of +the retained descriptor, port, PID, and three fixed endpoint reads. `ollama/proxy-observation.ts` +observes the host platform and validates retained intent before requesting an authenticated read. +Only then does the proxy owner read its token, capturing it for that snapshot's proxy requests. +The token stays inside this owner and enters curl through stdin. Fixed loopback reads bypass ambient +HTTP proxies and have bounded responses. These reads must not acquire a lifecycle lock, migrate +state, restart a process, or call the token getters that can adopt legacy state. + +```mermaid +flowchart LR + retained[Retained proxy intent] --> owner[Existing proxy owner] + active[Authenticated active proxy configuration] --> owner + models[Daemon and proxy model inventories] --> owner + host[Native host observation] --> observation[Nonsecret observation validation] + owner --> observation + observation --> export[Shared export snapshot and verifier] + export --> document[External daemon and managed proxy configuration] +``` + +The proxy's authenticated `GET /_nemoclaw/proxy-config` reports its current PID, actual listener, +and backend origin. It never forwards this request or returns backend userinfo, paths, queries, or +credentials. `ollama/proxy-observation.ts` compares that response with retained intent and the selected +model digest from both native model inventories. Older running proxies without this response cannot +be exported; upgrading or restarting them remains an operator lifecycle action. + +The public `serving.backend: ollama` branch records the daemon as external and the auth proxy as +NemoClaw-managed. It does not claim ownership of the daemon process, software installation, or model +cache. The initial contract covers `qwen3.5:9b` on native Linux Docker with managed OpenClaw and no +direct sandbox GPU. Hosted inference and managed vLLM retain their existing schema variants. diff --git a/src/lib/inference/ollama/proxy-observation.test.ts b/src/lib/inference/ollama/proxy-observation.test.ts new file mode 100644 index 00000000000..25e3611b2f5 --- /dev/null +++ b/src/lib/inference/ollama/proxy-observation.test.ts @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import os from "node:os"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { isWsl } from "../../platform"; +import { observeOllamaProxy, type OllamaProxyObservationInput } from "./proxy-observation"; + +vi.mock("../../platform", () => ({ isWsl: vi.fn(() => false) })); + +const digest = "a".repeat(64); +const active = { + schemaVersion: 1, + pid: 1234, + listener: { address: "0.0.0.0", port: 11440 }, + backendOrigin: "http://127.0.0.1:11439", +}; +const models = JSON.stringify({ models: [{ name: "qwen3.5:9b", digest, size: 6000000000 }] }); +function observation(): OllamaProxyObservationInput { + return { + model: "qwen3.5:9b", + backend: { kind: "ollama", url: active.backendOrigin }, + proxyPort: "11440", + pid: "1234", + processMatches: vi.fn(() => true), + readActiveConfig: vi.fn(() => JSON.stringify(active)), + readProxyModels: vi.fn(() => models), + readDaemonModels: vi.fn(() => models), + }; +} + +describe("read-only Ollama export observation", () => { + beforeEach(() => { + vi.spyOn(os, "platform").mockReturnValue("linux"); + vi.mocked(isWsl).mockReturnValue(false); + }); + + it.each([ + { platform: "darwin", wsl: false }, + { platform: "linux", wsl: true }, + ] as const)("refuses unsupported host facts before protected reads %# (#11435)", (host) => { + vi.spyOn(os, "platform").mockReturnValue(host.platform); + vi.mocked(isWsl).mockReturnValue(host.wsl); + const input = observation(); + expect(() => observeOllamaProxy(input)).toThrow("requires a native Linux host"); + expect(input.readActiveConfig).not.toHaveBeenCalled(); + expect(input.readProxyModels).not.toHaveBeenCalled(); + expect(input.readDaemonModels).not.toHaveBeenCalled(); + }); + + it("exports the active nondefault mapping and attached model identity (#11435)", () => { + const input = observation(); + expect(observeOllamaProxy(input)).toEqual({ + pid: 1234, + listenerAddress: "0.0.0.0", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { servedName: "qwen3.5:9b", digest: `sha256:${digest}` }, + }, + }); + expect(input.readActiveConfig).toHaveBeenCalledWith(11440); + expect(input.readProxyModels).toHaveBeenCalledWith(11440); + expect(input.readDaemonModels).toHaveBeenCalledWith(11439); + }); + + it.each([ + { backend: { kind: "compatible", url: active.backendOrigin } }, + { backend: { kind: "ollama", url: "http://127.0.0.1:11439/private?secret" } }, + { backend: { kind: "ollama", url: "http://remote.example:11439" } }, + { proxyPort: "11439" }, + { proxyPort: null }, + { proxyPort: "011440" }, + { pid: null }, + { pid: "-1" }, + { pid: "1234junk" }, + { model: "different:tag" }, + { processMatches: () => false }, + ])( + "refuses unsupported or incomplete retained intent before network reads %# (#11435)", + (change) => { + const input = { ...observation(), ...change }; + expect(() => observeOllamaProxy(input)).toThrow("could not be verified"); + expect(input.readActiveConfig).not.toHaveBeenCalled(); + expect(input.readProxyModels).not.toHaveBeenCalled(); + expect(input.readDaemonModels).not.toHaveBeenCalled(); + }, + ); + + it.each([ + "", + "legacy proxy response", + JSON.stringify({ error: "not found" }), + "a".repeat(65537), + JSON.stringify({ ...active, pid: 1235 }), + JSON.stringify({ ...active, listener: { address: "127.0.0.1", port: 11440 } }), + JSON.stringify({ ...active, listener: { address: "0.0.0.0", port: 11435 } }), + JSON.stringify({ ...active, backendOrigin: "http://127.0.0.1:11434" }), + JSON.stringify({ ...active, backendOrigin: "http://user:secret@127.0.0.1:11439/private" }), + ])("refuses a missing or mismatched active response without guessing %# (#11435)", (body) => { + const input = { ...observation(), readActiveConfig: () => body }; + expect(() => observeOllamaProxy(input)).toThrow("could not be verified"); + expect(input.readProxyModels).not.toHaveBeenCalled(); + }); + + it.each([ + { models: [] }, + { models: [{ name: "qwen3.5:9b", digest: "invalid" }] }, + { models: [{ name: "other:tag", digest }] }, + { + models: [ + { name: "qwen3.5:9b", digest }, + { name: "qwen3.5:9b", digest }, + ], + }, + { models: [{ name: "qwen3.5:9b", digest: "b".repeat(64) }] }, + ])("refuses a missing, ambiguous, or changed daemon model %# (#11435)", (value) => { + expect(() => + observeOllamaProxy({ ...observation(), readDaemonModels: () => JSON.stringify(value) }), + ).toThrow("could not be verified"); + }); +}); diff --git a/src/lib/inference/ollama/proxy-observation.ts b/src/lib/inference/ollama/proxy-observation.ts new file mode 100644 index 00000000000..b682a551708 --- /dev/null +++ b/src/lib/inference/ollama/proxy-observation.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import os from "node:os"; +import type * as TypeBoxModule from "typebox" with { "resolution-mode": "import" }; +import type * as TypeBoxValueModule from "typebox/value" with { "resolution-mode": "import" }; +import { isWsl } from "../../platform"; +import { + BoundedTextSchema, + EXPORTED_OLLAMA_MODEL, + TcpPortSchema, + type NemoClawOllamaServing, +} from "../../config/model"; + +const { Type } = require("typebox") as typeof TypeBoxModule; +const { Check } = require("typebox/value") as typeof TypeBoxValueModule; +const ProcessId = Type.Integer({ minimum: 1, maximum: 2_147_483_647 }); +const ActiveConfig = Type.Object( + { + schemaVersion: Type.Literal(1), + pid: ProcessId, + listener: Type.Object( + { address: Type.Literal("0.0.0.0"), port: TcpPortSchema }, + { additionalProperties: false }, + ), + backendOrigin: Type.String({ maxLength: 64 }), + }, + { additionalProperties: false }, +); +const Models = Type.Object({ + models: Type.Array( + Type.Object({ + name: BoundedTextSchema, + digest: Type.String({ pattern: "^(?:sha256:)?[a-f0-9]{64}$(?![\\s\\S])" }), + }), + { maxItems: 512 }, + ), +}); + +export interface ObservedOllamaProxy { + readonly serving: NemoClawOllamaServing; + readonly pid: number; + readonly listenerAddress: "0.0.0.0"; +} + +export interface OllamaProxyObservationInput { + readonly model: string; + readonly backend: { readonly kind: string; readonly url: string | null }; + readonly proxyPort: string | null; + readonly pid: string | null; + readonly processMatches: (pid: number) => boolean; + readonly readActiveConfig: (port: number) => string; + readonly readProxyModels: (port: number) => string; + readonly readDaemonModels: (port: number) => string; +} + +function fail(): never { + throw new Error( + "The live Ollama daemon, proxy mapping, or model could not be verified for export.", + ); +} + +function parse( + schema: Schema, + body: string, +): TypeBoxModule.Type.Static { + if (Buffer.byteLength(body) > 64 * 1024) fail(); + let value: unknown; + try { + value = JSON.parse(body); + } catch { + fail(); + } + if (!Check(schema, value)) fail(); + return value; +} + +function selectedDigest(body: string, model: string): string { + const matches = parse(Models, body).models.filter((entry) => entry.name === model); + if (matches.length !== 1) fail(); + const digest = matches[0].digest; + return digest.startsWith("sha256:") ? digest : `sha256:${digest}`; +} + +/** Validate retained intent against current proxy and model reads; receives no credentials. */ +export function observeOllamaProxy(input: OllamaProxyObservationInput): ObservedOllamaProxy { + if (os.platform() !== "linux" || isWsl()) { + throw new Error("Ollama export requires a native Linux host."); + } + const proxyPort = Number(input.proxyPort); + const pid = Number(input.pid); + const backend = input.backend.url?.match(/^http:\/\/127\.0\.0\.1:([1-9][0-9]{0,4})$/u); + const daemonPort = Number(backend?.[1]); + if ( + input.backend.kind !== "ollama" || + input.model !== EXPORTED_OLLAMA_MODEL || + !Check(TcpPortSchema, daemonPort) || + !Check(TcpPortSchema, proxyPort) || + input.proxyPort !== String(proxyPort) || + !Check(ProcessId, pid) || + input.pid !== String(pid) || + daemonPort === proxyPort || + !input.processMatches(pid) + ) + fail(); + + const active = parse(ActiveConfig, input.readActiveConfig(proxyPort)); + if ( + !isDeepStrictEqual( + [active.pid, active.listener.port, active.backendOrigin], + [pid, proxyPort, input.backend.url], + ) + ) + fail(); + const digest = selectedDigest(input.readProxyModels(proxyPort), input.model); + if (selectedDigest(input.readDaemonModels(daemonPort), input.model) !== digest) fail(); + return { + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: daemonPort }, + proxy: { management: "nemoclaw", hostPort: proxyPort }, + model: { servedName: EXPORTED_OLLAMA_MODEL, digest }, + }, + pid, + listenerAddress: active.listener.address, + }; +} diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 0017c692b20..ac06112f2e4 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -355,6 +355,38 @@ function runCurlCaptureWithAuthConfig( return result.status === 0 ? String(result.stdout || "") : ""; } +/** Fixed export reads capture one credential only after the observer admits retained intent. */ +export function createOllamaExportProbe() { + let token: string | null = null; + const args = [ + "-q", + "--noproxy", + "*", + "-fsS", + "--connect-timeout", + "3", + "--max-time", + "5", + "--max-filesize", + "65536", + ]; + const readProxy = (port: number, route: string) => { + token ??= readProxyStateFile(PROXY_TOKEN_PATH); + if (!token) throw new Error("The existing Ollama proxy credential is unavailable."); + return runCurlCaptureWithAuthConfig(args, `http://127.0.0.1:${port}${route}`, token); + }; + return { + backend: readProxyBackendIdentity(), + proxyPort: readProxyStateFile(PROXY_PORT_PATH), + pid: readProxyStateFile(PROXY_PID_PATH), + processMatches: isOllamaProxyProcess, + readActiveConfig: (port: number) => readProxy(port, "/_nemoclaw/proxy-config"), + readProxyModels: (port: number) => readProxy(port, "/api/tags"), + readDaemonModels: (port: number) => + runCurlCaptureWithAuthConfig(args, `http://127.0.0.1:${port}/api/tags`), + }; +} + // ── PID persistence ────────────────────────────────────────────── function persistProxyPid(pid: number | null | undefined): void { diff --git a/test/e2e/README.md b/test/e2e/README.md index 6c638c00e67..785066b54e2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -459,6 +459,15 @@ The standard layout writes product evidence and `evidence-manifest.json` under ` When `shard` is not `default`, the standard layout adds the shard directory. The security-posture matrix uses the reviewed flat-shard layout to preserve its existing artifact names. The `gpu-double-onboard`, `gpu-e2e`, and `llama-cpp-generic-gpu` targets keep the standard layout and select `linux-amd64-gpu-rtxpro6000-latest-1` through the catalogue. +The `gpu-e2e` target also qualifies configuration export for an attached native Linux Ollama daemon. +A separate OpenClaw scenario disables direct sandbox GPU, starts a fixture-owned daemon on port +11439, and uses normal onboarding to create the managed proxy on port 11440. It exports twice through +the candidate CLI and real SDK, validates both documents, compares their specs and model digest, +checks credential omission, and requires a stopped daemon to prevent publication. Private YAML is +removed through the cleanup registry; retained evidence contains only the selected model, ports, +managed image, and result booleans. The existing CUDA, authentication, and inference lifecycle +scenarios remain separate. Daemon readiness polls only connection refusal, for at most 20 reads, +and records each attempt; onboarding and export mutations are not retried. Retained workflow jobs are exceptions to the catalogue shape. Keep one only for a multi-job handoff, an unrepresented credential boundary, or an execution contract the reusable profile cannot represent. diff --git a/test/e2e/live/gpu-e2e-helpers.ts b/test/e2e/live/gpu-e2e-helpers.ts index 2d6a7f2dd51..e21ac560e76 100644 --- a/test/e2e/live/gpu-e2e-helpers.ts +++ b/test/e2e/live/gpu-e2e-helpers.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; +import { ownChildProcess } from "../../helpers/child-process-lifecycle.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; @@ -10,11 +11,23 @@ import { resultText } from "../fixtures/clients/index.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { stripAnsi } from "./json-envelope.ts"; export { REPO_ROOT }; +/** The export fixture owns only this attached daemon; onboarding owns the auth proxy. */ +export function startAttachedOllama(progress: TestProgress, environment: NodeJS.ProcessEnv) { + const child = spawnObservedChild("ollama", ["serve"], { + activityLabel: "attached Ollama daemon", + progress, + spawn: { cwd: REPO_ROOT, env: environment, stdio: "ignore" }, + }); + return ownChildProcess(child); +} + export const CLI = CLI_ENTRYPOINT; export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-gpu-ollama"; const DEFAULT_GPU_E2E_MODEL = "qwen3.5:9b"; diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index c95578a2371..7efa438b0f6 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -1,11 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { validateNemoClawConfig } from "../../../src/lib/config/schema.ts"; +import { load as loadRegistry } from "../../../src/lib/state/registry/persistence.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { pollUntil } from "../fixtures/polling.ts"; import { assertAgentExecutionSucceeded, assertGpuInstallProofs, @@ -25,6 +32,7 @@ import { readTokenFileChecked, restartProxy, SANDBOX_NAME, + startAttachedOllama, } from "./gpu-e2e-helpers.ts"; import { assertHermesFollowUpReplies } from "./hermes-cli-adapter-live.ts"; @@ -413,3 +421,180 @@ test( }); }, ); + +test( + "OpenClaw exports an attached Ollama daemon and refuses a stopped backend (#11435)", + { + timeout: TIMEOUT_MS, + meta: { + e2ePhases: [ + "prepare an attached Ollama daemon", + "onboard OpenClaw without sandbox GPU", + "export and compare the active Ollama configuration", + "refuse export after the attached daemon stops", + ], + }, + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { + await artifacts.target.declare({ + id: "gpu-e2e", + boundary: + "native Linux Docker + attached Ollama daemon + NemoClaw proxy + managed OpenClaw + SDK configuration export", + credentialBoundary: + "The existing proxy owner authenticates observation; exported YAML contains no token or internal endpoint.", + }); + const exportEnv = env({ + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_SANDBOX_GPU: "0", + NEMOCLAW_SANDBOX_GPU_DEVICE: "", + NEMOCLAW_OLLAMA_PORT: "11439", + NEMOCLAW_OLLAMA_PROXY_PORT: "11440", + NEMOCLAW_MODEL: "qwen3.5:9b", + NEMOCLAW_WEB_SEARCH_PROVIDER: "none", + OLLAMA_HOST: "127.0.0.1:11439", + OLLAMA_CONTEXT_LENGTH: "32768", + }); + cleanup.trackDisposable("stop Ollama processes after export qualification", async () => { + const result = await cleanupOllama(host, "export-cleanup-ollama-processes"); + expect(result.exitCode, resultText(result)).toBe(0); + }); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-export-")); + cleanup.trackDisposable("remove private Ollama export documents", () => + fs.rmSync(directory, { recursive: true, force: true }), + ); + await cleanupGpu(host, sandbox); + await runtimeProvider.requireAvailable({ + artifactName: "export-runtime-info", + scenarioLabel: "attached Ollama export", + }); + await ensureOllama(host); + await cleanupOllama(host, "export-stop-default-ollama"); + const daemonOwner = startAttachedOllama(progress, exportEnv); + cleanup.trackDisposable("stop the fixture-owned Ollama daemon", () => daemonOwner.terminate()); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "export-cleanup-gateway", + env: exportEnv, + timeoutMs: 60000, + }); + cleanup.trackDisposable("delete the export sandbox", () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "export-cleanup-openshell", + env: exportEnv, + timeoutMs: 60000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "export-cleanup-sandbox", + env: exportEnv, + timeoutMs: 120000, + }); + // Only connection refusal is transient while this fixture's child starts. Every read is recorded. + await pollUntil({ + artifactPrefix: "export-daemon-ready", + attempts: 20, + delayMs: 500, + probe: (_attempt, artifactName) => + host.command( + "curl", + ["-q", "--noproxy", "*", "-fsS", "--max-time", "2", "http://127.0.0.1:11439/api/tags"], + { artifactName, env: exportEnv, timeoutMs: 5000 }, + ), + accept: (result) => result.exitCode === 0, + terminal: (result) => + result.exitCode !== 0 && result.exitCode !== 7 + ? "The attached daemon readiness read failed." + : undefined, + }); + + progress.phase("onboard OpenClaw without sandbox GPU"); + const onboard = await host.command( + "node", + [CLI, "onboard", "--fresh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "export-onboard-ollama", + cwd: REPO_ROOT, + env: exportEnv, + timeoutMs: execTimeout(55 * 60000), + }, + ); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + progress.phase("export and compare the active Ollama configuration"); + const firstPath = path.join(directory, "first.yaml"); + const exported = await host.command( + "node", + [CLI, "config", "export", SANDBOX_NAME, "--output", firstPath, "--json"], + { artifactName: "export-ollama-first", cwd: REPO_ROOT, env: exportEnv, timeoutMs: 60000 }, + ); + expect(exported.exitCode, resultText(exported)).toBe(0); + const raw = fs.readFileSync(firstPath, "utf8"); + const document = validateNemoClawConfig(YAML.parse(raw)); + const token = readTokenFileChecked(ollamaProxyTokenFile()).token; + artifacts.addRedactionValues([token]); + expect(raw.includes(token), "Export must omit the proxy credential").toBe(false); + expect(raw).not.toMatch(/NEMOCLAW_OLLAMA_PROXY_TOKEN|host\.openshell\.internal/u); + const tags = await host.command( + "curl", + ["-q", "--noproxy", "*", "-fsS", "--max-time", "5", "http://127.0.0.1:11439/api/tags"], + { artifactName: "export-attached-model-identity", env: exportEnv, timeoutMs: 10000 }, + ); + const model = ( + JSON.parse(tags.stdout) as { models: Array<{ name: string; digest: string }> } + ).models.find(({ name }) => name === "qwen3.5:9b"); + expect(document.spec.inferenceProviders).toEqual([ + { + name: "local-ollama", + provider: "ollama-local", + api: "openai-completions", + serving: { + backend: "ollama", + daemon: { management: "external", hostPort: 11439 }, + proxy: { management: "nemoclaw", hostPort: 11440 }, + model: { + servedName: "qwen3.5:9b", + digest: `sha256:${model?.digest.replace(/^sha256:/u, "")}`, + }, + }, + }, + ]); + const entry = loadRegistry().sandboxes[SANDBOX_NAME]; + expect(document.spec.sandboxes[0].runtime.image.ref).toBe( + entry.workload?.kind === "managed-image" ? entry.workload.reference : null, + ); + const repeatPath = path.join(directory, "repeat.yaml"); + const repeated = await host.command( + "node", + [CLI, "config", "export", SANDBOX_NAME, "--output", repeatPath, "--json"], + { artifactName: "export-ollama-repeat", cwd: REPO_ROOT, env: exportEnv, timeoutMs: 60000 }, + ); + expect(repeated.exitCode, resultText(repeated)).toBe(0); + expect(validateNemoClawConfig(YAML.parse(fs.readFileSync(repeatPath, "utf8"))).spec).toEqual( + document.spec, + ); + + progress.phase("refuse export after the attached daemon stops"); + await daemonOwner.terminate(); + const rejectedPath = path.join(directory, "must-not-exist.yaml"); + const rejected = await host.command( + "node", + [CLI, "config", "export", SANDBOX_NAME, "--output", rejectedPath, "--json"], + { + artifactName: "export-ollama-stopped-daemon", + cwd: REPO_ROOT, + env: exportEnv, + timeoutMs: 60000, + }, + ); + expect(rejected.exitCode, resultText(rejected)).not.toBe(0); + expect(fs.existsSync(rejectedPath), "A stopped daemon must prevent publication").toBe(false); + await artifacts.writeJson("ollama-config-export-evidence.json", { + sandboxName: SANDBOX_NAME, + daemonPort: 11439, + proxyPort: 11440, + model: "qwen3.5:9b", + image: document.spec.sandboxes[0].runtime.image.ref, + repeatedSpecMatches: true, + stoppedDaemonPreventedPublication: true, + }); + }, +); diff --git a/test/inference/ollama/ollama-auth-proxy-handler.test.ts b/test/inference/ollama/ollama-auth-proxy-handler.test.ts index 4cb03fc3fdd..85a8d3ce4c6 100644 --- a/test/inference/ollama/ollama-auth-proxy-handler.test.ts +++ b/test/inference/ollama/ollama-auth-proxy-handler.test.ts @@ -68,6 +68,60 @@ describe("ollama-auth-proxy request handler", () => { expect(backend?.captured).toHaveLength(0); }); + it.each([undefined, "Bearer wrong-token"])( + "protects active configuration with the proxy token (#11435)", + async (auth) => { + const response = await request(proxyPort, { path: "/_nemoclaw/proxy-config", auth }); + expect(response.status).toBe(401); + expect(backend?.captured).toHaveLength(0); + }, + ); + + it("reports the active listener and backend without forwarding or credentials (#11435)", async () => { + const response = await request(proxyPort, { + path: "/_nemoclaw/proxy-config", + auth: `Bearer ${TOKEN}`, + }); + expect(response.status).toBe(200); + expect(JSON.parse(response.body)).toEqual({ + schemaVersion: 1, + pid: proxy?.pid, + listener: { address: "0.0.0.0", port: proxyPort }, + backendOrigin: `http://localhost:${backend?.port}`, + }); + expect(response.body).not.toContain(TOKEN); + expect(backend?.captured).toHaveLength(0); + }); + + it.each(["POST", "PUT", "DELETE"])( + "rejects %s for active configuration without forwarding (#11435)", + async (method) => { + const response = await request(proxyPort, { + path: "/_nemoclaw/proxy-config", + method, + auth: `Bearer ${TOKEN}`, + body: "{}", + }); + expect(response.status).toBe(405); + expect(backend?.captured).toHaveLength(0); + }, + ); + + it("omits backend userinfo, path, and query from active configuration (#11435)", async () => { + await terminate(proxy); + proxy = await startProxy(proxyPort, backend!.port, TOKEN, { + backendUrl: `http://user:secret@localhost:${backend!.port}/private?canary=value`, + }); + const response = await request(proxyPort, { + path: "/_nemoclaw/proxy-config", + auth: `Bearer ${TOKEN}`, + }); + expect(response.status).toBe(200); + expect(JSON.parse(response.body).backendOrigin).toBe(`http://localhost:${backend!.port}`); + expect(response.body).not.toMatch(/user|secret|private|canary|value/u); + expect(backend?.captured).toHaveLength(0); + }); + it("returns 401 for unauthenticated /api/tags — no health-check bypass (#3338)", async () => { const res = await request(proxyPort, { path: "/api/tags" }); expect(res.status).toBe(401); diff --git a/test/inference/ollama/ollama-proxy-export.test.ts b/test/inference/ollama/ollama-proxy-export.test.ts new file mode 100644 index 00000000000..428055122e1 --- /dev/null +++ b/test/inference/ollama/ollama-proxy-export.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { expect } from "vitest"; +import { test } from "../../helpers/owned-test-resources"; +import { + closeServer, + freePort, + startBackend, + startProxy, + terminate, +} from "../../ollama-auth-proxy-handler-helpers"; + +const execFileAsync = promisify(execFile); + +test.skipIf(process.platform !== "linux")( + "observes existing proxy state without migration or credential exposure (#11435)", + async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-proxy-export-")); + const state = path.join(directory, ".nemoclaw"); + const token = "export-credential-canary"; + const digest = "a".repeat(64); + const backend = await startBackend({ ok: true, models: [{ name: "qwen3.5:9b", digest }] }); + let proxy; + try { + const port = await freePort(); + proxy = await startProxy(port, backend.port, token); + fs.mkdirSync(state); + const retained = { + "ollama-proxy-token": token, + "ollama-auth-proxy.pid": String(proxy.pid), + "ollama-proxy-port": String(port), + "ollama-backend": `http://127.0.0.1:${backend.port}`, + "ollama-backend.json": JSON.stringify({ + schemaVersion: 1, + kind: "ollama", + url: `http://127.0.0.1:${backend.port}`, + }), + }; + fs.writeFileSync(path.join(state, "ollama-proxy-token"), retained["ollama-proxy-token"], { + mode: 0o600, + }); + fs.writeFileSync( + path.join(state, "ollama-auth-proxy.pid"), + retained["ollama-auth-proxy.pid"], + { + mode: 0o600, + }, + ); + fs.writeFileSync(path.join(state, "ollama-proxy-port"), retained["ollama-proxy-port"], { + mode: 0o600, + }); + fs.writeFileSync(path.join(state, "ollama-backend"), retained["ollama-backend"], { + mode: 0o600, + }); + fs.writeFileSync(path.join(state, "ollama-backend.json"), retained["ollama-backend.json"], { + mode: 0o600, + }); + const script = ` +const fs = require("node:fs"); +const {createOllamaExportProbe} = require(${JSON.stringify(path.resolve(import.meta.dirname, "../../../src/lib/inference/ollama/proxy.ts"))}); +const {observeOllamaProxy} = require(${JSON.stringify(path.resolve(import.meta.dirname, "../../../src/lib/inference/ollama/proxy-observation.ts"))}); +const originalOpen = fs.openSync; +let credentialReads = 0; +fs.openSync = (...args) => { + credentialReads += Number(args[0] === ${JSON.stringify(path.join(state, "ollama-proxy-token"))}); + return originalOpen(...args); +}; +const rejection = (input) => { + try { observeOllamaProxy(input); return "unexpected success"; } + catch (error) { return error.message; } +}; +process.env.WSL_DISTRO_NAME = "unsupported-export-host"; +const unsupportedHost = rejection({model: "qwen3.5:9b", ...createOllamaExportProbe()}); +delete process.env.WSL_DISTRO_NAME; +const invalidMetadata = rejection({model: "qwen3.5:9b", ...createOllamaExportProbe(), proxyPort: "invalid"}); +const credentialReadsBeforeValid = credentialReads; +const observed = observeOllamaProxy({model: "qwen3.5:9b", ...createOllamaExportProbe()}); +process.stdout.write(JSON.stringify({observed, unsupportedHost, invalidMetadata, credentialReadsBeforeValid, credentialReads})); +`; + const { stdout, stderr } = await execFileAsync(process.execPath, ["-e", script], { + env: { + ...process.env, + HOME: directory, + NEMOCLAW_OLLAMA_PORT: "11434", + NEMOCLAW_OLLAMA_PROXY_PORT: "11435", + WSL_DISTRO_NAME: "", + WSL_INTEROP: "", + HTTP_PROXY: "http://127.0.0.1:1", + http_proxy: "http://127.0.0.1:1", + ALL_PROXY: "http://127.0.0.1:1", + all_proxy: "http://127.0.0.1:1", + NO_PROXY: "", + no_proxy: "", + }, + timeout: 30000, + }); + expect(JSON.parse(stdout)).toMatchObject({ + observed: { + pid: proxy.pid, + serving: { + daemon: { management: "external", hostPort: backend.port }, + proxy: { management: "nemoclaw", hostPort: port }, + model: { digest: `sha256:${digest}` }, + }, + }, + unsupportedHost: "Ollama export requires a native Linux host.", + invalidMetadata: + "The live Ollama daemon, proxy mapping, or model could not be verified for export.", + credentialReadsBeforeValid: 0, + credentialReads: 1, + }); + expect(stdout + stderr).not.toContain(token); + expect( + Object.fromEntries( + fs + .readdirSync(state) + .map((name) => [name, fs.readFileSync(path.join(state, name), "utf8")]), + ), + ).toEqual(retained); + expect(backend.captured).toHaveLength(2); + expect(backend.captured.every(({ headers }) => headers.authorization === undefined)).toBe( + true, + ); + } finally { + await terminate(proxy); + await closeServer(backend.server); + fs.rmSync(directory, { recursive: true, force: true }); + } + }, +); diff --git a/test/ollama-auth-proxy-handler-helpers.ts b/test/ollama-auth-proxy-handler-helpers.ts index a8ee0ffbcbf..350835c6aba 100644 --- a/test/ollama-auth-proxy-handler-helpers.ts +++ b/test/ollama-auth-proxy-handler-helpers.ts @@ -35,7 +35,7 @@ export type StartProxyOptions = { }; /** Start a loopback stub backend that records the request it received. */ -export function startBackend(): Promise<{ +export function startBackend(response = { ok: true, models: [] as unknown[] }): Promise<{ server: http.Server; port: number; captured: BackendCapture[]; @@ -51,7 +51,7 @@ export function startBackend(): Promise<{ req.resume(); req.on("end", () => { res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ ok: true, models: [] })); + res.end(JSON.stringify(response)); }); }); return new Promise((resolve) => { diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 98ceac67a7e..b76a896ead3 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -392,6 +392,7 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map Date: Thu, 10 Sep 2026 13:37:47 -0700 Subject: [PATCH 2/6] test(config): separate live export setup fixtures --- .../config/live-export-source-test-fixture.ts | 128 +++++++++++++++++ .../config/live-export-source.test.ts | 133 ++---------------- 2 files changed, 139 insertions(+), 122 deletions(-) create mode 100644 src/lib/adapters/config/live-export-source-test-fixture.ts diff --git a/src/lib/adapters/config/live-export-source-test-fixture.ts b/src/lib/adapters/config/live-export-source-test-fixture.ts new file mode 100644 index 00000000000..63d23c194ec --- /dev/null +++ b/src/lib/adapters/config/live-export-source-test-fixture.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildManagedStartupProfile } from "../../onboard/managed-startup/profile-builder"; +import type { ManagedStartupProfileBuilderInput } from "../../onboard/managed-startup/profile-builder"; +import type { SandboxEntry } from "../../state/registry/types"; +import { fingerprintOpenShellSandboxId } from "../openshell/sandbox-identity"; + +const sandboxId = "123e4567-e89b-42d3-a456-426614174000"; +const identityFingerprint = fingerprintOpenShellSandboxId(sandboxId) as string; +export const endpoint = "https://integrate.api.nvidia.com/v1"; +export const readFailureCanary = "credential-canary-value"; +export const imageRef = "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:" + "a".repeat(64); +export const startupInput = { + agent: "openclaw", + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model: "model-a", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: "inference/model-a", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, +} satisfies ManagedStartupProfileBuilderInput; +export const startup = buildManagedStartupProfile(startupInput); + +export const entry = { + name: "alpha", + createdAt: "not-export-evidence", + agent: "openclaw", + openshellDriver: "docker", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: identityFingerprint, + provider: "nvidia-prod", + model: "model-a", + preferredInferenceApi: "openai-completions", + endpointUrl: endpoint, + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + imageTag: imageRef, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: imageRef, + platform: "linux/amd64", + release: "v1.0.0", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-1-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: startup.encodedProfile, + startupProfileSha256: startup.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, +} satisfies SandboxEntry; + +export function inventory(resourceVersion = 7, policyVersion = 3) { + return { + sandbox: { + metadata: { + id: sandboxId, + name: "alpha", + workspace: "default", + resourceVersion: BigInt(resourceVersion), + }, + status: { phase: 2, currentPolicyVersion: policyVersion }, + spec: { template: { image: imageRef }, providers: [] }, + }, + }; +} +export function provider() { + return { + provider: { + metadata: { + id: "provider-id", + name: "nvidia-prod", + workspace: "default", + resourceVersion: 8n, + }, + type: "openai", + credentials: { NVIDIA_INFERENCE_API_KEY: readFailureCanary }, + config: { OPENAI_BASE_URL: endpoint }, + }, + }; +} +export function configuration(revision = 3) { + return { + policy: { + version: 1, + process: { run_as_user: "sandbox", run_as_group: "sandbox" }, + filesystem_policy: { include_workdir: false, read_only: ["/usr"], read_write: ["/sandbox"] }, + network_policies: { + api: { + name: "api", + endpoints: [{ host: "api.example.com", port: 443 }], + binaries: [{ path: "/usr/bin/curl" }], + }, + }, + }, + workspace: "default", + version: revision, + policyHash: "a".repeat(64), + configRevision: 11n, + providerEnvRevision: 12n, + policySource: 1, + globalPolicyVersion: 0, + }; +} diff --git a/src/lib/adapters/config/live-export-source.test.ts b/src/lib/adapters/config/live-export-source.test.ts index 052415ad10f..febf3fcaa4c 100644 --- a/src/lib/adapters/config/live-export-source.test.ts +++ b/src/lib/adapters/config/live-export-source.test.ts @@ -66,76 +66,18 @@ import type { SandboxEntry } from "../../state/registry/types"; import { connectManagedOpenShellSdk } from "../openshell/sdk"; import { observeStableExportSource } from "../../actions/config/observe-export-source"; import { captureSanitizedResolvedOpenshell } from "../openshell/sanitized-capture"; -import { fingerprintOpenShellSandboxId } from "../openshell/sandbox-identity"; import { createLiveExportSnapshotReader } from "./live-export-source"; - -const sandboxId = "123e4567-e89b-42d3-a456-426614174000"; -const identityFingerprint = fingerprintOpenShellSandboxId(sandboxId)!; -const endpoint = "https://integrate.api.nvidia.com/v1"; -const readFailureCanary = "credential-canary-value"; -const imageRef = "ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:" + "a".repeat(64); -const startupInput = { - agent: "openclaw", - inference: { - routeProvider: "inference", - upstreamProvider: "nvidia-prod", - model: "model-a", - routedBaseUrl: "https://inference.local/v1", - upstreamEndpointUrl: null, - api: "openai-completions", - primaryModelRef: "inference/model-a", - compatibility: {}, - }, - dashboard: { - agent: "openclaw", - mode: "loopback", - url: "http://127.0.0.1:18789", - port: 18_789, - bindAddress: "127.0.0.1", - wslExposure: false, - }, - webSearch: null, - toolDisclosure: "progressive", - hermesToolGateways: [], - messagingPlan: null, - dcodeAutoApprovalMode: null, - observabilityEnabled: null, - environment: {}, - corporateCa: null, -} satisfies ManagedStartupProfileBuilderInput; -const startup = buildManagedStartupProfile(startupInput); - -const entry = { - name: "alpha", - createdAt: "not-export-evidence", - agent: "openclaw", - openshellDriver: "docker", - gatewayName: "nemoclaw", - gatewayPort: 8080, - lifecycleGeneration: "generation-1", - lifecycleLiveIdentityFingerprint: identityFingerprint, - provider: "nvidia-prod", - model: "model-a", - preferredInferenceApi: "openai-completions", - endpointUrl: endpoint, - credentialEnv: "NVIDIA_INFERENCE_API_KEY", - imageTag: imageRef, - workload: { - schemaVersion: 1, - kind: "managed-image", - reference: imageRef, - platform: "linux/amd64", - release: "v1.0.0", - sourceRevision: "b".repeat(40), - sourceCohort: "ghrun-1-1", - capabilityContractVersion: 1, - startupProfileContractVersion: 1, - encodedProfile: startup.encodedProfile, - startupProfileSha256: startup.startupProfileSha256, - credentialProxyReplayRequired: false, - shared: true, - }, -} satisfies SandboxEntry; +import { + endpoint, + readFailureCanary, + imageRef, + startupInput, + startup, + entry, + inventory, + provider, + configuration, +} from "./live-export-source-test-fixture"; function telemetryEntry( telemetry: Readonly> = {}, @@ -168,59 +110,6 @@ const raw = { getSandbox: vi.fn(), getSandboxConfig: vi.fn(), }; -function inventory(resourceVersion = 7, policyVersion = 3) { - return { - sandbox: { - metadata: { - id: sandboxId, - name: "alpha", - workspace: "default", - resourceVersion: BigInt(resourceVersion), - }, - status: { phase: 2, currentPolicyVersion: policyVersion }, - spec: { template: { image: imageRef }, providers: [] }, - }, - }; -} -function provider() { - return { - provider: { - metadata: { - id: "provider-id", - name: "nvidia-prod", - workspace: "default", - resourceVersion: 8n, - }, - type: "openai", - credentials: { NVIDIA_INFERENCE_API_KEY: readFailureCanary }, - config: { OPENAI_BASE_URL: endpoint }, - }, - }; -} -function configuration(revision = 3) { - return { - policy: { - version: 1, - process: { run_as_user: "sandbox", run_as_group: "sandbox" }, - filesystem_policy: { include_workdir: false, read_only: ["/usr"], read_write: ["/sandbox"] }, - network_policies: { - api: { - name: "api", - endpoints: [{ host: "api.example.com", port: 443 }], - binaries: [{ path: "/usr/bin/curl" }], - }, - }, - }, - workspace: "default", - version: revision, - policyHash: "a".repeat(64), - configRevision: 11n, - providerEnvRevision: 12n, - policySource: 1, - globalPolicyVersion: 0, - }; -} - function mockSupportedLiveSource( policyVersion = 3, appliedRevision = 3, From dd2caa5388d8d6b83c2533f4573b1c48b4562fb6 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 10 Sep 2026 14:10:18 -0700 Subject: [PATCH 3/6] test(config): cover attached Ollama fixture ownership Verify the nondefault listener environment and owned daemon cleanup in the existing fast test mapped to the GPU export qualification. Refs #11435, #10904 Signed-off-by: Carlos Villela --- test/e2e/support/gpu-e2e-helpers.test.ts | 54 +++++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/test/e2e/support/gpu-e2e-helpers.test.ts b/test/e2e/support/gpu-e2e-helpers.test.ts index b1cfafaa90a..cdcdab04fd6 100644 --- a/test/e2e/support/gpu-e2e-helpers.test.ts +++ b/test/e2e/support/gpu-e2e-helpers.test.ts @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; +import { ChildProcess, execFileSync } from "node:child_process"; import { once } from "node:events"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; @@ -20,7 +20,11 @@ import { hasExactReadyPhase, ollamaCleanupScript, openClawModelConfigProjectionScript, + REPO_ROOT, + startAttachedOllama, } from "../live/gpu-e2e-helpers.ts"; +import * as observedChild from "../fixtures/observed-child-process.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; import { PROTECTED_OLLAMA_CURL_MAX_SECONDS, PROTECTED_OLLAMA_READY_ATTEMPTS, @@ -160,6 +164,52 @@ const invalidExecutionProofs: Array<{ ]; describe("GPU E2E helpers", () => { + it("owns the attached Ollama daemon with the supplied listener and cleanup (#11435)", async () => { + const progress = startTestProgress( + "attached Ollama helper", + ["start the attached daemon", "close the owned daemon"], + { + logLine: () => undefined, + }, + ); + const child = new ChildProcess(); + const spawn = vi.spyOn(observedChild, "spawnObservedChild").mockReturnValue(child); + const kill = vi.spyOn(child, "kill").mockReturnValue(true); + const environment = { OLLAMA_HOST: "127.0.0.1:11444", OLLAMA_MODELS: "/tmp/owned-models" }; + try { + const owner = startAttachedOllama(progress, environment); + expect(spawn).toHaveBeenCalledExactlyOnceWith( + "ollama", + ["serve"], + expect.objectContaining({ + progress, + spawn: { cwd: REPO_ROOT, env: environment, stdio: "ignore" }, + }), + ); + expect(owner.child).toBe(child); + let closed = false; + void owner.closed.then(() => { + closed = true; + }); + let terminated = false; + const termination = owner.terminate().then(() => { + terminated = true; + }); + expect(kill).toHaveBeenCalledExactlyOnceWith("SIGTERM"); + await Promise.resolve(); + expect([closed, terminated]).toEqual([false, false]); + child.emit("close", null, "SIGTERM"); + await termination; + await owner.closed; + expect(closed).toBe(true); + await owner.terminate(); + expect(kill).toHaveBeenCalledTimes(1); + } finally { + child.emit("close", null, "SIGTERM"); + progress.stop(); + } + }); + it("stops the Ollama system service before cleanup completes", async () => { const root = mkdtempSync(path.join(tmpdir(), "nemoclaw-ollama-cleanup-")); const listenerPort = await unusedLoopbackPort(); From 22015d84fc316571970c62e685d704a1b9c54020 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 10 Sep 2026 15:49:39 -0700 Subject: [PATCH 4/6] test(config): clarify unstable Ollama export refusal Signed-off-by: Carlos Villela --- .../adapters/config/live-export-source.test.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/lib/adapters/config/live-export-source.test.ts b/src/lib/adapters/config/live-export-source.test.ts index 7394f89ef9e..5b1f15f067f 100644 --- a/src/lib/adapters/config/live-export-source.test.ts +++ b/src/lib/adapters/config/live-export-source.test.ts @@ -1424,16 +1424,24 @@ describe("attached Ollama export pipeline", () => { vi.mocked(createOllamaExportProbe).mockImplementation(() => ollamaProbe({ ...observed, pid: ++pid }), ); - const { result, writeStdout } = await exportLiveSource(); - expect(result).toMatchObject({ ok: false }); + const { result, writeStdout, publish } = await exportLiveSource(); + expect(result).toMatchObject({ + ok: false, + failure: { + kind: "observation", + findings: expect.arrayContaining([ + expect.objectContaining({ category: "unstable-source" }), + ]), + }, + }); expect(writeStdout).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); }); }); describe("dashboard export observation", () => { it("projects registered dashboard and direct tools through complete live observation (#10904)", async () => { const sourceEntry = dashboardSource(); - expect(sourceEntry.workload?.kind).toBe("managed-image"); mockSupportedLiveSource(3, 3, sourceEntry); const reader = createLiveExportSnapshotReader(); const observed = await reader.read("alpha"); @@ -1461,7 +1469,6 @@ describe("dashboard export observation", () => { }, ); expect(result).toEqual({ ok: true, completion: { kind: "stdout" } }); - expect(raw.getSandbox).toHaveBeenCalledTimes(3); expect(JSON.stringify(result)).not.toContain(readFailureCanary); const yaml = writeStdout.mock.calls[0]?.[0] ?? ""; expect(yaml).not.toContain(readFailureCanary); From cc1eeb34f677a534f371420de5694351b14060ff Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:51:36 -0700 Subject: [PATCH 5/6] test: tighten Ollama export coverage Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- ci/e2e-assertion-budget.json | 18 ++--- .../config/live-export-source.test.ts | 72 ++++++++++--------- test/e2e/live/gpu-e2e.test.ts | 49 ++++--------- 3 files changed, 59 insertions(+), 80 deletions(-) diff --git a/ci/e2e-assertion-budget.json b/ci/e2e-assertion-budget.json index 17a406dc03e..1cce311ab2b 100644 --- a/ci/e2e-assertion-budget.json +++ b/ci/e2e-assertion-budget.json @@ -15,26 +15,26 @@ "testFileCount": 86, "liveFileCount": 222, "direct": { - "expectCalls": 1808, - "matcherAssertions": 1777, + "expectCalls": 1796, + "matcherAssertions": 1765, "nodeAssertions": 105, "namedAssertionHelpers": 601, "failCalls": 8, "throwGuards": 87, - "objectFieldAssertions": 252, - "assertionPoints": 2830, + "objectFieldAssertions": 243, + "assertionPoints": 2809, "generatedProbeBlocks": 122, "generatedProbeConditions": 322 }, "unique": { - "expectCalls": 2300, - "matcherAssertions": 2264, + "expectCalls": 2288, + "matcherAssertions": 2252, "nodeAssertions": 126, "namedAssertionHelpers": 905, "failCalls": 38, "throwGuards": 633, - "objectFieldAssertions": 351, - "assertionPoints": 4317, + "objectFieldAssertions": 342, + "assertionPoints": 4296, "generatedProbeBlocks": 276, "generatedProbeConditions": 943 }, @@ -65,7 +65,7 @@ "test/e2e/live/full-e2e.test.ts": [30,34,39,72,7], "test/e2e/live/gateway-guard-recovery.test.ts": [48,54,48,57,3], "test/e2e/live/gpu-double-onboard.test.ts": [21,24,21,24,0], - "test/e2e/live/gpu-e2e.test.ts": [51,65,84,111,3], + "test/e2e/live/gpu-e2e.test.ts": [39,44,72,90,3], "test/e2e/live/hermes-discord.test.ts": [10,25,17,64,14], "test/e2e/live/hermes-e2e.test.ts": [82,93,112,127,3], "test/e2e/live/hermes-gpu-startup.test.ts": [19,27,63,106,12], diff --git a/src/lib/adapters/config/live-export-source.test.ts b/src/lib/adapters/config/live-export-source.test.ts index 5b1f15f067f..e9f5af17a8a 100644 --- a/src/lib/adapters/config/live-export-source.test.ts +++ b/src/lib/adapters/config/live-export-source.test.ts @@ -187,6 +187,20 @@ async function exportLiveSource() { return { result, writeStdout, publish }; } +function expectExportRefusal( + exported: Awaited>, + finding: Readonly<{ field?: string; category: string }>, +) { + expect(exported.result).toMatchObject({ + ok: false, + failure: { + kind: "observation", + findings: expect.arrayContaining([expect.objectContaining(finding)]), + }, + }); + expect(exported.writeStdout).not.toHaveBeenCalled(); + expect(exported.publish).not.toHaveBeenCalled(); +} function nativeNvidiaProvider() { return { ...provider().provider, type: "nvidia", profileWorkspace: "", config: {} }; } @@ -1385,7 +1399,6 @@ describe("attached Ollama export pipeline", () => { ); expect(publish).not.toHaveBeenCalled(); }); - it("sanitizes a failed or legacy proxy observation and publishes nothing (#11435)", async () => { mockOllamaSource(); vi.mocked(createOllamaExportProbe).mockImplementation(() => { @@ -1398,47 +1411,41 @@ describe("attached Ollama export pipeline", () => { }); it.each([ - { endpointUrl: "http://host.openshell.internal:11435/v1" }, - { credentialEnv: "OTHER_TOKEN" }, - { agent: "hermes" }, - { sandboxGpuEnabled: true, sandboxGpuDevice: "nvidia.com/gpu=all" }, - ])("refuses unsupported or drifted local route intent %# (#11435)", async (change) => { + [ + { endpointUrl: "http://host.openshell.internal:11435/v1" }, + { field: "spec.inferenceProviders[].endpoint", category: "drifted" }, + ], + [ + { credentialEnv: "OTHER_TOKEN" }, + { field: "source.live", category: "live-verification-failed" }, + ], + [{ agent: "hermes" }, { field: "spec.inferenceProviders[].serving", category: "drifted" }], + [ + { sandboxGpuEnabled: true, sandboxGpuDevice: "nvidia.com/gpu=all" }, + { field: "spec.sandboxes[].runtime.gpu", category: "unsupported" }, + ], + ])("refuses unsupported or drifted local route intent %# (#11435)", async (change, finding) => { const { source } = mockOllamaSource(); Object.assign(source, change); - const { result, writeStdout } = await exportLiveSource(); - expect(result).toMatchObject({ ok: false }); - expect(writeStdout).not.toHaveBeenCalled(); + expectExportRefusal(await exportLiveSource(), finding); }); - it("refuses an absent provider attachment (#11435)", async () => { mockOllamaSource(); raw.getSandbox.mockResolvedValue(inventory()); - const { result, writeStdout } = await exportLiveSource(); - expect(result).toMatchObject({ ok: false }); - expect(writeStdout).not.toHaveBeenCalled(); + expectExportRefusal(await exportLiveSource(), { + field: "spec.inferenceProviders[].serving", + category: "drifted", + }); }); - it("refuses a continuously changing active proxy identity (#11435)", async () => { const { observed } = mockOllamaSource(); let pid = observed.pid; vi.mocked(createOllamaExportProbe).mockImplementation(() => ollamaProbe({ ...observed, pid: ++pid }), ); - const { result, writeStdout, publish } = await exportLiveSource(); - expect(result).toMatchObject({ - ok: false, - failure: { - kind: "observation", - findings: expect.arrayContaining([ - expect.objectContaining({ category: "unstable-source" }), - ]), - }, - }); - expect(writeStdout).not.toHaveBeenCalled(); - expect(publish).not.toHaveBeenCalled(); + expectExportRefusal(await exportLiveSource(), { category: "unstable-source" }); }); }); - describe("dashboard export observation", () => { it("projects registered dashboard and direct tools through complete live observation (#10904)", async () => { const sourceEntry = dashboardSource(); @@ -1480,19 +1487,14 @@ describe("dashboard export observation", () => { }); }); - it("retains dashboard registry changes in both complete snapshots (#10904)", async () => { + it("refuses dashboard registry changes without publishing (#10904)", async () => { mockSupportedLiveSource(); let reads = 0; vi.mocked(loadRegistry).mockImplementation(() => ({ sandboxes: { alpha: { ...entry, dashboardPort: reads++ % 2 === 0 ? 18789 : 19000 } }, defaultSandbox: null, })); - const result = await observeStableExportSource("alpha", createLiveExportSnapshotReader()); - expect(result).toMatchObject({ - ok: false, - attempts: 2, - findings: [expect.objectContaining({ category: "unstable-source" })], - }); - expect(loadRegistry).toHaveBeenCalledTimes(4); + const exported = await exportLiveSource(); + expectExportRefusal(exported, { category: "unstable-source" }); }); }); diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 7efa438b0f6..4ec84255684 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -119,8 +119,7 @@ test( assertNvidiaAvailable(nvidia, skip); await ensureOllama(host); - const ollamaCleanup = await cleanupOllama(host, "pre-cleanup-ollama"); - expect(ollamaCleanup.exitCode, resultText(ollamaCleanup)).toBe(0); + await cleanupOllama(host, "pre-cleanup-ollama"); progress.phase("install Ollama and GPU sandbox"); const install = await host.command("bash", ["install.sh", "--non-interactive"], { @@ -138,20 +137,17 @@ test( env: env(), timeoutMs: 120_000, }); - expect(status.exitCode, resultText(status)).toBe(0); expect(resultText(status)).toContain("Sandbox GPU: enabled"); - expect(resultText(status)).toMatch(/CUDA verified|CUDA unverified|last CUDA proof failed/i); - expect(resultText(status)).not.toMatch(/last CUDA proof failed|CUDA unverified/i); + expect(resultText(status)).toContain("CUDA verified"); const installLog = resultText(install); assertGpuInstallProofs(installLog); expect(installLog).toContain( "Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.", ); - expect(installLog).not.toContain( - "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", + expect(installLog).not.toMatch( + /Recreating OpenShell Docker sandbox container with NVIDIA GPU access|Docker GPU mode selected/u, ); - expect(installLog).not.toContain("Docker GPU mode selected"); const sandboxContainers = await runtimeProvider.command( [ @@ -169,7 +165,6 @@ test( timeoutMs: 30_000, }, ); - expect(sandboxContainers.exitCode, resultText(sandboxContainers)).toBe(0); const sandboxContainerInventory = sandboxContainers.stdout .split(/\r?\n/) .map((line) => line.trim()) @@ -192,7 +187,6 @@ test( env: env(), timeoutMs: 30_000, }); - expect(route.exitCode, resultText(route)).toBe(0); expect(resultText(route)).toMatch(/ollama/i); progress.phase("validate Ollama proxy credential boundary"); @@ -206,7 +200,6 @@ test( ["-sS", "-o", "/dev/null", "-w", "%{http_code}", `http://127.0.0.1:${PROXY_PORT}/api/tags`], { artifactName: "ollama-proxy-unauthorized", env: env(), timeoutMs: 30_000 }, ); - expect(proxyUnauth.exitCode, resultText(proxyUnauth)).toBe(0); expect(proxyUnauth.stdout).toBe("401"); const proxyAuth = await host.command( @@ -219,11 +212,8 @@ test( timeoutMs: 30_000, }, ); - expect(proxyAuth.exitCode, resultText(proxyAuth)).toBe(0); expect(proxyAuth.stdout).toMatch(/models|name/i); - const proxyBefore = await proxyStatus(host, token, "proxy-status-before-restart"); - expect(proxyBefore.exitCode, resultText(proxyBefore)).toBe(0); await restartProxy(host, token); const proxyAfter = await proxyStatus(host, token, "proxy-status-after-restart"); expect(proxyAfter.exitCode, resultText(proxyAfter)).toBe(0); @@ -258,7 +248,6 @@ test( ), { artifactName: "sandbox-inference-local-chat", env: env(), timeoutMs: 150_000 }, ); - expect(chat.exitCode, resultText(chat)).toBe(0); expect(chatContent(chat.stdout)).toMatch(/pong/i); const readySandbox = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { @@ -266,7 +255,6 @@ test( env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); - expect(readySandbox.exitCode, resultText(readySandbox)).toBe(0); expect( hasExactReadyPhase(readySandbox.stdout), `OpenShell sandbox must be exactly Ready after routed inference; got ${resultText(readySandbox)}`, @@ -303,7 +291,6 @@ exit 1`, ], { artifactName: "ollama-daemon-restart-unloaded", env: env(), timeoutMs: 90_000 }, ); - expect(restart.exitCode, resultText(restart)).toBe(0); const restartLines = restart.stdout.trim().split("\n"); expect(restartLines[0]).toMatch(/^restart_mode=(system|user|manual)$/u); expect(loadedOllamaModels(restartLines.slice(1).join("\n"))).toEqual([]); @@ -326,7 +313,6 @@ exit 1`, timeoutMs: 12 * 60_000, }, ); - expect(recovered.exitCode, resultText(recovered)).toBe(0); assertAgentExecutionSucceeded(recovered.stdout, "inference", model); const loaded = await host.command("curl", ["-fsS", "http://127.0.0.1:11434/api/ps"], { @@ -334,7 +320,6 @@ exit 1`, env: env(), timeoutMs: 30_000, }); - expect(loaded.exitCode, resultText(loaded)).toBe(0); expect(loadedOllamaModels(loaded.stdout)).toContain(model); }, ); @@ -541,22 +526,15 @@ test( const model = ( JSON.parse(tags.stdout) as { models: Array<{ name: string; digest: string }> } ).models.find(({ name }) => name === "qwen3.5:9b"); - expect(document.spec.inferenceProviders).toEqual([ - { - name: "local-ollama", - provider: "ollama-local", - api: "openai-completions", - serving: { - backend: "ollama", - daemon: { management: "external", hostPort: 11439 }, - proxy: { management: "nemoclaw", hostPort: 11440 }, - model: { - servedName: "qwen3.5:9b", - digest: `sha256:${model?.digest.replace(/^sha256:/u, "")}`, - }, - }, - }, - ]); + const exportedProvider = document.spec.inferenceProviders[0]; + const serving = + "serving" in exportedProvider && exportedProvider.serving.backend === "ollama" + ? exportedProvider.serving + : undefined; + expect(exportedProvider.provider).toBe("ollama-local"); + expect(serving?.daemon.hostPort).toBe(11439); + expect(serving?.proxy.hostPort).toBe(11440); + expect(serving?.model.digest).toBe(`sha256:${model?.digest.replace(/^sha256:/u, "")}`); const entry = loadRegistry().sandboxes[SANDBOX_NAME]; expect(document.spec.sandboxes[0].runtime.image.ref).toBe( entry.workload?.kind === "managed-image" ? entry.workload.reference : null, @@ -567,7 +545,6 @@ test( [CLI, "config", "export", SANDBOX_NAME, "--output", repeatPath, "--json"], { artifactName: "export-ollama-repeat", cwd: REPO_ROOT, env: exportEnv, timeoutMs: 60000 }, ); - expect(repeated.exitCode, resultText(repeated)).toBe(0); expect(validateNemoClawConfig(YAML.parse(fs.readFileSync(repeatPath, "utf8"))).spec).toEqual( document.spec, ); From 14d41804031ce2523172f2371cc94345c5ec055d Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:13:38 -0700 Subject: [PATCH 6/6] test: remove unused export result Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/live/gpu-e2e.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 4ec84255684..b6474487a89 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -540,7 +540,7 @@ test( entry.workload?.kind === "managed-image" ? entry.workload.reference : null, ); const repeatPath = path.join(directory, "repeat.yaml"); - const repeated = await host.command( + await host.command( "node", [CLI, "config", "export", SANDBOX_NAME, "--output", repeatPath, "--json"], { artifactName: "export-ollama-repeat", cwd: REPO_ROOT, env: exportEnv, timeoutMs: 60000 },