From d591a909800978d2ce046267eb698bf4bbbdf778 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 31 Aug 2026 20:13:54 -0700 Subject: [PATCH 01/47] fix(inference): keep Windows Ollama requests in Docker Signed-off-by: Aaron Erickson --- .../local-windows-ollama-transport.test.ts | 65 +++++++++++++++++++ src/lib/inference/local.ts | 62 +++++++++++++----- src/lib/inference/ollama/proxy.test.ts | 7 +- src/lib/inference/ollama/proxy.ts | 9 ++- 4 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 src/lib/inference/local-windows-ollama-transport.test.ts diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts new file mode 100644 index 00000000000..0462124f232 --- /dev/null +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + CONTAINER_REACHABILITY_IMAGE, + getOllamaModelOptions, + getOllamaProbeCommand, + getOllamaWarmupCommand, + OLLAMA_HOST_DOCKER_INTERNAL, + resetOllamaHostCache, + setResolvedOllamaHost, +} from "./local"; + +describe("Windows-host Ollama transport", () => { + afterEach(() => { + resetOllamaHostCache(); + }); + + it("reads the model inventory through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn(() => JSON.stringify({ models: [{ name: "qwen3.5:9b" }] })); + + expect(getOllamaModelOptions(capture)).toEqual(["qwen3.5:9b"]); + expect(capture).toHaveBeenCalledWith( + [ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "-sf", + "--connect-timeout", + "3", + "--max-time", + "5", + "http://host.docker.internal:11434/api/tags", + ], + { ignoreError: true }, + ); + }); + + it("warms and validates models through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + + const warmup = getOllamaWarmupCommand("qwen3.5:9b"); + expect(warmup[2]).toContain(`'docker' 'run' '--rm' '${CONTAINER_REACHABILITY_IMAGE}'`); + expect(warmup[2]).toContain("http://host.docker.internal:11434/api/generate"); + + expect(getOllamaProbeCommand("qwen3.5:9b")).toEqual([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "-sS", + "--max-time", + "120", + "http://host.docker.internal:11434/api/generate", + "-H", + "Content-Type: application/json", + "-d", + expect.stringContaining('"model":"qwen3.5:9b"'), + ]); + }); +}); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 3910f02a04b..78f458112ad 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -195,6 +195,16 @@ export function getResolvedOllamaHost(): string { return _resolvedOllamaHost ?? OLLAMA_LOCALHOST; } +/** Keep Windows-host Ollama requests in Docker Desktop's verified network context. */ +export function getOllamaApiCommand( + curlArgs: readonly string[], + host: string = getResolvedOllamaHost(), +): string[] { + return host === OLLAMA_HOST_DOCKER_INTERNAL + ? ["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE, ...curlArgs] + : ["curl", ...curlArgs]; +} + export function resetOllamaHostCache(): void { _resolvedOllamaHost = null; } @@ -1581,21 +1591,21 @@ export function getOllamaModelOptions( const capture = runCaptureImpl ?? runCapture; const host = getResolvedOllamaHost(); const modelDiscoveryRetryDelaysMs = [500, 1_000] as const; + // Docker Desktop owns Windows-host reachability because host.docker.internal + // may not resolve from WSL. Keep model discovery on the verified transport. + const tagsCommand = getOllamaApiCommand( + buildValidatedCurlCommandArgs([ + "-sf", + "--connect-timeout", + "3", + "--max-time", + "5", + `http://${host}:${OLLAMA_PORT}/api/tags`, + ]), + host, + ); const readTags = () => { - const tagsOutput = capture( - [ - "curl", - ...buildValidatedCurlCommandArgs([ - "-sf", - "--connect-timeout", - "3", - "--max-time", - "5", - `http://${host}:${OLLAMA_PORT}/api/tags`, - ]), - ], - { ignoreError: true }, - ); + const tagsOutput = capture(tagsCommand, { ignoreError: true }); return parseOllamaModelInventory(String(tagsOutput || "")); }; // The daemon can become unreachable after the earlier readiness check. @@ -1728,6 +1738,24 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string options: { num_predict: 16 }, }); const host = getResolvedOllamaHost(); + if (host !== OLLAMA_HOST_DOCKER_INTERNAL) { + return [ + "bash", + "-c", + `nohup curl -s http://${host}:${OLLAMA_PORT}/api/generate -H 'Content-Type: application/json' -d ${shellQuote(payload)} >/dev/null 2>&1 &`, + ]; + } + const command = getOllamaApiCommand( + [ + "-s", + `http://${host}:${OLLAMA_PORT}/api/generate`, + "-H", + "Content-Type: application/json", + "-d", + payload, + ], + host, + ); // backgrounding (nohup ... &) and output redirection require a shell wrapper. // The payload is safe: model name is JSON-serialized (escaping all special // chars) then shellQuote'd (single-quoted), so injection through model @@ -1735,7 +1763,7 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string return [ "bash", "-c", - `nohup curl -s http://${host}:${OLLAMA_PORT}/api/generate -H 'Content-Type: application/json' -d ${shellQuote(payload)} >/dev/null 2>&1 &`, + `nohup ${command.map((arg) => shellQuote(arg)).join(" ")} >/dev/null 2>&1 &`, ]; } @@ -1763,8 +1791,7 @@ export function getOllamaProbeCommand( payload, endpoint, ]); - return [ - "curl", + const curlArgs = [ "-sS", "--max-time", String(timeoutSeconds), @@ -1774,6 +1801,7 @@ export function getOllamaProbeCommand( "-d", payload, ]; + return getOllamaApiCommand(curlArgs, host); } export function validateOllamaModel( diff --git a/src/lib/inference/ollama/proxy.test.ts b/src/lib/inference/ollama/proxy.test.ts index fc56d900ec2..87b9a0384ea 100644 --- a/src/lib/inference/ollama/proxy.test.ts +++ b/src/lib/inference/ollama/proxy.test.ts @@ -438,13 +438,16 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { vi.restoreAllMocks(); }); - it("pulls over HTTP when the daemon resolves on the Windows host", async () => { + it("pulls through Docker when the daemon resolves on the Windows host (#10553)", async () => { vi.spyOn(console, "log").mockImplementation(() => {}); active = loadProxyForDispatch({ host: "host.docker.internal", hasLocalCli: true }); await active.proxy.pullOllamaModel("qwen3.5:9b"); - expect(active.httpCommands.map((command) => command[0])).toContain("curl"); + expect(active.httpCommands.map((command) => command[0])).toContain("docker"); + expect(active.httpCommands[0]).toEqual( + expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"]), + ); expect(active.cliCommands.map((command) => command[0])).not.toContain("bash"); }); diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 94354ab059a..d6393debaa6 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -30,6 +30,7 @@ const { ensurePulledOllamaModel }: typeof import("./model-discovery") = const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./model-discovery"); const { getBootstrapOllamaModelOptions, + getOllamaApiCommand, getOllamaModelOptions, getOllamaWarmupCommand, getResolvedOllamaHost, @@ -1043,8 +1044,7 @@ function pullOllamaModelViaHttp(model: string): Promise { // The endpoint is restricted to the local Ollama hosts NemoClaw probes and // the model id is normalized before being serialized as JSON request data. - const proc = spawn( - "curl", + const [executable, ...args] = getOllamaApiCommand( [ "-sN", "--connect-timeout", @@ -1060,6 +1060,11 @@ function pullOllamaModelViaHttp(model: string): Promise { body, url, ], + host, + ); + const proc = spawn( + executable, + args, { stdio: ["ignore", "pipe", "pipe"], // #2616: inject NO_PROXY=localhost so the streamed pull against the From a2e4956b85ba35435755ab5a1c7845821f87f73b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 31 Aug 2026 21:09:59 -0700 Subject: [PATCH 02/47] fix(inference): complete Windows Ollama transport Signed-off-by: Aaron Erickson --- .../local-windows-ollama-transport.test.ts | 83 +++++++++++++++++-- src/lib/inference/local.test.ts | 3 +- src/lib/inference/local.ts | 51 ++++++++---- src/lib/inference/ollama/proxy.test.ts | 20 ++++- src/lib/inference/ollama/proxy.ts | 66 ++++++++++----- .../ollama/ollama-gpu-cleanup.test.ts | 9 +- .../ollama/ollama-pull-timeout.test.ts | 5 +- 7 files changed, 189 insertions(+), 48 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 0462124f232..f0430c6c36c 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -4,18 +4,22 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + applyOllamaRuntimeContextWindow, CONTAINER_REACHABILITY_IMAGE, getOllamaModelOptions, getOllamaProbeCommand, - getOllamaWarmupCommand, + getOllamaWarmupRequestCommand, OLLAMA_HOST_DOCKER_INTERNAL, + probeOllamaModelCapabilities, resetOllamaHostCache, + resetOllamaRuntimeContextWindowAutoState, setResolvedOllamaHost, } from "./local"; describe("Windows-host Ollama transport", () => { afterEach(() => { resetOllamaHostCache(); + resetOllamaRuntimeContextWindowAutoState(); }); it("reads the model inventory through Docker Desktop (#10553)", () => { @@ -40,12 +44,25 @@ describe("Windows-host Ollama transport", () => { ); }); - it("warms and validates models through Docker Desktop (#10553)", () => { + it("builds warm-up and validation requests for Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const warmup = getOllamaWarmupCommand("qwen3.5:9b"); - expect(warmup[2]).toContain(`'docker' 'run' '--rm' '${CONTAINER_REACHABILITY_IMAGE}'`); - expect(warmup[2]).toContain("http://host.docker.internal:11434/api/generate"); + expect(getOllamaWarmupRequestCommand("qwen3.5:9b")).toEqual([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "-s", + "--connect-timeout", + "10", + "--max-time", + "120", + "http://host.docker.internal:11434/api/generate", + "-H", + "Content-Type: application/json", + "-d", + expect.stringContaining('"model":"qwen3.5:9b"'), + ]); expect(getOllamaProbeCommand("qwen3.5:9b")).toEqual([ "docker", @@ -62,4 +79,60 @@ describe("Windows-host Ollama transport", () => { expect.stringContaining('"model":"qwen3.5:9b"'), ]); }); + + it("checks the Hermes context window and model metadata through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const responses: Record = { + "http://host.docker.internal:11434/api/ps": JSON.stringify({ + models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }], + }), + "http://host.docker.internal:11434/api/show": JSON.stringify({ capabilities: ["tools"] }), + }; + const capture = vi.fn((command: readonly string[]) => { + return responses[String(command.at(-1))] ?? ""; + }); + const env: NodeJS.ProcessEnv = {}; + + expect( + applyOllamaRuntimeContextWindow("qwen3.5:9b", { + contextWindowFloor: 64_000, + env, + logger: { log: vi.fn(), warn: vi.fn() }, + runCaptureImpl: capture, + }), + ).toEqual({ ok: true }); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); + expect(probeOllamaModelCapabilities("qwen3.5:9b", capture)).toMatchObject({ + source: "api", + supportsTools: true, + }); + expect(capture).toHaveBeenCalledTimes(2); + capture.mock.calls.forEach(([command]) => { + expect(command).toEqual( + expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + ); + }); + }); + + it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((_command: readonly string[]) => + JSON.stringify({ models: [{ name: "qwen3.5:9b", context_length: "invalid" }] }), + ); + + const result = applyOllamaRuntimeContextWindow("qwen3.5:9b", { + contextWindowFloor: 64_000, + env: {}, + logger: { log: vi.fn(), warn: vi.fn() }, + runCaptureImpl: capture, + }); + + expect(result).toMatchObject({ + ok: false, + message: expect.stringContaining("cannot verify the required 64000-token window"), + }); + expect(capture.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + ); + }); }); diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 7b5cdb27c57..abdf0c09d64 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -1212,7 +1212,8 @@ describe("local inference helpers", () => { it("builds a background warmup command for ollama models", () => { const command = getOllamaWarmupCommand("nemotron-3-nano:30b"); expect(command).toEqual(expect.arrayContaining(["bash", "-c"])); - expect(command[2]).toMatch(/^nohup curl -s http:\/\/127.0.0.1:11434\/api\/generate /); + expect(command[2]).toContain("'--connect-timeout' '10' '--max-time' '120'"); + expect(command[2]).toContain("http://127.0.0.1:11434/api/generate"); expect(command[2]).toMatch(/"model":"nemotron-3-nano:30b"/); expect(command[2]).toMatch(/"keep_alive":"15m"/); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 78f458112ad..1db3f229430 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -205,6 +205,14 @@ export function getOllamaApiCommand( : ["curl", ...curlArgs]; } +function createOllamaApiCapture(runCaptureImpl?: RunCaptureFn): RunCaptureFn { + const capture = runCaptureImpl ?? runCapture; + return (command, options) => { + const [executable, ...args] = command; + return capture(executable === "curl" ? getOllamaApiCommand(args) : command, options); + }; +} + export function resetOllamaHostCache(): void { _resolvedOllamaHost = null; } @@ -1538,7 +1546,11 @@ export function probeOllamaRuntimeModelStatus( model: string, runCaptureImpl?: RunCaptureFn, ): OllamaRuntimeModelStatus { - return probeOllamaRuntimeModelStatusWithHost(model, getResolvedOllamaHost, runCaptureImpl); + return probeOllamaRuntimeModelStatusWithHost( + model, + getResolvedOllamaHost, + createOllamaApiCapture(runCaptureImpl), + ); } export function resolveOllamaRuntimeContextWindow( @@ -1550,7 +1562,7 @@ export function resolveOllamaRuntimeContextWindow( model, currentContextWindow, getResolvedOllamaHost, - runCaptureImpl, + createOllamaApiCapture(runCaptureImpl), ); } @@ -1559,9 +1571,15 @@ export { resetOllamaRuntimeContextWindowAutoState }; /** Apply Ollama runtime context-window adoption using the resolved local host. */ export function applyOllamaRuntimeContextWindow( selectedModel: string, - options: Pick = {}, + options: Pick< + ApplyOllamaRuntimeContextWindowOptions, + "contextWindowFloor" | "env" | "logger" | "runCaptureImpl" + > = {}, ): ApplyOllamaRuntimeContextWindowResult { - return applyOllamaRuntimeContextWindowWithHost(selectedModel, getResolvedOllamaHost, options); + return applyOllamaRuntimeContextWindowWithHost(selectedModel, getResolvedOllamaHost, { + ...options, + runCaptureImpl: createOllamaApiCapture(options.runCaptureImpl), + }); } export function applyVllmRuntimeContextWindow( @@ -1729,7 +1747,7 @@ export function selectDefaultOllamaModel( : pool[0]; } -export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string[] { +export function getOllamaWarmupRequestCommand(model: string, keepAlive = "15m"): string[] { const payload = JSON.stringify({ model, prompt: "Hello, reply in less than 5 words", @@ -1738,16 +1756,13 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string options: { num_predict: 16 }, }); const host = getResolvedOllamaHost(); - if (host !== OLLAMA_HOST_DOCKER_INTERNAL) { - return [ - "bash", - "-c", - `nohup curl -s http://${host}:${OLLAMA_PORT}/api/generate -H 'Content-Type: application/json' -d ${shellQuote(payload)} >/dev/null 2>&1 &`, - ]; - } - const command = getOllamaApiCommand( + return getOllamaApiCommand( [ "-s", + "--connect-timeout", + "10", + "--max-time", + "120", `http://${host}:${OLLAMA_PORT}/api/generate`, "-H", "Content-Type: application/json", @@ -1756,6 +1771,10 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string ], host, ); +} + +export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string[] { + const command = getOllamaWarmupRequestCommand(model, keepAlive); // backgrounding (nohup ... &) and output redirection require a shell wrapper. // The payload is safe: model name is JSON-serialized (escaping all special // chars) then shellQuote'd (single-quoted), so injection through model @@ -1961,7 +1980,11 @@ export function probeOllamaModelCapabilities( model: string, runCaptureImpl?: RunCaptureFn, ): OllamaCapabilities { - const metadata = fetchOllamaModelShowMetadata(model, getResolvedOllamaHost, runCaptureImpl); + const metadata = fetchOllamaModelShowMetadata( + model, + getResolvedOllamaHost, + createOllamaApiCapture(runCaptureImpl), + ); if (!metadata.ok) { return { source: "unknown", diff --git a/src/lib/inference/ollama/proxy.test.ts b/src/lib/inference/ollama/proxy.test.ts index 87b9a0384ea..4a4b6d7489a 100644 --- a/src/lib/inference/ollama/proxy.test.ts +++ b/src/lib/inference/ollama/proxy.test.ts @@ -442,12 +442,26 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { vi.spyOn(console, "log").mockImplementation(() => {}); active = loadProxyForDispatch({ host: "host.docker.internal", hasLocalCli: true }); - await active.proxy.pullOllamaModel("qwen3.5:9b"); + const result = await active.proxy.pullOllamaModel("qwen3.5:9b"); + expect(result).toBe(true); expect(active.httpCommands.map((command) => command[0])).toContain("docker"); - expect(active.httpCommands[0]).toEqual( - expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"]), + const request = active.httpCommands[0]; + expect(request).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "curlimages/curl:8.10.1", + "-X", + "POST", + "Content-Type: application/json", + "http://host.docker.internal:11434/api/pull", + ]), ); + expect(JSON.parse(request[request.indexOf("-d") + 1])).toEqual({ + model: "qwen3.5:9b", + stream: true, + }); expect(active.cliCommands.map((command) => command[0])).not.toContain("bash"); }); diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index d6393debaa6..b1e101423d9 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -1384,15 +1384,20 @@ function defaultReleaseSleep(milliseconds: number): void { function discoverResidentOllamaModels( attempt: number, selectedModels: readonly string[] | null, + releaseHost: string, releaseEndpoint: string, spawnSyncImpl: typeof spawnSync, ): OllamaModelDiscoveryEvidence { const endpoint = `${releaseEndpoint}/api/ps`; + const [command, ...args] = getOllamaApiCommand( + ["-sS", "--fail-with-body", "--max-time", "3", endpoint], + releaseHost, + ); let result; try { result = spawnSyncImpl( - "curl", - ["-sS", "--fail-with-body", "--max-time", "3", endpoint], + command, + args, // #2616: env-sanitize so an ambient HTTP proxy cannot intercept the // loopback-only Ollama ownership and release checks. { encoding: "utf8", env: buildSubprocessEnv() }, @@ -1476,9 +1481,8 @@ function unloadOllamaModels( onlyModels?: readonly string[], options: OllamaUnloadOptions = {}, ): OllamaUnloadResult { - const releaseEndpoint = buildLocalOllamaEndpoint( - options.getResolvedOllamaHost ?? getResolvedOllamaHost, - ); + const releaseHost = (options.getResolvedOllamaHost ?? getResolvedOllamaHost)(); + const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost); const spawnSyncImpl = options.spawnSync ?? spawnSync; const sleepImpl = options.sleep ?? defaultReleaseSleep; const maxAttempts = Math.max(1, options.maxAttempts ?? OLLAMA_RELEASE_MAX_ATTEMPTS); @@ -1489,7 +1493,13 @@ function unloadOllamaModels( let lastMatchedModels: readonly string[] = []; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const discovery = discoverResidentOllamaModels(attempt, selectedModels, releaseEndpoint, spawnSyncImpl); + const discovery = discoverResidentOllamaModels( + attempt, + selectedModels, + releaseHost, + releaseEndpoint, + spawnSyncImpl, + ); discoveries.push(discovery); if (discovery.error) { if (attempt < maxAttempts && transientCurlFailure(discovery.status)) { @@ -1523,25 +1533,29 @@ function unloadOllamaModels( let retryRequest = false; for (const model of lastMatchedModels) { const endpoint = `${releaseEndpoint}/api/generate`; + const [command, ...args] = getOllamaApiCommand( + [ + "-sS", + "--fail-with-body", + "-o", + "/dev/null", + "--max-time", + "3", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + JSON.stringify({ model, keep_alive: 0 }), + endpoint, + ], + releaseHost, + ); let result; try { result = spawnSyncImpl( - "curl", - [ - "-sS", - "--fail-with-body", - "-o", - "/dev/null", - "--max-time", - "3", - "-X", - "POST", - "-H", - "Content-Type: application/json", - "-d", - JSON.stringify({ model, keep_alive: 0 }), - endpoint, - ], + command, + args, { encoding: "utf8", env: buildSubprocessEnv() }, ); } catch (error) { @@ -1582,7 +1596,13 @@ function unloadOllamaModels( } sleepImpl(OLLAMA_RELEASE_VERIFY_DELAY_MS); - const verification = discoverResidentOllamaModels(attempt, selectedModels, releaseEndpoint, spawnSyncImpl); + const verification = discoverResidentOllamaModels( + attempt, + selectedModels, + releaseHost, + releaseEndpoint, + spawnSyncImpl, + ); discoveries.push(verification); if (verification.error) { if (attempt < maxAttempts && transientCurlFailure(verification.status)) { diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index b2641a3fbb6..528ece0b477 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -91,13 +91,20 @@ describe("Ollama GPU cleanup", () => { outcome: "released", endpoint: "http://host.docker.internal:11434", }); + const dockerCalls = calls.filter(({ command }) => command === "docker"); + expect(dockerCalls).toHaveLength(3); expect( - calls.filter(({ command }) => command === "curl").map(({ args }) => args.at(-1)), + dockerCalls.map(({ args }) => args.at(-1)), ).toEqual([ "http://host.docker.internal:11434/api/ps", "http://host.docker.internal:11434/api/generate", "http://host.docker.internal:11434/api/ps", ]); + dockerCalls.forEach(({ args }) => { + expect(args).toEqual( + expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"]), + ); + }); }, "host.docker.internal", ); diff --git a/test/inference/ollama/ollama-pull-timeout.test.ts b/test/inference/ollama/ollama-pull-timeout.test.ts index a61f95a556a..0296dd05789 100644 --- a/test/inference/ollama/ollama-pull-timeout.test.ts +++ b/test/inference/ollama/ollama-pull-timeout.test.ts @@ -100,7 +100,10 @@ pullOllamaModel("qwen3.5:9b") expect(result.status, result.stderr).toBe(0); const payload = JSON.parse(result.stdout.trim()); expect(payload.ok).toBe(true); - expect(payload.captured.cmd).toBe("curl"); + expect(payload.captured.cmd).toBe("docker"); + expect(payload.captured.args).toEqual( + expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"]), + ); const maxTimeIndex = payload.captured.args.indexOf("--max-time"); expect(maxTimeIndex).toBeGreaterThanOrEqual(0); expect(payload.captured.args[maxTimeIndex + 1]).toBe("0.5"); From a426ab8bd62c4bbde81a377cfc8ba28c464559f4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 31 Aug 2026 22:02:54 -0700 Subject: [PATCH 03/47] fix(inference): persist Windows Ollama cleanup route --- .../agent/ollama-restart-recovery.test.ts | 6 +- .../sandbox/agent/ollama-restart-recovery.ts | 26 +++-- src/lib/actions/sandbox/destroy.ts | 31 +++-- .../local-windows-ollama-transport.test.ts | 108 +++++++++++++++--- src/lib/inference/local.ts | 63 +++++++++- src/lib/inference/ollama/proxy.ts | 6 +- .../inference-providers/ollama-local.test.ts | 36 +++++- .../inference-providers/ollama-local.ts | 10 ++ src/lib/onboard/inference-providers/types.ts | 1 + src/lib/tunnel/services.test.ts | 21 ++++ src/lib/tunnel/services.ts | 34 ++++-- .../ollama/ollama-gpu-cleanup.test.ts | 51 ++++++++- .../destroy-cleanup-sandbox-services.test.ts | 19 +++ 13 files changed, 358 insertions(+), 54 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 43319de1d81..8939234c5cb 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -63,9 +63,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, ); + expect(runCaptureImpl.mock.calls[0][0][0]).toBe("docker"); expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); + expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("docker"); expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({ model: "qwen3.6:35b", stream: false, @@ -89,9 +91,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, ); + expect(runCaptureImpl.mock.calls[0][0][0]).toBe("curl"); expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, ); + expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("curl"); }); it("falls back to an allowlisted host instead of probing an arbitrary registry URL", () => { @@ -228,7 +232,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, inventoryLabel: "llama3.2:1b", }); - expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", undefined); + expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", expect.any(Function)); }); it("keeps the warm failure when the daemon does hold the model (#9455)", () => { diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 8fa79227865..aaaea51133d 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -18,6 +18,7 @@ import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args" import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, + getOllamaApiCommand, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, @@ -30,7 +31,7 @@ import { type OllamaRuntimeRunCaptureFn, probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; -import { runCaptureEx } from "../../../runner"; +import { runCapture, runCaptureEx } from "../../../runner"; export interface OllamaRestartRecoveryRoute { provider?: string | null; @@ -150,9 +151,8 @@ function buildWarmCommand(model: string, hostname: string): string[] { keep_alive: "15m", options: { num_predict: 16 }, }); - return [ - "curl", - ...buildValidatedCurlCommandArgs([ + return getOllamaApiCommand( + buildValidatedCurlCommandArgs([ "-sS", "--connect-timeout", "3", @@ -164,7 +164,18 @@ function buildWarmCommand(model: string, hostname: string): string[] { body, `http://${hostname}:${OLLAMA_PORT}/api/generate`, ]), - ]; + hostname, + ); +} + +function createRawOllamaCapture( + hostname: string, + capture: OllamaRuntimeRunCaptureFn, +): OllamaRuntimeRunCaptureFn { + return (command, options) => { + const [executable, ...args] = command; + return capture(executable === "curl" ? getOllamaApiCommand(args, hostname) : command, options); + }; } function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" { @@ -209,9 +220,10 @@ export function maybeWarmOllamaAfterDaemonRestart( const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; + const rawCapture = createRawOllamaCapture(rawHost, deps.runCaptureImpl ?? runCapture); let status: OllamaRuntimeModelStatus; try { - status = probe(model, () => rawHost, deps.runCaptureImpl); + status = probe(model, () => rawHost, rawCapture); } catch { return { kind: "skipped", reason: "unreachable" }; } @@ -239,7 +251,7 @@ export function maybeWarmOllamaAfterDaemonRestart( // an unreadable inventory keeps the original warm-failure reason. if (response === "ollama-error") { const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory; - const inventory = probeInventory(rawHost, deps.runCaptureImpl); + const inventory = probeInventory(rawHost, rawCapture); if (inventory && !ollamaInventoryContainsModel(inventory, model)) { return { kind: "skipped", diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 2521f0c6ed8..0e77b653d4e 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -22,6 +22,7 @@ import { revokeHttpsPinRuntimeAdapterRoute, } from "../../inference/https-pin-runtime-adapter"; import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup"; +import type { OllamaUnloadResult } from "../../inference/ollama/proxy"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, normalizeRuntimeProviderIdentity, @@ -165,8 +166,8 @@ type RunOpenshell = (args: string[], opts?: Record) => { status export type CleanupSandboxServicesDeps = { getSandbox?: typeof registry.getSandbox; - stopAll?: (opts: { sandboxName: string }) => void; - unloadOllamaModels?: () => void; + stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void; + unloadOllamaModels?: () => OllamaUnloadResult | void; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; @@ -227,17 +228,17 @@ export function cleanupSandboxServices( deps.stopAll ?? ((opts: { sandboxName: string }) => { const services = require("../../tunnel/services") as { - stopAll: (opts: { sandboxName: string }) => void; + stopAll: (opts: { sandboxName: string }) => OllamaUnloadResult | void; }; - services.stopAll(opts); + return services.stopAll(opts); }); const unloadOllamaModels = deps.unloadOllamaModels ?? (() => { const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as { - unloadOllamaModels: () => void; + unloadOllamaModels: () => OllamaUnloadResult; }; - unload(); + return unload(); }); const runOpenshell = deps.runOpenshell ?? @@ -289,14 +290,28 @@ export function cleanupSandboxServices( if (stopHostServices) { // `stopAll()` already runs `unloadOllamaModels()` unconditionally — // see src/lib/tunnel/services.ts. Don't double-call here. - stopAll({ sandboxName: validatedSandboxName }); + const cleanup = stopAll({ sandboxName: validatedSandboxName }); + if (cleanup && !cleanup.ok) { + throw new Error( + `Sandbox host services stopped, but Ollama model cleanup failed at ${cleanup.endpoint} ` + + `(${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was ` + + "retained; repair Ollama and retry destroy.", + ); + } } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this // branch a single-sandbox destroy would leave models loaded on the GPU. const sb = getSandbox(validatedSandboxName); if (sb?.provider?.includes("ollama")) { - unloadOllamaModels(); + const cleanup = unloadOllamaModels(); + if (cleanup && !cleanup.ok) { + throw new Error( + `Sandbox resources were removed, but Ollama model cleanup failed at ${cleanup.endpoint} ` + + `(${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was ` + + "retained; repair Ollama and retry destroy.", + ); + } } } diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index f0430c6c36c..30e16af54cd 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -1,19 +1,27 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { applyOllamaRuntimeContextWindow, CONTAINER_REACHABILITY_IMAGE, + getLocalProviderHealthCheck, + getOllamaHostForCleanup, getOllamaModelOptions, getOllamaProbeCommand, getOllamaWarmupRequestCommand, OLLAMA_HOST_DOCKER_INTERNAL, + loadPersistedOllamaHost, + persistResolvedOllamaHost, probeOllamaModelCapabilities, resetOllamaHostCache, resetOllamaRuntimeContextWindowAutoState, setResolvedOllamaHost, + validateLocalProvider, } from "./local"; describe("Windows-host Ollama transport", () => { @@ -22,6 +30,20 @@ describe("Windows-host Ollama transport", () => { resetOllamaRuntimeContextWindowAutoState(); }); + it("restores the accepted route for cleanup in a fresh process", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-receipt-")); + try { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + persistResolvedOllamaHost(undefined, stateRoot); + resetOllamaHostCache(); + + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + expect(getOllamaHostForCleanup(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("reads the model inventory through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const capture = vi.fn(() => JSON.stringify({ models: [{ name: "qwen3.5:9b" }] })); @@ -80,17 +102,61 @@ describe("Windows-host Ollama transport", () => { ]); }); - it("checks the Hermes context window and model metadata through Docker Desktop (#10553)", () => { + it("validates health and container reachability through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const responses: Record = { - "http://host.docker.internal:11434/api/ps": JSON.stringify({ - models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }], - }), - "http://host.docker.internal:11434/api/show": JSON.stringify({ capabilities: ["tools"] }), - }; - const capture = vi.fn((command: readonly string[]) => { - return responses[String(command.at(-1))] ?? ""; - }); + const capture = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + + expect(getLocalProviderHealthCheck("ollama-local")).toEqual([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "-sf", + "http://host.docker.internal:11434/api/tags", + ]); + expect( + validateLocalProvider( + "ollama-local", + capture, + () => {}, + () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + ), + ).toEqual({ ok: true }); + + expect(capture).toHaveBeenCalledTimes(2); + expect(capture.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/tags", + ]), + ); + expect(capture.mock.calls[1]?.[0]).toEqual( + expect.arrayContaining([ + "docker", + "run", + "--rm", + "--add-host", + "host.openshell.internal:host-gateway", + ]), + ); + }); + + it("checks the Hermes context window through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((command: readonly string[]) => + String(command.at(-1)).endsWith("/api/ps") + ? JSON.stringify({ + models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }], + }) + : "", + ); const env: NodeJS.ProcessEnv = {}; expect( @@ -102,16 +168,26 @@ describe("Windows-host Ollama transport", () => { }), ).toEqual({ ok: true }); expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); + expect(capture).toHaveBeenCalledOnce(); + expect(capture.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + ); + }); + + it("checks model capability metadata through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((_command: readonly string[]) => + JSON.stringify({ capabilities: ["tools"] }), + ); + expect(probeOllamaModelCapabilities("qwen3.5:9b", capture)).toMatchObject({ source: "api", supportsTools: true, }); - expect(capture).toHaveBeenCalledTimes(2); - capture.mock.calls.forEach(([command]) => { - expect(command).toEqual( - expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), - ); - }); + expect(capture).toHaveBeenCalledOnce(); + expect(capture.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + ); }); it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 1db3f229430..e183d344e37 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -30,7 +30,11 @@ import { containerCanReachHostLoopback, isWsl, type WslDetectionOptions } from " import { type CaptureResult, runCapture, runCaptureEx, shellQuote } from "../runner"; import { buildSubprocessEnv } from "../subprocess-env"; -import { resolveSharedLocalAdapterStateRoot } from "./local-adapter-lifecycle"; +import { + readLocalAdapterJsonFile, + resolveSharedLocalAdapterStateRoot, + writeLocalAdapterJsonFile, +} from "./local-adapter-lifecycle"; import { detectNvidiaPlatform } from "./nim"; import { anyRegistryModelFits, @@ -147,6 +151,22 @@ export function getWindowsHostOllamaDockerReachabilityArgs(): string[] { } let _resolvedOllamaHost: string | null = null; +const OLLAMA_HOST_RECEIPT_NAME = "ollama-host.json"; + +type OllamaHostReceipt = { + readonly schemaVersion: 1; + readonly host: typeof OLLAMA_LOCALHOST | typeof OLLAMA_HOST_DOCKER_INTERNAL; +}; + +function isSupportedOllamaHost( + host: unknown, +): host is typeof OLLAMA_LOCALHOST | typeof OLLAMA_HOST_DOCKER_INTERNAL { + return host === OLLAMA_LOCALHOST || host === OLLAMA_HOST_DOCKER_INTERNAL; +} + +function ollamaHostReceiptPath(stateRoot: string): string { + return nodePath.join(stateRoot, OLLAMA_HOST_RECEIPT_NAME); +} function ollamaCandidateHosts(wslDetection: WslDetectionOptions = {}): string[] { return isWsl(wslDetection) ? [OLLAMA_LOCALHOST, OLLAMA_HOST_DOCKER_INTERNAL] : [OLLAMA_LOCALHOST]; @@ -195,6 +215,35 @@ export function getResolvedOllamaHost(): string { return _resolvedOllamaHost ?? OLLAMA_LOCALHOST; } +/** Persist the accepted local Ollama route for later CLI processes. */ +export function persistResolvedOllamaHost( + host: string = getResolvedOllamaHost(), + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + if (!isSupportedOllamaHost(host)) { + throw new Error(`Refusing to persist unexpected Ollama host: ${host}`); + } + writeLocalAdapterJsonFile(ollamaHostReceiptPath(stateRoot), { + schemaVersion: 1, + host, + } satisfies OllamaHostReceipt); +} + +/** Read only the two fixed local Ollama routes NemoClaw can establish. */ +export function loadPersistedOllamaHost( + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): typeof OLLAMA_LOCALHOST | typeof OLLAMA_HOST_DOCKER_INTERNAL | null { + const receipt = readLocalAdapterJsonFile(ollamaHostReceiptPath(stateRoot)); + return receipt?.schemaVersion === 1 && isSupportedOllamaHost(receipt.host) ? receipt.host : null; +} + +/** Resolve cleanup transport after process-local onboarding state is gone. */ +export function getOllamaHostForCleanup( + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): string { + return _resolvedOllamaHost ?? loadPersistedOllamaHost(stateRoot) ?? OLLAMA_LOCALHOST; +} + /** Keep Windows-host Ollama requests in Docker Desktop's verified network context. */ export function getOllamaApiCommand( curlArgs: readonly string[], @@ -790,7 +839,9 @@ export function getLocalProviderHealthCheck(provider: string): string[] | null { endpoint, ]; } - return endpoint ? ["curl", ...buildValidatedCurlCommandArgs(["-sf", endpoint])] : null; + if (!endpoint) return null; + const curlArgs = buildValidatedCurlCommandArgs(["-sf", endpoint]); + return provider === "ollama-local" ? getOllamaApiCommand(curlArgs) : ["curl", ...curlArgs]; } /** @@ -1172,9 +1223,8 @@ export function probeOllamaEndpointInventory( ): string[] | null { const capture = runCaptureImpl ?? runCapture; const body = capture( - [ - "curl", - ...buildValidatedCurlCommandArgs([ + getOllamaApiCommand( + buildValidatedCurlCommandArgs([ "-sf", "--connect-timeout", "3", @@ -1182,7 +1232,8 @@ export function probeOllamaEndpointInventory( "5", `http://${host}:${OLLAMA_PORT}/api/tags`, ]), - ], + host, + ), { ignoreError: true }, ); return parseOllamaModelInventory(body); diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index b1e101423d9..fb7ebec65cd 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -31,6 +31,7 @@ const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./ const { getBootstrapOllamaModelOptions, getOllamaApiCommand, + getOllamaHostForCleanup, getOllamaModelOptions, getOllamaWarmupCommand, getResolvedOllamaHost, @@ -1362,6 +1363,7 @@ export type OllamaUnloadResult = { type OllamaUnloadOptions = { readonly getResolvedOllamaHost?: typeof getResolvedOllamaHost; + readonly ollamaHostStateRoot?: string; readonly maxAttempts?: number; readonly sleep?: (milliseconds: number) => void; readonly spawnSync?: typeof spawnSync; @@ -1481,7 +1483,9 @@ function unloadOllamaModels( onlyModels?: readonly string[], options: OllamaUnloadOptions = {}, ): OllamaUnloadResult { - const releaseHost = (options.getResolvedOllamaHost ?? getResolvedOllamaHost)(); + const releaseHost = options.getResolvedOllamaHost + ? options.getResolvedOllamaHost() + : getOllamaHostForCleanup(options.ollamaHostStateRoot); const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost); const spawnSyncImpl = options.spawnSync ?? spawnSync; const sleepImpl = options.sleep ?? defaultReleaseSleep; diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index d560bf7d292..27210e96624 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -99,11 +99,19 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("records the route when the sandbox endpoint serves the model", async () => { const upsertProvider = vi.fn(() => ({ ok: true })); + const persistResolvedOllamaHost = vi.fn(); await expect( setupOllamaLocalInference( { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, - deps({ upsertProvider }), + deps({ + upsertProvider, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), ), ).resolves.toEqual({ done: false }); @@ -114,5 +122,31 @@ describe("Ollama local provider sandbox-facing model gate", () => { "http://host.openshell.internal:11434/v1", { [CREDENTIAL_ENV]: "ollama" }, ); + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("fails before recording the provider when the cleanup route cannot be persisted", async () => { + const upsertProvider = vi.fn(() => ({ ok: true })); + const error = vi.fn(); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + upsertProvider, + error, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => { + throw new Error("state path is unsafe"); + }, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(upsertProvider).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("state path is unsafe")); }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index e2552a7aa0c..3dbfdb2e9a2 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -78,6 +78,16 @@ export async function setupOllamaLocalInference( error(` ${sandboxModel.message}`); return exitProcess(1); } + try { + localInference.persistResolvedOllamaHost?.(); + } catch (persistError) { + error( + ` Could not record the selected local Ollama route for later stop/destroy cleanup: ${ + persistError instanceof Error ? persistError.message : String(persistError) + }`, + ); + return exitProcess(1); + } const baseUrl = getLocalProviderBaseUrl(provider); let ollamaCredential = "ollama"; if (frontOllamaWithProxy) { diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index ce6d3f6cf48..a8292897e51 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -255,6 +255,7 @@ export type OllamaDeps = CommonDeps & { allowToolsIncompatible: boolean, ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; + persistResolvedOllamaHost?(): void; }; /** Exact provider-owned proof used instead of legacy host warmup/probes. */ providerOwnedInferenceProof?: { diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index a5b9101cb2d..68622f6a08b 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -615,6 +615,27 @@ describe("stopAll", () => { expect(cleanup).toHaveBeenCalledOnce(); expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(stoppedCallOrder ?? 0); }); + + it("returns and reports Ollama cleanup failure instead of suppressing it", () => { + const failure = { + ok: false as const, + outcome: "discovery-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: [], + discoveries: [], + requests: [], + message: "could not connect", + }; + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + const result = stopAll({ pidDir, unloadOllamaModels: () => failure }); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(result).toBe(failure); + expect(output).toContain("Ollama model cleanup failed at http://host.docker.internal:11434"); + expect(output).toContain("saved local route was retained"); + }); }); // #6212: after cloudflared yields a public URL, startAll must register that diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 1f9e902b7f0..b06ed76c6ae 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -19,7 +19,10 @@ import { renderBox } from "../cli/banner"; import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding"; import { isObjectRecord } from "../core/json-types"; import { DASHBOARD_PORT } from "../core/ports"; -import { unloadOllamaModels as unloadDefaultOllamaModels } from "../inference/ollama/proxy"; +import { + unloadOllamaModels as unloadDefaultOllamaModels, + type OllamaUnloadResult, +} from "../inference/ollama/proxy"; import { buildSubprocessEnv } from "../subprocess-env"; import * as agentForwardStop from "./agent-forward-stop"; import { registerTunnelOrigin } from "./allowed-origins"; @@ -45,7 +48,7 @@ export interface ServiceOptions { /** Injectable process operations (identity + signalling) for tests. */ processControl?: ProcessControl; /** Injectable Ollama model cleanup for tests. */ - unloadOllamaModels?: () => void; + unloadOllamaModels?: () => OllamaUnloadResult | void; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ cloudflareTunnelToken?: string; /** Also release the managed host gateway port (legacy full-stop only). */ @@ -491,7 +494,7 @@ export function showStatus(opts: ServiceOptions = {}): void { } } -export function stopAll(opts: ServiceOptions = {}): void { +export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { // Resolve the target sandbox once and reuse it for in-sandbox and host-side cleanup. const rawSandboxName = opts.sandboxName ?? @@ -522,11 +525,22 @@ export function stopAll(opts: ServiceOptions = {}): void { warn("Hint: run 'nemoclaw stop' with a registered sandbox or set NEMOCLAW_SANDBOX_NAME."); } + let ollamaCleanup: OllamaUnloadResult | undefined; + let ollamaCleanupError: unknown; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; - unloadOllamaModels(); - } catch { - /* best-effort */ + const cleanup = unloadOllamaModels(); + if (cleanup) ollamaCleanup = cleanup; + if (cleanup && !cleanup.ok) { + warn( + `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; repair Ollama and retry this command.`, + ); + } + } catch (error) { + ollamaCleanupError = error; + warn( + `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, + ); } // Stop host-side services only when their state directory is explicit or @@ -560,15 +574,19 @@ export function stopAll(opts: ServiceOptions = {}): void { "Hint: rerun with NEMOCLAW_GATEWAY_PORT= to release that gateway, or 'openshell gateway list' to find it.", ); info("Host services stopped; managed gateway not released."); - return; + if (ollamaCleanupError) throw ollamaCleanupError; + return ollamaCleanup; } if (gatewayOutcome === "unconfirmed") { info("Host services stopped; managed gateway release was not confirmed."); - return; + if (ollamaCleanupError) throw ollamaCleanupError; + return ollamaCleanup; } info("All services stopped."); + if (ollamaCleanupError) throw ollamaCleanupError; + return ollamaCleanup; } /** diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index 528ece0b477..0723699eb95 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -2,8 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 import type { SpawnSyncReturns } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { + OLLAMA_HOST_DOCKER_INTERNAL, + persistResolvedOllamaHost, + resetOllamaHostCache, +} from "../../../src/lib/inference/local.js"; import { unloadOllamaModels as unloadOllamaModelsImpl } from "../../../src/lib/inference/ollama/proxy.js"; type SpawnCall = { command: string; args: readonly string[] }; @@ -80,6 +88,41 @@ function unloadOf(model: string) { } describe("Ollama GPU cleanup", () => { + it("restores the persisted Windows-host transport after the process cache is cleared", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-cleanup-route-")); + const calls: SpawnCall[] = []; + const respond = respondWithLoadedModels("llama3.2:1b"); + const spawnSync = ((command: string, args: readonly string[]) => { + const call = { command, args }; + calls.push(call); + return respond(call); + }) as SpawnSync; + + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + resetOllamaHostCache(); + const result = unloadOllamaModelsImpl(["llama3.2:1b"], { + ollamaHostStateRoot: stateRoot, + sleep: () => {}, + spawnSync, + }); + + expect(result).toMatchObject({ + ok: true, + outcome: "released", + endpoint: "http://host.docker.internal:11434", + }); + expect(calls).toHaveLength(3); + calls.forEach(({ command, args }) => { + expect(command).toBe("docker"); + expect(args).toEqual(expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"])); + }); + } finally { + resetOllamaHostCache(); + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("uses the resolved local Ollama host for discovery, release, and verification (#10074)", async () => { await withMockedSpawnSync( respondWithLoadedModels("llama3.2:1b"), @@ -93,17 +136,13 @@ describe("Ollama GPU cleanup", () => { }); const dockerCalls = calls.filter(({ command }) => command === "docker"); expect(dockerCalls).toHaveLength(3); - expect( - dockerCalls.map(({ args }) => args.at(-1)), - ).toEqual([ + expect(dockerCalls.map(({ args }) => args.at(-1))).toEqual([ "http://host.docker.internal:11434/api/ps", "http://host.docker.internal:11434/api/generate", "http://host.docker.internal:11434/api/ps", ]); dockerCalls.forEach(({ args }) => { - expect(args).toEqual( - expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"]), - ); + expect(args).toEqual(expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"])); }); }, "host.docker.internal", diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 2bdb5ab6e5c..9bba77fbc00 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -69,6 +69,25 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.unloadCalls).toBe(0); }); + it("does not suppress a failed Windows-host unload from stopAll during destroy", () => { + const harness = buildDeps({ provider: "ollama-local" }); + vi.mocked(harness.deps.stopAll).mockReturnValue({ + ok: false, + outcome: "discovery-failed", + endpoint: "http://host.docker.internal:11434", + selectedModels: [], + discoveries: [], + requests: [], + message: "could not connect", + }); + + expect(() => + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps), + ).toThrow(/host\.docker\.internal:11434.*retry destroy/); + + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + }); + it("calls unloadOllamaModels() exactly once for an Ollama sandbox when stopHostServices=false", () => { const harness = buildDeps({ provider: "ollama-local" }); From 057ebc5e4bd493c94ee2a2534b85bdf0e772f56f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 31 Aug 2026 22:33:29 -0700 Subject: [PATCH 04/47] fix(inference): preserve destroy cleanup contract Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/destroy.ts | 31 +++++-------------- src/lib/tunnel/services.test.ts | 5 ++- src/lib/tunnel/services.ts | 14 ++------- .../destroy-cleanup-sandbox-services.test.ts | 19 ------------ 4 files changed, 13 insertions(+), 56 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 0e77b653d4e..2521f0c6ed8 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -22,7 +22,6 @@ import { revokeHttpsPinRuntimeAdapterRoute, } from "../../inference/https-pin-runtime-adapter"; import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup"; -import type { OllamaUnloadResult } from "../../inference/ollama/proxy"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, normalizeRuntimeProviderIdentity, @@ -166,8 +165,8 @@ type RunOpenshell = (args: string[], opts?: Record) => { status export type CleanupSandboxServicesDeps = { getSandbox?: typeof registry.getSandbox; - stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void; - unloadOllamaModels?: () => OllamaUnloadResult | void; + stopAll?: (opts: { sandboxName: string }) => void; + unloadOllamaModels?: () => void; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; @@ -228,17 +227,17 @@ export function cleanupSandboxServices( deps.stopAll ?? ((opts: { sandboxName: string }) => { const services = require("../../tunnel/services") as { - stopAll: (opts: { sandboxName: string }) => OllamaUnloadResult | void; + stopAll: (opts: { sandboxName: string }) => void; }; - return services.stopAll(opts); + services.stopAll(opts); }); const unloadOllamaModels = deps.unloadOllamaModels ?? (() => { const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as { - unloadOllamaModels: () => OllamaUnloadResult; + unloadOllamaModels: () => void; }; - return unload(); + unload(); }); const runOpenshell = deps.runOpenshell ?? @@ -290,28 +289,14 @@ export function cleanupSandboxServices( if (stopHostServices) { // `stopAll()` already runs `unloadOllamaModels()` unconditionally — // see src/lib/tunnel/services.ts. Don't double-call here. - const cleanup = stopAll({ sandboxName: validatedSandboxName }); - if (cleanup && !cleanup.ok) { - throw new Error( - `Sandbox host services stopped, but Ollama model cleanup failed at ${cleanup.endpoint} ` + - `(${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was ` + - "retained; repair Ollama and retry destroy.", - ); - } + stopAll({ sandboxName: validatedSandboxName }); } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this // branch a single-sandbox destroy would leave models loaded on the GPU. const sb = getSandbox(validatedSandboxName); if (sb?.provider?.includes("ollama")) { - const cleanup = unloadOllamaModels(); - if (cleanup && !cleanup.ok) { - throw new Error( - `Sandbox resources were removed, but Ollama model cleanup failed at ${cleanup.endpoint} ` + - `(${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was ` + - "retained; repair Ollama and retry destroy.", - ); - } + unloadOllamaModels(); } } diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index 68622f6a08b..c361d0607e2 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -616,7 +616,7 @@ describe("stopAll", () => { expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(stoppedCallOrder ?? 0); }); - it("returns and reports Ollama cleanup failure instead of suppressing it", () => { + it("reports Ollama cleanup failure and retains its recovery route", () => { const failure = { ok: false as const, outcome: "discovery-failed" as const, @@ -628,11 +628,10 @@ describe("stopAll", () => { }; const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const result = stopAll({ pidDir, unloadOllamaModels: () => failure }); + stopAll({ pidDir, unloadOllamaModels: () => failure }); const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); logSpy.mockRestore(); - expect(result).toBe(failure); expect(output).toContain("Ollama model cleanup failed at http://host.docker.internal:11434"); expect(output).toContain("saved local route was retained"); }); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index b06ed76c6ae..25559492c1c 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -494,7 +494,7 @@ export function showStatus(opts: ServiceOptions = {}): void { } } -export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { +export function stopAll(opts: ServiceOptions = {}): void { // Resolve the target sandbox once and reuse it for in-sandbox and host-side cleanup. const rawSandboxName = opts.sandboxName ?? @@ -525,19 +525,15 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { warn("Hint: run 'nemoclaw stop' with a registered sandbox or set NEMOCLAW_SANDBOX_NAME."); } - let ollamaCleanup: OllamaUnloadResult | undefined; - let ollamaCleanupError: unknown; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; const cleanup = unloadOllamaModels(); - if (cleanup) ollamaCleanup = cleanup; if (cleanup && !cleanup.ok) { warn( `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; repair Ollama and retry this command.`, ); } } catch (error) { - ollamaCleanupError = error; warn( `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, ); @@ -574,19 +570,15 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { "Hint: rerun with NEMOCLAW_GATEWAY_PORT= to release that gateway, or 'openshell gateway list' to find it.", ); info("Host services stopped; managed gateway not released."); - if (ollamaCleanupError) throw ollamaCleanupError; - return ollamaCleanup; + return; } if (gatewayOutcome === "unconfirmed") { info("Host services stopped; managed gateway release was not confirmed."); - if (ollamaCleanupError) throw ollamaCleanupError; - return ollamaCleanup; + return; } info("All services stopped."); - if (ollamaCleanupError) throw ollamaCleanupError; - return ollamaCleanup; } /** diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 9bba77fbc00..2bdb5ab6e5c 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -69,25 +69,6 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.unloadCalls).toBe(0); }); - it("does not suppress a failed Windows-host unload from stopAll during destroy", () => { - const harness = buildDeps({ provider: "ollama-local" }); - vi.mocked(harness.deps.stopAll).mockReturnValue({ - ok: false, - outcome: "discovery-failed", - endpoint: "http://host.docker.internal:11434", - selectedModels: [], - discoveries: [], - requests: [], - message: "could not connect", - }); - - expect(() => - cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps), - ).toThrow(/host\.docker\.internal:11434.*retry destroy/); - - expect(harness.deps.rmSync).not.toHaveBeenCalled(); - }); - it("calls unloadOllamaModels() exactly once for an Ollama sandbox when stopHostServices=false", () => { const harness = buildDeps({ provider: "ollama-local" }); From e4cc146fa4479b657ebd18256f238a2012b2482e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 31 Aug 2026 23:15:30 -0700 Subject: [PATCH 05/47] refactor(inference): finalize Ollama route ownership --- .../sandbox/agent/ollama-restart-recovery.ts | 26 +++-------- src/lib/inference/local.ts | 7 ++- .../inference-providers/ollama-local.test.ts | 44 ++++++++++++++++++- .../inference-providers/ollama-local.ts | 20 ++++----- 4 files changed, 64 insertions(+), 33 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index aaaea51133d..6591b10e2cf 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -18,20 +18,21 @@ import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args" import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, + createOllamaApiCapture, getOllamaApiCommand, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, probeOllamaEndpointInventory, + type RunCaptureFn, type RunCaptureExFn, } from "../../../inference/local"; import { type OllamaRuntimeModelStatus, - type OllamaRuntimeRunCaptureFn, probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; -import { runCapture, runCaptureEx } from "../../../runner"; +import { runCaptureEx } from "../../../runner"; export interface OllamaRestartRecoveryRoute { provider?: string | null; @@ -43,15 +44,12 @@ export interface OllamaRestartRecoveryDeps { probeRuntimeModelStatus?: ( model: string, getOllamaHost: () => string, - runCaptureImpl?: OllamaRuntimeRunCaptureFn, + runCaptureImpl?: RunCaptureFn, ) => OllamaRuntimeModelStatus; - probeModelInventory?: ( - host: string, - runCaptureImpl?: OllamaRuntimeRunCaptureFn, - ) => string[] | null; + probeModelInventory?: (host: string, runCaptureImpl?: RunCaptureFn) => string[] | null; runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; - runCaptureImpl?: OllamaRuntimeRunCaptureFn; + runCaptureImpl?: RunCaptureFn; } export type OllamaRestartRecoveryFailureReason = @@ -168,16 +166,6 @@ function buildWarmCommand(model: string, hostname: string): string[] { ); } -function createRawOllamaCapture( - hostname: string, - capture: OllamaRuntimeRunCaptureFn, -): OllamaRuntimeRunCaptureFn { - return (command, options) => { - const [executable, ...args] = command; - return capture(executable === "curl" ? getOllamaApiCommand(args, hostname) : command, options); - }; -} - function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" { try { const parsed = JSON.parse(stdout) as { @@ -220,7 +208,7 @@ export function maybeWarmOllamaAfterDaemonRestart( const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; - const rawCapture = createRawOllamaCapture(rawHost, deps.runCaptureImpl ?? runCapture); + const rawCapture = createOllamaApiCapture(deps.runCaptureImpl, rawHost); let status: OllamaRuntimeModelStatus; try { status = probe(model, () => rawHost, rawCapture); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index e183d344e37..c2b34296f8d 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -254,11 +254,14 @@ export function getOllamaApiCommand( : ["curl", ...curlArgs]; } -function createOllamaApiCapture(runCaptureImpl?: RunCaptureFn): RunCaptureFn { +export function createOllamaApiCapture( + runCaptureImpl?: RunCaptureFn, + host: string = getResolvedOllamaHost(), +): RunCaptureFn { const capture = runCaptureImpl ?? runCapture; return (command, options) => { const [executable, ...args] = command; - return capture(executable === "curl" ? getOllamaApiCommand(args) : command, options); + return capture(executable === "curl" ? getOllamaApiCommand(args, host) : command, options); }; } diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index 27210e96624..27c006e07ac 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -125,7 +125,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); }); - it("fails before recording the provider when the cleanup route cannot be persisted", async () => { + it("fails setup when the accepted cleanup route cannot be persisted", async () => { const upsertProvider = vi.fn(() => ({ ok: true })); const error = vi.fn(); @@ -146,7 +146,47 @@ describe("Ollama local provider sandbox-facing model gate", () => { ), ).rejects.toThrow("exit 1"); - expect(upsertProvider).not.toHaveBeenCalled(); + expect(upsertProvider).toHaveBeenCalledOnce(); expect(error).toHaveBeenCalledWith(expect.stringContaining("state path is unsafe")); }); + + it("does not persist a route when provider registration fails", async () => { + const persistResolvedOllamaHost = vi.fn(); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + upsertProvider: () => ({ ok: false, status: 1, message: "provider rejected" }), + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(persistResolvedOllamaHost).not.toHaveBeenCalled(); + }); + + it("does not persist a route when route application requests reselection", async () => { + const persistResolvedOllamaHost = vi.fn(); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + applyLocalInferenceRoute: async () => true, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).resolves.toEqual({ done: true, result: { retry: "selection" } }); + + expect(persistResolvedOllamaHost).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index 3dbfdb2e9a2..82a9d429464 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -78,16 +78,6 @@ export async function setupOllamaLocalInference( error(` ${sandboxModel.message}`); return exitProcess(1); } - try { - localInference.persistResolvedOllamaHost?.(); - } catch (persistError) { - error( - ` Could not record the selected local Ollama route for later stop/destroy cleanup: ${ - persistError instanceof Error ? persistError.message : String(persistError) - }`, - ); - return exitProcess(1); - } const baseUrl = getLocalProviderBaseUrl(provider); let ollamaCredential = "ollama"; if (frontOllamaWithProxy) { @@ -144,6 +134,16 @@ export async function setupOllamaLocalInference( return exitProcess(1); } } + try { + localInference.persistResolvedOllamaHost?.(); + } catch (persistError) { + error( + ` Could not record the accepted local Ollama route for later stop/destroy cleanup: ${ + persistError instanceof Error ? persistError.message : String(persistError) + }`, + ); + return exitProcess(1); + } // Do not mutate ~/.nemoclaw/credentials.json here: local Ollama now uses // OLLAMA_PROXY_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available // to unrelated OpenAI-backed sandboxes. From a0059bea08feb85e84b1a15f08d7eba9d9f3e522 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 01:04:45 -0700 Subject: [PATCH 06/47] fix(inference): restore accepted Ollama route --- .../local-windows-ollama-transport.test.ts | 97 ++++++++++++++++++- src/lib/inference/local.ts | 63 +++++++++++- .../inference-providers/ollama-local.test.ts | 20 ++-- .../inference-providers/ollama-local.ts | 65 +++++++++---- src/lib/onboard/inference-providers/types.ts | 2 +- 5 files changed, 214 insertions(+), 33 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 30e16af54cd..e1d327fc865 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -9,14 +9,17 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { applyOllamaRuntimeContextWindow, CONTAINER_REACHABILITY_IMAGE, + findReachableOllamaHost, getLocalProviderHealthCheck, getOllamaHostForCleanup, getOllamaModelOptions, getOllamaProbeCommand, + getResolvedOllamaHost, getOllamaWarmupRequestCommand, OLLAMA_HOST_DOCKER_INTERNAL, loadPersistedOllamaHost, persistResolvedOllamaHost, + probeLocalProviderHealth, probeOllamaModelCapabilities, resetOllamaHostCache, resetOllamaRuntimeContextWindowAutoState, @@ -44,6 +47,98 @@ describe("Windows-host Ollama transport", () => { } }); + it("restores the prior receipt when staged provider setup rolls back", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-rollback-")); + try { + persistResolvedOllamaHost("127.0.0.1", stateRoot); + const rollback = persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + + rollback(); + + expect(loadPersistedOllamaHost(stateRoot)).toBe("127.0.0.1"); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("restores the persisted route before fresh-process connect discovery", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-connect-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + resetOllamaHostCache(); + const capture = vi.fn(() => { + throw new Error("fresh-process connect must not probe WSL loopback"); + }); + + expect(findReachableOllamaHost(capture, {}, stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + expect(capture).not.toHaveBeenCalled(); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("rejects an untrusted persisted host", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-invalid-")); + try { + writeFileSync( + join(stateRoot, "ollama-host.json"), + JSON.stringify({ schemaVersion: 1, host: "example.com" }), + ); + resetOllamaHostCache(); + + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + expect(getResolvedOllamaHost(stateRoot)).toBe("127.0.0.1"); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("probes persisted Windows-host health through Docker Desktop", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-health-")); + const calls: Array<{ command: string; args: readonly string[] }> = []; + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + resetOllamaHostCache(); + expect(getResolvedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + + const result = probeLocalProviderHealth("ollama-local", { + loadOllamaProxyTokenImpl: () => null, + ollamaSpawnSyncImpl: (command, args) => { + calls.push({ command, args }); + const statusOutput = args[args.indexOf("-w") + 1].replace("%{http_code}", "200"); + const stdout = `${JSON.stringify({ models: [] })}${statusOutput}`; + return { + pid: 1, + output: ["", stdout, ""], + stdout, + stderr: "", + status: 0, + signal: null, + }; + }, + }); + + expect(result).toMatchObject({ + ok: true, + endpoint: "http://host.docker.internal:11434/api/tags", + }); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ command: "docker" }); + expect(calls[0]?.args).toEqual( + expect.arrayContaining([ + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/tags", + ]), + ); + } finally { + resetOllamaHostCache(); + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("reads the model inventory through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const capture = vi.fn(() => JSON.stringify({ models: [{ name: "qwen3.5:9b" }] })); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index c2b34296f8d..31e95e5cd7a 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -6,6 +6,7 @@ * health checks, and command generators for vLLM and Ollama. */ +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import nodePath from "node:path"; @@ -32,6 +33,8 @@ import { buildSubprocessEnv } from "../subprocess-env"; import { readLocalAdapterJsonFile, + LOCAL_ADAPTER_HEALTH_MAX_RESPONSE_BYTES, + removeLocalAdapterFile, resolveSharedLocalAdapterStateRoot, writeLocalAdapterJsonFile, } from "./local-adapter-lifecycle"; @@ -181,8 +184,14 @@ function ollamaCandidateHosts(wslDetection: WslDetectionOptions = {}): string[] export function findReachableOllamaHost( runCaptureImpl?: RunCaptureFn, wslDetection: WslDetectionOptions = {}, + stateRoot: string = resolveSharedLocalAdapterStateRoot(), ): string | null { if (_resolvedOllamaHost !== null) return _resolvedOllamaHost; + const persistedHost = loadPersistedOllamaHost(stateRoot); + if (persistedHost) { + _resolvedOllamaHost = persistedHost; + return persistedHost; + } const capture = runCaptureImpl ?? runCapture; for (const host of ollamaCandidateHosts(wslDetection)) { // Explicit timeouts: a blackholed host (e.g., firewalled host.docker.internal) @@ -211,22 +220,39 @@ export function findReachableOllamaHost( // Returns the resolved host if a probe has succeeded, otherwise OLLAMA_LOCALHOST. // Used by URL-builder helpers that need a string and don't want to re-probe. -export function getResolvedOllamaHost(): string { - return _resolvedOllamaHost ?? OLLAMA_LOCALHOST; +export function getResolvedOllamaHost( + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): string { + if (_resolvedOllamaHost) return _resolvedOllamaHost; + const persistedHost = loadPersistedOllamaHost(stateRoot); + if (persistedHost) _resolvedOllamaHost = persistedHost; + return persistedHost ?? OLLAMA_LOCALHOST; } /** Persist the accepted local Ollama route for later CLI processes. */ export function persistResolvedOllamaHost( host: string = getResolvedOllamaHost(), stateRoot: string = resolveSharedLocalAdapterStateRoot(), -): void { +): () => void { if (!isSupportedOllamaHost(host)) { throw new Error(`Refusing to persist unexpected Ollama host: ${host}`); } - writeLocalAdapterJsonFile(ollamaHostReceiptPath(stateRoot), { + const receiptPath = ollamaHostReceiptPath(stateRoot); + const previousHost = loadPersistedOllamaHost(stateRoot); + writeLocalAdapterJsonFile(receiptPath, { schemaVersion: 1, host, } satisfies OllamaHostReceipt); + return () => { + if (previousHost) { + writeLocalAdapterJsonFile(receiptPath, { + schemaVersion: 1, + host: previousHost, + } satisfies OllamaHostReceipt); + } else { + removeLocalAdapterFile(receiptPath); + } + }; } /** Read only the two fixed local Ollama routes NemoClaw can establish. */ @@ -355,6 +381,8 @@ export interface LocalProviderHealthProbeOptions { /** Configured runtime model that must be present in the provider inventory. */ model?: string | null; runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; + /** Executes the translated Windows-host Docker probe. Injectable for transport tests. */ + ollamaSpawnSyncImpl?: NonNullable; /** * Lets callers that perform their own Ollama auth-proxy check avoid the * legacy inline proxy subprobe. The inline subprobe is retained for status @@ -392,6 +420,24 @@ function runLocalCurlProbe(argv: string[], opts: CurlProbeOptions = {}): CurlPro return runCurlProbe(argv, { ...opts, env: buildSubprocessEnv(), replaceEnv: true }); } +function runOllamaLocalCurlProbe( + argv: string[], + host: string, + opts: CurlProbeOptions = {}, + spawnSyncImpl: NonNullable = spawnSync, +): CurlProbeResult { + return runCurlProbe(argv, { + ...opts, + env: buildSubprocessEnv(), + maxResponseBytes: opts.maxResponseBytes ?? LOCAL_ADAPTER_HEALTH_MAX_RESPONSE_BYTES, + replaceEnv: true, + spawnSyncImpl: (_command, args, spawnOptions) => { + const [executable, ...translatedArgs] = getOllamaApiCommand(args, host); + return spawnSyncImpl(executable, translatedArgs, spawnOptions); + }, + }); +} + export interface VllmModelsProbeOptions { runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; } @@ -1051,7 +1097,14 @@ export function probeLocalProviderHealth( : getLocalProviderHealthEndpoint(provider); if (!endpoint) return null; - const runCurlProbeImpl = options.runCurlProbeImpl ?? runLocalCurlProbe; + const resolvedOllamaHost = + provider === "ollama-local" ? getResolvedOllamaHost() : OLLAMA_LOCALHOST; + const runCurlProbeImpl = + options.runCurlProbeImpl ?? + (provider === "ollama-local" && resolvedOllamaHost === OLLAMA_HOST_DOCKER_INTERNAL + ? (argv: string[], opts?: CurlProbeOptions) => + runOllamaLocalCurlProbe(argv, resolvedOllamaHost, opts, options.ollamaSpawnSyncImpl) + : runLocalCurlProbe); let result: CurlProbeResult; if (managedBinding) { result = probeVllmModels(managedValidationBaseUrl!, managedBinding.apiKey, { diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index 27c006e07ac..ad3d4a177a4 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -125,7 +125,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); }); - it("fails setup when the accepted cleanup route cannot be persisted", async () => { + it("fails before provider registration when the cleanup route cannot be staged", async () => { const upsertProvider = vi.fn(() => ({ ok: true })); const error = vi.fn(); @@ -146,12 +146,13 @@ describe("Ollama local provider sandbox-facing model gate", () => { ), ).rejects.toThrow("exit 1"); - expect(upsertProvider).toHaveBeenCalledOnce(); + expect(upsertProvider).not.toHaveBeenCalled(); expect(error).toHaveBeenCalledWith(expect.stringContaining("state path is unsafe")); }); - it("does not persist a route when provider registration fails", async () => { - const persistResolvedOllamaHost = vi.fn(); + it("restores the prior cleanup route when provider registration fails", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); await expect( setupOllamaLocalInference( @@ -167,11 +168,13 @@ describe("Ollama local provider sandbox-facing model gate", () => { ), ).rejects.toThrow("exit 1"); - expect(persistResolvedOllamaHost).not.toHaveBeenCalled(); + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); }); - it("does not persist a route when route application requests reselection", async () => { - const persistResolvedOllamaHost = vi.fn(); + it("restores the prior cleanup route when route application requests reselection", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); await expect( setupOllamaLocalInference( @@ -187,6 +190,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { ), ).resolves.toEqual({ done: true, result: { retry: "selection" } }); - expect(persistResolvedOllamaHost).not.toHaveBeenCalled(); + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index 82a9d429464..404e44e7a75 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -95,22 +95,61 @@ export async function setupOllamaLocalInference( await persistAndProbeOllamaProxy(proxyToken); } } + let rollbackPersistedOllamaHost: (() => void) | undefined; + try { + rollbackPersistedOllamaHost = localInference.persistResolvedOllamaHost?.() ?? undefined; + } catch (persistError) { + error( + ` Could not stage the selected local Ollama route for later stop/destroy cleanup: ${ + persistError instanceof Error ? persistError.message : String(persistError) + }`, + ); + return exitProcess(1); + } + const rollbackCleanupRoute = (): boolean => { + try { + rollbackPersistedOllamaHost?.(); + return true; + } catch (rollbackError) { + error( + ` Could not restore the prior local Ollama cleanup route: ${ + rollbackError instanceof Error ? rollbackError.message : String(rollbackError) + }`, + ); + return false; + } + }; // Use a dedicated internal credential env (NEMOCLAW_OLLAMA_PROXY_TOKEN) // so the gateway never reads the user's host OPENAI_API_KEY for local // Ollama. GH #2519: a stale host OPENAI_API_KEY was leaking into the // inference path and producing 401s. - const providerResult = upsertProvider( - "ollama-local", - "openai", - OLLAMA_PROXY_CREDENTIAL_ENV, - baseUrl, - { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, - ); + let providerResult: ReturnType; + try { + providerResult = upsertProvider( + "ollama-local", + "openai", + OLLAMA_PROXY_CREDENTIAL_ENV, + baseUrl, + { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, + ); + } catch (providerError) { + rollbackCleanupRoute(); + throw providerError; + } if (!providerResult.ok) { + rollbackCleanupRoute(); error(` ${providerResult.message}`); return exitProcess(providerResult.status || 1); } - if (await applyLocalInferenceRoute("ollama-local", model)) { + let retrySelection: boolean; + try { + retrySelection = await applyLocalInferenceRoute("ollama-local", model); + } catch (routeError) { + rollbackCleanupRoute(); + throw routeError; + } + if (retrySelection) { + if (!rollbackCleanupRoute()) return exitProcess(1); return { done: true, result: { retry: "selection" } }; } if (providerOwnedInferenceProof) { @@ -134,16 +173,6 @@ export async function setupOllamaLocalInference( return exitProcess(1); } } - try { - localInference.persistResolvedOllamaHost?.(); - } catch (persistError) { - error( - ` Could not record the accepted local Ollama route for later stop/destroy cleanup: ${ - persistError instanceof Error ? persistError.message : String(persistError) - }`, - ); - return exitProcess(1); - } // Do not mutate ~/.nemoclaw/credentials.json here: local Ollama now uses // OLLAMA_PROXY_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available // to unrelated OpenAI-backed sandboxes. diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index a8292897e51..1cadb39c3c7 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -255,7 +255,7 @@ export type OllamaDeps = CommonDeps & { allowToolsIncompatible: boolean, ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; - persistResolvedOllamaHost?(): void; + persistResolvedOllamaHost?(): (() => void) | void; }; /** Exact provider-owned proof used instead of legacy host warmup/probes. */ providerOwnedInferenceProof?: { From 4fc9f3d1b8bd730b127a7e7a866d2ee4690fc7d9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 01:52:16 -0700 Subject: [PATCH 07/47] fix(inference): complete Ollama route recovery --- .../agent/ollama-restart-recovery.test.ts | 53 +++++++++++++++-- .../sandbox/agent/ollama-restart-recovery.ts | 57 +++++++++++++++++-- .../agent/passthrough-ollama-recovery.test.ts | 14 ++++- .../agent/passthrough-ollama-recovery.ts | 4 +- .../local-windows-ollama-transport.test.ts | 26 +++++++-- src/lib/inference/local.test.ts | 4 +- src/lib/inference/local.ts | 34 +++++------ .../inference-providers/ollama-local.test.ts | 24 ++++++++ 8 files changed, 175 insertions(+), 41 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 8939234c5cb..fd6d45adec3 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -186,7 +186,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: true, reason: "timeout" }); + ).toEqual({ + kind: "warmed", + ok: false, + timedOut: true, + reason: "timeout", + endpoint: "http://127.0.0.1:11434", + detail: "warm-up exceeded 300 seconds", + }); }); it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { @@ -203,7 +210,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); + ).toMatchObject({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "ollama-error", + endpoint: "http://127.0.0.1:11434", + detail: expect.stringContaining("model not found"), + }); }); it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", () => { @@ -249,7 +263,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); + ).toMatchObject({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "ollama-error", + endpoint: "http://127.0.0.1:11434", + detail: expect.stringContaining("runner stopped unexpectedly"), + }); }); it("accepts a completed thinking-only response from a thinking model", () => { @@ -282,7 +303,13 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "invalid-response" }); + ).toMatchObject({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "invalid-response", + endpoint: "http://127.0.0.1:11434", + }); }); it("reports a non-zero warm command exit", () => { @@ -294,7 +321,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }); + ).toEqual({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "command-failed", + endpoint: "http://127.0.0.1:11434", + detail: "warm-up exited 7", + }); }); it("reports a warm process spawn failure without throwing", () => { @@ -307,6 +341,13 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect( maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local", model: "qwen3.6:35b" }, deps), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }); + ).toEqual({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "spawn-failed", + endpoint: "http://127.0.0.1:11434", + detail: "spawn failed", + }); }); }); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 6591b10e2cf..7b800076867 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -68,6 +68,8 @@ export type OllamaRestartRecoveryResult = ok: false; timedOut: boolean; reason: OllamaRestartRecoveryFailureReason; + endpoint: string; + detail: string; }; export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; @@ -188,6 +190,13 @@ function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid- } } +function boundedWarmFailureDetail(value: unknown, fallback: string): string { + const detail = String(value ?? "") + .replace(/\s+/g, " ") + .trim(); + return (detail || fallback).slice(0, 300); +} + /** * Warm a registered local Ollama model only when `/api/ps` proves that the * daemon is reachable and the selected model is no longer loaded. @@ -207,6 +216,7 @@ export function maybeWarmOllamaAfterDaemonRestart( const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); + const rawEndpoint = `http://${rawHost}:${OLLAMA_PORT}`; const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; const rawCapture = createOllamaApiCapture(deps.runCaptureImpl, rawHost); let status: OllamaRuntimeModelStatus; @@ -226,10 +236,30 @@ export function maybeWarmOllamaAfterDaemonRestart( try { const result = captureEx(buildWarmCommand(model, rawHost)); if (result.timedOut) { - return { kind: "warmed", ok: false, timedOut: true, reason: "timeout" }; + return { + kind: "warmed", + ok: false, + timedOut: true, + reason: "timeout", + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail( + result.stderr, + `warm-up exceeded ${OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS} seconds`, + ), + }; } if (result.exitCode !== 0) { - return { kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }; + return { + kind: "warmed", + ok: false, + timedOut: false, + reason: "command-failed", + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail( + result.stderr || result.stdout, + `warm-up exited ${String(result.exitCode)}`, + ), + }; } const response = validateWarmResponse(result.stdout); // An Ollama error can mean a broken runner or a daemon that simply does not @@ -250,10 +280,27 @@ export function maybeWarmOllamaAfterDaemonRestart( } } if (response !== "ok") { - return { kind: "warmed", ok: false, timedOut: false, reason: response }; + return { + kind: "warmed", + ok: false, + timedOut: false, + reason: response, + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail(result.stdout, `Ollama returned ${response}`), + }; } return { kind: "warmed", ok: true, timedOut: false }; - } catch { - return { kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }; + } catch (error) { + return { + kind: "warmed", + ok: false, + timedOut: false, + reason: "spawn-failed", + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail( + error instanceof Error ? error.message : error, + "warm-up process could not start", + ), + }; } } diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index d8516b67e5f..a78d9e38528 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -55,11 +55,16 @@ describe("runOllamaRestartRecovery", () => { ok: false, timedOut: true, reason: "timeout", + endpoint: "http://host.docker.internal:11434", + detail: "curl timed out after 300 seconds", })); const stderr = writes.join(""); expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); - expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b' timed out"); + expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b'"); + expect(stderr).toContain("timed out"); + expect(stderr).toContain("at http://host.docker.internal:11434"); + expect(stderr).toContain("Repair that Ollama route and rerun this command"); expect(stderr).toContain("continuing to OpenClaw dispatch"); }); @@ -76,9 +81,14 @@ describe("runOllamaRestartRecovery", () => { ok: false, timedOut: false, reason, + endpoint: "http://host.docker.internal:11434", + detail: "bounded failure detail", })); - expect(writes.join("")).toContain(message); + const stderr = writes.join(""); + expect(stderr).toContain(message); + expect(stderr).toContain("http://host.docker.internal:11434"); + expect(stderr).toContain("Repair that Ollama route and rerun this command"); }); it.each([ diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index dd1c0556942..b8681103c74 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -46,7 +46,9 @@ function reportRecovery( return; } proc.stderr.write( - ` Ollama warm-up for '${model}' ${describeWarmFailure(result.reason)}; continuing to OpenClaw dispatch.\n`, + ` Ollama warm-up for '${model}' at ${result.endpoint} ${describeWarmFailure(result.reason)} ` + + `(${result.detail}). Repair that Ollama route and rerun this command; continuing to ` + + `OpenClaw dispatch.\n`, ); return; } diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index e1d327fc865..7f04115d894 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -189,11 +189,11 @@ describe("Windows-host Ollama transport", () => { "-sS", "--max-time", "120", - "http://host.docker.internal:11434/api/generate", "-H", "Content-Type: application/json", "-d", expect.stringContaining('"model":"qwen3.5:9b"'), + "http://host.docker.internal:11434/api/generate", ]); }); @@ -265,7 +265,13 @@ describe("Windows-host Ollama transport", () => { expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); expect(capture).toHaveBeenCalledOnce(); expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/ps", + ]), ); }); @@ -281,7 +287,13 @@ describe("Windows-host Ollama transport", () => { }); expect(capture).toHaveBeenCalledOnce(); expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/show", + ]), ); }); @@ -303,7 +315,13 @@ describe("Windows-host Ollama transport", () => { message: expect.stringContaining("cannot verify the required 64000-token window"), }); expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/ps", + ]), ); }); }); diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index abdf0c09d64..a99ba4a5d58 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -1224,7 +1224,7 @@ describe("local inference helpers", () => { const probe1 = getOllamaProbeCommand("qwen3.5:9b", 30, "5m"); expect(probe1).toContain("--max-time"); expect(probe1).toContain("30"); - const payload1 = probe1[probe1.length - 1]; + const payload1 = probe1[probe1.indexOf("-d") + 1]; expect(payload1).toMatch(/"keep_alive":"5m"/); }); @@ -1235,7 +1235,7 @@ describe("local inference helpers", () => { expect(command).toContain("--max-time"); expect(command).toContain("120"); expect(command).toContain("http://127.0.0.1:11434/api/generate"); - const payload = command[command.length - 1]; + const payload = command[command.indexOf("-d") + 1]; expect(payload).toMatch(/"model":"nemotron-3-nano:30b"/); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 31e95e5cd7a..f16eb1dde84 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -1907,27 +1907,19 @@ export function getOllamaProbeCommand( }); const host = getResolvedOllamaHost(); const endpoint = `http://${host}:${OLLAMA_PORT}/api/generate`; - buildValidatedCurlCommandArgs([ - "-sS", - "--max-time", - String(timeoutSeconds), - "-H", - "Content-Type: application/json", - "-d", - payload, - endpoint, - ]); - const curlArgs = [ - "-sS", - "--max-time", - String(timeoutSeconds), - endpoint, - "-H", - "Content-Type: application/json", - "-d", - payload, - ]; - return getOllamaApiCommand(curlArgs, host); + return getOllamaApiCommand( + buildValidatedCurlCommandArgs([ + "-sS", + "--max-time", + String(timeoutSeconds), + "-H", + "Content-Type: application/json", + "-d", + payload, + endpoint, + ]), + host, + ); } export function validateOllamaModel( diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index ad3d4a177a4..cc3867a4927 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -193,4 +193,28 @@ describe("Ollama local provider sandbox-facing model gate", () => { expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); }); + + it("restores the prior cleanup route when route application throws", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + applyLocalInferenceRoute: async () => { + throw new Error("route application failed"); + }, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("route application failed"); + + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); }); From abc65bdf3c1c6f6648d65769fae26b1129769353 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 03:15:26 -0700 Subject: [PATCH 08/47] fix(inference): finalize Ollama lifecycle recovery --- .../agent/passthrough-ollama-recovery.test.ts | 6 +-- .../agent/passthrough-ollama-recovery.ts | 4 +- .../local-windows-ollama-transport.test.ts | 1 + src/lib/inference/local.ts | 5 +- .../inference-providers/ollama-local.test.ts | 48 +++++++++++++++++++ .../inference-providers/ollama-local.ts | 2 + src/lib/tunnel/services.test.ts | 2 + src/lib/tunnel/services.ts | 9 +++- 8 files changed, 70 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index a78d9e38528..d5ef2ca1eb9 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -60,7 +60,7 @@ describe("runOllamaRestartRecovery", () => { })); const stderr = writes.join(""); - expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); + expect(stderr).toContain("Checking whether the Ollama model is loaded"); expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b'"); expect(stderr).toContain("timed out"); expect(stderr).toContain("at http://host.docker.internal:11434"); @@ -93,9 +93,9 @@ describe("runOllamaRestartRecovery", () => { it.each([ ["already-loaded", "Ollama model 'qwen3.6:35b' is already loaded"], - ["unreachable", "Ollama was unreachable during the restart check"], + ["unreachable", "Ollama was unreachable during the model check"], ["missing-model", "No Ollama model is recorded for this sandbox"], - ["not-ollama", "Checking Ollama model readiness after daemon restart"], + ["not-ollama", "Checking whether the Ollama model is loaded"], ] as const)("handles the %s skip reason", (reason, message) => { const { writes, proc } = makeProcMock(); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index b8681103c74..58f437ffc26 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -73,7 +73,7 @@ function reportRecovery( break; case "unreachable": proc.stderr.write( - " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", + " Ollama was unreachable during the model check; continuing to OpenClaw dispatch.\n", ); break; case "missing-model": @@ -96,7 +96,7 @@ export function runOllamaRestartRecovery( proc: OllamaRestartRecoveryProcess, recoverOllama: OllamaRestartRecoveryFn = maybeWarmOllamaAfterDaemonRestart, ): void { - proc.stderr.write(" Checking Ollama model readiness after daemon restart...\n"); + proc.stderr.write(" Checking whether the Ollama model is loaded...\n"); try { reportRecovery(route, recoverOllama(route), proc); } catch { diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 7f04115d894..30e65cb07a0 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -239,6 +239,7 @@ describe("Windows-host Ollama transport", () => { "--rm", "--add-host", "host.openshell.internal:host-gateway", + "http://host.openshell.internal:11434/api/tags", ]), ); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index f16eb1dde84..cf0d6e6a1c8 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -1250,7 +1250,10 @@ export function getLocalProviderContainerReachabilityCheck( // requires a Bearer token on every endpoint (#3338) and the ephemeral // probe container doesn't carry one, but the goal here is connectivity // not authorisation. - const containerPort = getOllamaContainerPort(); + const containerPort = + getResolvedOllamaHost() === OLLAMA_HOST_DOCKER_INTERNAL + ? OLLAMA_PORT + : getOllamaContainerPort(); if (responseMode === "body" && containerPort !== OLLAMA_PORT) return null; return [ "docker", diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index cc3867a4927..baf7f7b53cb 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -217,4 +217,52 @@ describe("Ollama local provider sandbox-facing model gate", () => { expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); }); + + it("restores the prior cleanup route when provider-owned proof mismatches", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + providerOwnedInferenceProof: { + protocol: "openai-chat-completions", + model: "ollama/wrong-model", + toolCallingRequired: true, + }, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("restores the prior cleanup route when model validation fails", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + localInference: { + validateOllamaModelWithToolsOverride: () => ({ + ok: false, + message: "model validation failed", + }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index 404e44e7a75..5968e16b1d9 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -158,6 +158,7 @@ export async function setupOllamaLocalInference( providerOwnedInferenceProof.model !== normalizeHostLocalOllamaModelRef(model) || providerOwnedInferenceProof.toolCallingRequired !== !allowToolsIncompatible ) { + rollbackCleanupRoute(); error(" Provider-owned Ollama proof does not match the accepted model capability request."); return exitProcess(1); } @@ -169,6 +170,7 @@ export async function setupOllamaLocalInference( allowToolsIncompatible, ); if (!probe.ok) { + rollbackCleanupRoute(); error(` ${probe.message}`); return exitProcess(1); } diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index c361d0607e2..b83df9bd62b 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -634,6 +634,8 @@ describe("stopAll", () => { expect(output).toContain("Ollama model cleanup failed at http://host.docker.internal:11434"); expect(output).toContain("saved local route was retained"); + expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); + expect(output).not.toContain("All services stopped"); }); }); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 25559492c1c..e3339313fc6 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -525,15 +525,18 @@ export function stopAll(opts: ServiceOptions = {}): void { warn("Hint: run 'nemoclaw stop' with a registered sandbox or set NEMOCLAW_SANDBOX_NAME."); } + let ollamaCleanupIncomplete = false; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; const cleanup = unloadOllamaModels(); if (cleanup && !cleanup.ok) { + ollamaCleanupIncomplete = true; warn( `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; repair Ollama and retry this command.`, ); } } catch (error) { + ollamaCleanupIncomplete = true; warn( `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, ); @@ -578,7 +581,11 @@ export function stopAll(opts: ServiceOptions = {}): void { return; } - info("All services stopped."); + if (ollamaCleanupIncomplete) { + info("Host services stopped; Ollama model cleanup remains incomplete."); + } else { + info("All services stopped."); + } } /** From 3dbf08b26c9b35e604460108c8e118d8284405b8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 03:37:20 -0700 Subject: [PATCH 09/47] fix(tunnel): scope Ollama cleanup completion --- .../local-windows-ollama-transport.test.ts | 20 ++++------ src/lib/inference/local.ts | 38 ++++++++++--------- src/lib/inference/ollama/proxy.ts | 6 +++ src/lib/tunnel/services.test.ts | 6 ++- src/lib/tunnel/services.ts | 9 ++++- 5 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 30e65cb07a0..f612ecff98f 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -96,7 +96,7 @@ describe("Windows-host Ollama transport", () => { it("probes persisted Windows-host health through Docker Desktop", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-health-")); - const calls: Array<{ command: string; args: readonly string[] }> = []; + const calls: (readonly string[])[] = []; try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); resetOllamaHostCache(); @@ -104,17 +104,13 @@ describe("Windows-host Ollama transport", () => { const result = probeLocalProviderHealth("ollama-local", { loadOllamaProxyTokenImpl: () => null, - ollamaSpawnSyncImpl: (command, args) => { - calls.push({ command, args }); - const statusOutput = args[args.indexOf("-w") + 1].replace("%{http_code}", "200"); - const stdout = `${JSON.stringify({ models: [] })}${statusOutput}`; + ollamaRunCaptureExImpl: (command) => { + calls.push(command); return { - pid: 1, - output: ["", stdout, ""], - stdout, + stdout: JSON.stringify({ models: [] }), stderr: "", - status: 0, - signal: null, + exitCode: 0, + timedOut: false, }; }, }); @@ -124,9 +120,9 @@ describe("Windows-host Ollama transport", () => { endpoint: "http://host.docker.internal:11434/api/tags", }); expect(calls).toHaveLength(1); - expect(calls[0]).toMatchObject({ command: "docker" }); - expect(calls[0]?.args).toEqual( + expect(calls[0]).toEqual( expect.arrayContaining([ + "docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE, diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index cf0d6e6a1c8..35f75550cb6 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -6,7 +6,6 @@ * health checks, and command generators for vLLM and Ollama. */ -import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import nodePath from "node:path"; @@ -33,7 +32,6 @@ import { buildSubprocessEnv } from "../subprocess-env"; import { readLocalAdapterJsonFile, - LOCAL_ADAPTER_HEALTH_MAX_RESPONSE_BYTES, removeLocalAdapterFile, resolveSharedLocalAdapterStateRoot, writeLocalAdapterJsonFile, @@ -382,7 +380,7 @@ export interface LocalProviderHealthProbeOptions { model?: string | null; runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; /** Executes the translated Windows-host Docker probe. Injectable for transport tests. */ - ollamaSpawnSyncImpl?: NonNullable; + ollamaRunCaptureExImpl?: RunCaptureExFn; /** * Lets callers that perform their own Ollama auth-proxy check avoid the * legacy inline proxy subprobe. The inline subprobe is retained for status @@ -423,19 +421,25 @@ function runLocalCurlProbe(argv: string[], opts: CurlProbeOptions = {}): CurlPro function runOllamaLocalCurlProbe( argv: string[], host: string, - opts: CurlProbeOptions = {}, - spawnSyncImpl: NonNullable = spawnSync, + runCaptureExImpl: RunCaptureExFn = runCaptureEx, ): CurlProbeResult { - return runCurlProbe(argv, { - ...opts, - env: buildSubprocessEnv(), - maxResponseBytes: opts.maxResponseBytes ?? LOCAL_ADAPTER_HEALTH_MAX_RESPONSE_BYTES, - replaceEnv: true, - spawnSyncImpl: (_command, args, spawnOptions) => { - const [executable, ...translatedArgs] = getOllamaApiCommand(args, host); - return spawnSyncImpl(executable, translatedArgs, spawnOptions); - }, - }); + const command = getOllamaApiCommand(buildValidatedCurlCommandArgs(["-f", ...argv]), host); + const result = runCaptureExImpl(command); + const ok = result.exitCode === 0; + const stderr = String(result.stderr ?? ""); + return { + ok, + httpStatus: ok ? 200 : 0, + curlStatus: result.exitCode ?? 1, + body: result.stdout, + stderr, + message: ok + ? "HTTP 200" + : (stderr || result.stdout || `Docker Ollama probe exited ${String(result.exitCode)}`) + .replace(/\s+/g, " ") + .trim() + .slice(0, 300), + }; } export interface VllmModelsProbeOptions { @@ -1102,8 +1106,8 @@ export function probeLocalProviderHealth( const runCurlProbeImpl = options.runCurlProbeImpl ?? (provider === "ollama-local" && resolvedOllamaHost === OLLAMA_HOST_DOCKER_INTERNAL - ? (argv: string[], opts?: CurlProbeOptions) => - runOllamaLocalCurlProbe(argv, resolvedOllamaHost, opts, options.ollamaSpawnSyncImpl) + ? (argv: string[]) => + runOllamaLocalCurlProbe(argv, resolvedOllamaHost, options.ollamaRunCaptureExImpl) : runLocalCurlProbe); let result: CurlProbeResult; if (managedBinding) { diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index fb7ebec65cd..e5303acfb9e 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -35,6 +35,7 @@ const { getOllamaModelOptions, getOllamaWarmupCommand, getResolvedOllamaHost, + loadPersistedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, probeOllamaModelCapabilities, selectDefaultOllamaModel, @@ -1648,11 +1649,16 @@ function unloadOllamaModels( }; } +function hasPersistedOllamaRoute(): boolean { + return loadPersistedOllamaHost() !== null; +} + export { checkOllamaModelToolSupport, ensureOllamaAuthProxy, getOllamaProxyToken, getOllamaPullTimeoutMs, + hasPersistedOllamaRoute, isProxyHealthy, killStaleProxy, noAuthProxy, diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index b83df9bd62b..936f233f22d 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -628,7 +628,11 @@ describe("stopAll", () => { }; const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir, unloadOllamaModels: () => failure }); + stopAll({ + pidDir, + unloadOllamaModels: () => failure, + hasPersistedOllamaRoute: () => true, + }); const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); logSpy.mockRestore(); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index e3339313fc6..6ff3669e470 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -20,6 +20,7 @@ import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding" import { isObjectRecord } from "../core/json-types"; import { DASHBOARD_PORT } from "../core/ports"; import { + hasPersistedOllamaRoute as hasDefaultPersistedOllamaRoute, unloadOllamaModels as unloadDefaultOllamaModels, type OllamaUnloadResult, } from "../inference/ollama/proxy"; @@ -49,6 +50,8 @@ export interface ServiceOptions { processControl?: ProcessControl; /** Injectable Ollama model cleanup for tests. */ unloadOllamaModels?: () => OllamaUnloadResult | void; + /** Whether an accepted Ollama route makes cleanup part of this stop contract. */ + hasPersistedOllamaRoute?: () => boolean; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ cloudflareTunnelToken?: string; /** Also release the managed host gateway port (legacy full-stop only). */ @@ -525,18 +528,20 @@ export function stopAll(opts: ServiceOptions = {}): void { warn("Hint: run 'nemoclaw stop' with a registered sandbox or set NEMOCLAW_SANDBOX_NAME."); } + const ollamaCleanupExpected = + opts.hasPersistedOllamaRoute?.() ?? hasDefaultPersistedOllamaRoute(); let ollamaCleanupIncomplete = false; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; const cleanup = unloadOllamaModels(); if (cleanup && !cleanup.ok) { - ollamaCleanupIncomplete = true; + ollamaCleanupIncomplete = ollamaCleanupExpected; warn( `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; repair Ollama and retry this command.`, ); } } catch (error) { - ollamaCleanupIncomplete = true; + ollamaCleanupIncomplete = ollamaCleanupExpected; warn( `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, ); From 71e39e69d1f1cc02db984d091b1599301f16133f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 05:19:57 -0700 Subject: [PATCH 10/47] fix(inference): complete Ollama route lifecycle --- src/lib/actions/sandbox/destroy.test.ts | 34 +++++- src/lib/actions/sandbox/destroy.ts | 28 +++++ .../local-windows-ollama-transport.test.ts | 115 +++++++++++------- src/lib/inference/local.test.ts | 9 +- src/lib/inference/local.ts | 48 +++++--- src/lib/inference/ollama/proxy.ts | 41 +++++-- .../inference-providers/ollama-local.test.ts | 45 ++++++- src/lib/onboard/inference-providers/types.ts | 1 + src/lib/onboard/setup-inference.ts | 15 ++- src/lib/tunnel/services.test.ts | 6 +- src/lib/tunnel/services.ts | 9 +- .../onboard-inference-reconciliation.test.ts | 28 +++++ 12 files changed, 289 insertions(+), 90 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.test.ts b/src/lib/actions/sandbox/destroy.test.ts index 9af10ead97a..4a9724ab56c 100644 --- a/src/lib/actions/sandbox/destroy.test.ts +++ b/src/lib/actions/sandbox/destroy.test.ts @@ -4,12 +4,44 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { assertUnambiguousDestroyContainerIdentity, cleanupSandboxServices } from "./destroy"; +import { + assertUnambiguousDestroyContainerIdentity, + cleanupSandboxServices, + retireFinalOllamaRouteAfterSandboxRemoval, +} from "./destroy"; const SANDBOX = "mybox"; const mainPidDir = path.resolve("/tmp", `nemoclaw-services-${SANDBOX}`); const googlechatPidDir = `${mainPidDir}-googlechat`; +describe("final Ollama route retirement", () => { + it("retires the route after removing the final Ollama sandbox", () => { + const clearIfUnused = vi.fn(() => true); + + expect( + retireFinalOllamaRouteAfterSandboxRemoval( + { provider: "ollama-local" }, + [{ provider: "nvidia-prod" }], + clearIfUnused, + ), + ).toBe(true); + expect(clearIfUnused).toHaveBeenCalledWith(["nvidia-prod"]); + }); + + it("retains the route while another Ollama sandbox remains", () => { + const clearIfUnused = vi.fn(() => false); + + expect( + retireFinalOllamaRouteAfterSandboxRemoval( + { provider: "ollama-local" }, + [{ provider: "ollama-local" }], + clearIfUnused, + ), + ).toBe(false); + expect(clearIfUnused).toHaveBeenCalledWith(["ollama-local"]); + }); +}); + describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => { it("fails closed before later cleanup when the Google Chat tunnel cannot stop", () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 2521f0c6ed8..d108801306a 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -466,6 +466,15 @@ export function removeSandboxRegistryEntryWithReceipt( return removeSandboxWithReceipt(sandboxName); } +export function retireFinalOllamaRouteAfterSandboxRemoval( + removedSandbox: Pick | null, + remainingSandboxes: readonly Pick[], + clearIfUnused: (providers: readonly (string | null | undefined)[]) => boolean, +): boolean { + if (!removedSandbox?.provider?.includes("ollama")) return false; + return clearIfUnused(remainingSandboxes.map(({ provider }) => provider)); +} + function defaultDestroyWarn(message: string): void { console.warn(` ${YW}⚠${R} ${message}`); } @@ -957,6 +966,25 @@ async function destroySandboxUnlocked( ` ${YW}⚠${R} Failed to retire portable lifecycle authority for '${sandboxName}': ${redactDestroyError(error)}`, ); } + if (sandbox?.provider?.includes("ollama")) { + try { + const remainingSandboxes = registry.listSandboxes().sandboxes; + const { clearPersistedOllamaHostIfUnused } = require("../../inference/ollama/proxy") as { + clearPersistedOllamaHostIfUnused( + providers: readonly (string | null | undefined)[], + ): boolean; + }; + retireFinalOllamaRouteAfterSandboxRemoval( + sandbox, + remainingSandboxes, + clearPersistedOllamaHostIfUnused, + ); + } catch (error) { + console.warn( + ` ${YW}⚠${R} Failed to retire the final local Ollama route receipt: ${redactDestroyError(error)}`, + ); + } + } } if (deleteSucceededOrAlreadyGone && removed && priorHttpsPinRouteId) { await revokeDestroyedSandboxHttpsPinRoute(cleanupGatewayName, priorHttpsPinRouteId); diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index f612ecff98f..1a1ad11ee04 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -8,14 +8,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { applyOllamaRuntimeContextWindow, + clearPersistedOllamaHostIfUnused, CONTAINER_REACHABILITY_IMAGE, findReachableOllamaHost, getLocalProviderHealthCheck, getOllamaHostForCleanup, getOllamaModelOptions, - getOllamaProbeCommand, getResolvedOllamaHost, - getOllamaWarmupRequestCommand, OLLAMA_HOST_DOCKER_INTERNAL, loadPersistedOllamaHost, persistResolvedOllamaHost, @@ -25,6 +24,7 @@ import { resetOllamaRuntimeContextWindowAutoState, setResolvedOllamaHost, validateLocalProvider, + validateOllamaModel, } from "./local"; describe("Windows-host Ollama transport", () => { @@ -62,17 +62,50 @@ describe("Windows-host Ollama transport", () => { } }); - it("restores the persisted route before fresh-process connect discovery", () => { + it("retires the final persisted route after Ollama ownership ends", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-retire-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect(clearPersistedOllamaHostIfUnused(["nvidia-prod"], stateRoot)).toBe(true); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + expect(getOllamaHostForCleanup(stateRoot)).toBe("127.0.0.1"); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("retains the persisted route while another Ollama sandbox owns it", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-retain-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect(clearPersistedOllamaHostIfUnused(["ollama-local"], stateRoot)).toBe(false); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("re-probes a stale persisted route before fresh-process connect discovery", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-connect-")); try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); resetOllamaHostCache(); - const capture = vi.fn(() => { - throw new Error("fresh-process connect must not probe WSL loopback"); - }); + const capture = vi.fn((command: readonly string[]) => + command.includes("http://127.0.0.1:11434/api/tags") ? JSON.stringify({ models: [] }) : "", + ); - expect(findReachableOllamaHost(capture, {}, stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); - expect(capture).not.toHaveBeenCalled(); + expect(findReachableOllamaHost(capture, { isWsl: true }, stateRoot)).toBe("127.0.0.1"); + expect(capture).toHaveBeenCalledTimes(2); + expect(capture.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(["docker", "run", "http://host.docker.internal:11434/api/tags"]), + ); + expect(capture.mock.calls[1]?.[0]).toEqual( + expect.arrayContaining(["curl", "http://127.0.0.1:11434/api/tags"]), + ); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + expect(getResolvedOllamaHost()).toBe("127.0.0.1"); } finally { rmSync(stateRoot, { recursive: true, force: true }); } @@ -88,7 +121,7 @@ describe("Windows-host Ollama transport", () => { resetOllamaHostCache(); expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); - expect(getResolvedOllamaHost(stateRoot)).toBe("127.0.0.1"); + expect(getOllamaHostForCleanup(stateRoot)).toBe("127.0.0.1"); } finally { rmSync(stateRoot, { recursive: true, force: true }); } @@ -100,9 +133,10 @@ describe("Windows-host Ollama transport", () => { try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); resetOllamaHostCache(); - expect(getResolvedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const result = probeLocalProviderHealth("ollama-local", { + findReachableOllamaHostImpl: () => OLLAMA_HOST_DOCKER_INTERNAL, loadOllamaProxyTokenImpl: () => null, ollamaRunCaptureExImpl: (command) => { calls.push(command); @@ -157,40 +191,37 @@ describe("Windows-host Ollama transport", () => { ); }); - it("builds warm-up and validation requests for Docker Desktop (#10553)", () => { + it("validates a Windows-host model through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((command: readonly string[]) => + command.includes("http://host.docker.internal:11434/api/show") + ? JSON.stringify({ capabilities: ["tools"] }) + : "", + ); + const captureEx = vi.fn((command: readonly string[]) => ({ + stdout: + command[0] === "docker" && + command.includes("http://host.docker.internal:11434/api/generate") + ? JSON.stringify({ done: true, response: "ready" }) + : "", + stderr: "", + exitCode: command[0] === "docker" ? 0 : 7, + timedOut: false, + })); - expect(getOllamaWarmupRequestCommand("qwen3.5:9b")).toEqual([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "-s", - "--connect-timeout", - "10", - "--max-time", - "120", - "http://host.docker.internal:11434/api/generate", - "-H", - "Content-Type: application/json", - "-d", - expect.stringContaining('"model":"qwen3.5:9b"'), - ]); - - expect(getOllamaProbeCommand("qwen3.5:9b")).toEqual([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "-sS", - "--max-time", - "120", - "-H", - "Content-Type: application/json", - "-d", - expect.stringContaining('"model":"qwen3.5:9b"'), - "http://host.docker.internal:11434/api/generate", - ]); + expect(validateOllamaModel("qwen3.5:9b", capture, () => false, captureEx)).toEqual({ + ok: true, + }); + expect(captureEx).toHaveBeenCalledOnce(); + expect(captureEx.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/generate", + ]), + ); }); it("validates health and container reachability through Docker Desktop (#10553)", () => { diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index a99ba4a5d58..1b535e579fb 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -176,10 +176,11 @@ describe("local inference helpers", () => { "http://127.0.0.1:11434/api/tags", "http://host.docker.internal:11434/api/tags", ]); - expect(commands.map((command) => command.slice(2, 6))).toEqual([ - ["--connect-timeout", "3", "--max-time", "5"], - ["--connect-timeout", "3", "--max-time", "5"], - ]); + expect(commands.map((command) => command[0])).toEqual(["curl", "docker"]); + expect(commands[0]).toEqual(expect.arrayContaining(["--connect-timeout", "3", "--max-time", "5"])); + expect(commands[1]).toEqual( + expect.arrayContaining([CONTAINER_REACHABILITY_IMAGE, "--connect-timeout", "3", "--max-time", "5"]), + ); }); it("returns the expected base URL for vllm-local", () => { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 35f75550cb6..ae3d5b43c3d 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -186,45 +186,43 @@ export function findReachableOllamaHost( ): string | null { if (_resolvedOllamaHost !== null) return _resolvedOllamaHost; const persistedHost = loadPersistedOllamaHost(stateRoot); - if (persistedHost) { - _resolvedOllamaHost = persistedHost; - return persistedHost; - } const capture = runCaptureImpl ?? runCapture; - for (const host of ollamaCandidateHosts(wslDetection)) { + const candidates = [ + ...(persistedHost ? [persistedHost] : []), + ...ollamaCandidateHosts(wslDetection).filter((host) => host !== persistedHost), + ]; + for (const host of candidates) { // Explicit timeouts: a blackholed host (e.g., firewalled host.docker.internal) // would otherwise stall the synchronous onboard probe for the OS connect // timeout (~75-130s on Linux). Matches the convention used in // getLocalProviderHealthStatus probes. const result = capture( - [ - "curl", + getOllamaApiCommand( + [ "-sf", "--connect-timeout", "3", "--max-time", "5", `http://${host}:${OLLAMA_PORT}/api/tags`, - ], + ], + host, + ), { ignoreError: true }, ); if (result) { _resolvedOllamaHost = host; return host; } + if (host === persistedHost) clearPersistedOllamaHost(stateRoot); } return null; } // Returns the resolved host if a probe has succeeded, otherwise OLLAMA_LOCALHOST. // Used by URL-builder helpers that need a string and don't want to re-probe. -export function getResolvedOllamaHost( - stateRoot: string = resolveSharedLocalAdapterStateRoot(), -): string { - if (_resolvedOllamaHost) return _resolvedOllamaHost; - const persistedHost = loadPersistedOllamaHost(stateRoot); - if (persistedHost) _resolvedOllamaHost = persistedHost; - return persistedHost ?? OLLAMA_LOCALHOST; +export function getResolvedOllamaHost(): string { + return _resolvedOllamaHost ?? OLLAMA_LOCALHOST; } /** Persist the accepted local Ollama route for later CLI processes. */ @@ -261,6 +259,22 @@ export function loadPersistedOllamaHost( return receipt?.schemaVersion === 1 && isSupportedOllamaHost(receipt.host) ? receipt.host : null; } +export function clearPersistedOllamaHost( + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + removeLocalAdapterFile(ollamaHostReceiptPath(stateRoot)); + _resolvedOllamaHost = null; +} + +export function clearPersistedOllamaHostIfUnused( + providers: readonly (string | null | undefined)[], + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): boolean { + if (providers.some((provider) => provider?.includes("ollama"))) return false; + clearPersistedOllamaHost(stateRoot); + return true; +} + /** Resolve cleanup transport after process-local onboarding state is gone. */ export function getOllamaHostForCleanup( stateRoot: string = resolveSharedLocalAdapterStateRoot(), @@ -381,6 +395,7 @@ export interface LocalProviderHealthProbeOptions { runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; /** Executes the translated Windows-host Docker probe. Injectable for transport tests. */ ollamaRunCaptureExImpl?: RunCaptureExFn; + findReachableOllamaHostImpl?: () => string | null; /** * Lets callers that perform their own Ollama auth-proxy check avoid the * legacy inline proxy subprobe. The inline subprobe is retained for status @@ -1056,6 +1071,9 @@ export function probeLocalProviderHealth( ): LocalProviderHealthStatus | null { const providerLabel = getLocalProviderLabel(provider); if (!providerLabel) return null; + if (provider === "ollama-local") { + (options.findReachableOllamaHostImpl ?? findReachableOllamaHost)(); + } let managedState: ManagedVllmProviderState = { kind: "absent" }; if (provider === "vllm-local") { diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index e5303acfb9e..0c408a0b3c9 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -30,14 +30,16 @@ const { ensurePulledOllamaModel }: typeof import("./model-discovery") = const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./model-discovery"); const { getBootstrapOllamaModelOptions, + findReachableOllamaHost, + clearPersistedOllamaHostIfUnused, getOllamaApiCommand, - getOllamaHostForCleanup, getOllamaModelOptions, getOllamaWarmupCommand, getResolvedOllamaHost, loadPersistedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, probeOllamaModelCapabilities, + persistResolvedOllamaHost, selectDefaultOllamaModel, validateOllamaModel, } = require("../local"); @@ -1484,15 +1486,34 @@ function unloadOllamaModels( onlyModels?: readonly string[], options: OllamaUnloadOptions = {}, ): OllamaUnloadResult { - const releaseHost = options.getResolvedOllamaHost - ? options.getResolvedOllamaHost() - : getOllamaHostForCleanup(options.ollamaHostStateRoot); - const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost); + const requestedModels = onlyModels?.map((model) => model.trim()).filter(Boolean) ?? []; + let selectedModels: readonly string[] | null = onlyModels?.length ? requestedModels : null; + let releaseHost: string | null; + if (options.getResolvedOllamaHost) { + releaseHost = options.getResolvedOllamaHost(); + } else { + const persistedHost = loadPersistedOllamaHost(options.ollamaHostStateRoot); + releaseHost = + persistedHost ?? + findReachableOllamaHost(undefined, {}, options.ollamaHostStateRoot); + if (releaseHost && !persistedHost) { + persistResolvedOllamaHost(releaseHost, options.ollamaHostStateRoot); + } + } + if (!releaseHost) { + return { + ok: true, + outcome: "not-resident", + endpoint: buildLocalOllamaEndpoint(), + selectedModels: selectedModels ?? [], + discoveries: [], + requests: [], + }; + } + const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost!); const spawnSyncImpl = options.spawnSync ?? spawnSync; const sleepImpl = options.sleep ?? defaultReleaseSleep; const maxAttempts = Math.max(1, options.maxAttempts ?? OLLAMA_RELEASE_MAX_ATTEMPTS); - const requestedModels = onlyModels?.map((model) => model.trim()).filter(Boolean) ?? []; - let selectedModels: readonly string[] | null = onlyModels?.length ? requestedModels : null; const discoveries: OllamaModelDiscoveryEvidence[] = []; const requests: OllamaUnloadRequestEvidence[] = []; let lastMatchedModels: readonly string[] = []; @@ -1649,16 +1670,12 @@ function unloadOllamaModels( }; } -function hasPersistedOllamaRoute(): boolean { - return loadPersistedOllamaHost() !== null; -} - export { checkOllamaModelToolSupport, ensureOllamaAuthProxy, + clearPersistedOllamaHostIfUnused, getOllamaProxyToken, getOllamaPullTimeoutMs, - hasPersistedOllamaRoute, isProxyHealthy, killStaleProxy, noAuthProxy, diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index baf7f7b53cb..2a84e8c67aa 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -1,12 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + getOllamaWarmupCommand, + OLLAMA_HOST_DOCKER_INTERNAL, + resetOllamaHostCache, + setResolvedOllamaHost, +} from "../../inference/local"; import { setupOllamaLocalInference } from "./ollama-local"; import type { OllamaDeps } from "./types"; const CREDENTIAL_ENV = "NEMOCLAW_OLLAMA_PROXY_TOKEN"; +afterEach(() => resetOllamaHostCache()); + const SANDBOX_ENDPOINT_MISMATCH = "Selected Ollama model 'llama3.2:1b' answers on http://127.0.0.1:11434, but the daemon the " + "sandbox reaches through http://host.openshell.internal:11434 does not serve it " + @@ -74,7 +82,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("blocks even when every host-side check passes (#9454)", async () => { const validateOllamaModelWithToolsOverride = vi.fn(() => ({ ok: true })); - const run = vi.fn(() => ({ status: 0 })); + const run = vi.fn((_command) => ({ status: 0 })); await expect( setupOllamaLocalInference( @@ -125,6 +133,39 @@ describe("Ollama local provider sandbox-facing model gate", () => { expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); }); + it("dispatches Windows-host warm-up through Docker Desktop", async () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const runCommands: unknown[] = []; + const run = vi.fn((command: unknown) => { + runCommands.push(command); + return { status: 0 }; + }); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + getOllamaWarmupCommand, + run, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: vi.fn(), + }, + }), + ), + ).resolves.toEqual({ done: false }); + + expect(run).toHaveBeenCalledOnce(); + expect(runCommands[0]).toEqual([ + "bash", + "-c", + expect.stringMatching( + /docker.*curlimages\/curl:8\.10\.1.*host\.docker\.internal:11434\/api\/generate/, + ), + ]); + }); + it("fails before provider registration when the cleanup route cannot be staged", async () => { const upsertProvider = vi.fn(() => ({ ok: true })); const error = vi.fn(); diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 1cadb39c3c7..0b11fbbf0ca 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -256,6 +256,7 @@ export type OllamaDeps = CommonDeps & { ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; persistResolvedOllamaHost?(): (() => void) | void; + clearPersistedOllamaHostIfUnused?(providers: readonly (string | null | undefined)[]): boolean; }; /** Exact provider-owned proof used instead of legacy host warmup/probes. */ providerOwnedInferenceProof?: { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 60e40e1a96a..94242cc8c0b 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -546,6 +546,7 @@ export type SetupInference = ( */ function releaseSupersededOllamaModel( previous: OllamaModelHolder | null, + nextProvider: string, nextModel: string, result: SetupInferenceResult, deps: SetupInferenceDeps, @@ -560,14 +561,23 @@ function releaseSupersededOllamaModel( withOwnershipLock(() => { const peers = deps.listSandboxes?.().sandboxes ?? []; const superseded = supersededOllamaModel(previous, nextModel, peers); - if (!superseded) return; + const retireRoute = + !!previous.provider?.includes("ollama") && + !nextProvider.includes("ollama") && + !peers.some((peer) => peer.provider?.includes("ollama")); + if (!superseded && !retireRoute) return; try { revalidateSandboxIdentity?.("release the superseded Ollama model"); } catch (error) { authorityRefusal = error; return; } - deps.unloadOllamaModels?.([superseded]); + if (superseded) deps.unloadOllamaModels?.([superseded]); + if (retireRoute) { + deps.localInference.clearPersistedOllamaHostIfUnused?.( + peers.map((peer) => peer.provider), + ); + } }); } catch { /* Best-effort: a failed unload must not fail an onboarding that already committed its route. */ @@ -1162,6 +1172,7 @@ export function createSetupInference( const result = await mutateGatewayRoute(); releaseSupersededOllamaModel( previousSandbox, + provider, model, result, deps, diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index 936f233f22d..b83df9bd62b 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -628,11 +628,7 @@ describe("stopAll", () => { }; const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ - pidDir, - unloadOllamaModels: () => failure, - hasPersistedOllamaRoute: () => true, - }); + stopAll({ pidDir, unloadOllamaModels: () => failure }); const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); logSpy.mockRestore(); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 6ff3669e470..e3339313fc6 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -20,7 +20,6 @@ import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding" import { isObjectRecord } from "../core/json-types"; import { DASHBOARD_PORT } from "../core/ports"; import { - hasPersistedOllamaRoute as hasDefaultPersistedOllamaRoute, unloadOllamaModels as unloadDefaultOllamaModels, type OllamaUnloadResult, } from "../inference/ollama/proxy"; @@ -50,8 +49,6 @@ export interface ServiceOptions { processControl?: ProcessControl; /** Injectable Ollama model cleanup for tests. */ unloadOllamaModels?: () => OllamaUnloadResult | void; - /** Whether an accepted Ollama route makes cleanup part of this stop contract. */ - hasPersistedOllamaRoute?: () => boolean; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ cloudflareTunnelToken?: string; /** Also release the managed host gateway port (legacy full-stop only). */ @@ -528,20 +525,18 @@ export function stopAll(opts: ServiceOptions = {}): void { warn("Hint: run 'nemoclaw stop' with a registered sandbox or set NEMOCLAW_SANDBOX_NAME."); } - const ollamaCleanupExpected = - opts.hasPersistedOllamaRoute?.() ?? hasDefaultPersistedOllamaRoute(); let ollamaCleanupIncomplete = false; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; const cleanup = unloadOllamaModels(); if (cleanup && !cleanup.ok) { - ollamaCleanupIncomplete = ollamaCleanupExpected; + ollamaCleanupIncomplete = true; warn( `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; repair Ollama and retry this command.`, ); } } catch (error) { - ollamaCleanupIncomplete = ollamaCleanupExpected; + ollamaCleanupIncomplete = true; warn( `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, ); diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index 87c74264391..e0c08bf94ed 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -1018,6 +1018,9 @@ describe("re-onboard Ollama GPU release (#9110)", () => { sandboxes: (typeof priorEntry)[]; unloadOllamaModels: (onlyModels: readonly string[]) => void; applyLocalInferenceRoute?: () => Promise; + clearPersistedOllamaHostIfUnused?: ( + providers: readonly (string | null | undefined)[], + ) => boolean; }) { return createDirectSetupInferenceHarness({ runOpenshell: (args) => @@ -1034,6 +1037,15 @@ describe("re-onboard Ollama GPU release (#9110)", () => { getSandbox: options.getSandbox, listSandboxes: () => ({ sandboxes: options.sandboxes, defaultSandbox: null }), unloadOllamaModels: options.unloadOllamaModels, + ...(options.clearPersistedOllamaHostIfUnused + ? { + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, + }, + } + : {}), }, }); } @@ -1054,6 +1066,22 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).toHaveBeenCalledWith(["llama3"]); }); + it("retires the final Ollama route receipt after switching providers", async () => { + const clearPersistedOllamaHostIfUnused = vi.fn(() => true); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }], + unloadOllamaModels: vi.fn(), + clearPersistedOllamaHostIfUnused, + }); + + await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual({ + ok: true, + }); + + expect(clearPersistedOllamaHostIfUnused).toHaveBeenCalledWith(["vllm-local"]); + }); + it("keeps the successful route when the superseded model unload fails (#9110)", async () => { const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(() => { throw new Error("synthetic unload failure"); From e191f6aeeb6afe58386972ac054d0b56d857dded Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 05:43:32 -0700 Subject: [PATCH 11/47] test(inference): require exact Ollama probe URLs --- src/lib/inference/local-windows-ollama-transport.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 1a1ad11ee04..97a5e378a96 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -194,14 +194,14 @@ describe("Windows-host Ollama transport", () => { it("validates a Windows-host model through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const capture = vi.fn((command: readonly string[]) => - command.includes("http://host.docker.internal:11434/api/show") + command.at(-1) === "http://host.docker.internal:11434/api/show" ? JSON.stringify({ capabilities: ["tools"] }) : "", ); const captureEx = vi.fn((command: readonly string[]) => ({ stdout: command[0] === "docker" && - command.includes("http://host.docker.internal:11434/api/generate") + command.at(-1) === "http://host.docker.internal:11434/api/generate" ? JSON.stringify({ done: true, response: "ready" }) : "", stderr: "", From bed0a10b4c5869335b6df47833437a2ee4f837af Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 06:32:57 -0700 Subject: [PATCH 12/47] fix(destroy): preserve failed Ollama cleanup recovery --- src/lib/actions/sandbox/destroy.test.ts | 29 ------------ src/lib/actions/sandbox/destroy.ts | 44 ++++++++++--------- .../tunnel/services-gateway-ownership.test.ts | 29 +++++++++--- src/lib/tunnel/services.ts | 9 ++-- .../destroy-cleanup-sandbox-services.test.ts | 30 +++++++++++++ 5 files changed, 83 insertions(+), 58 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.test.ts b/src/lib/actions/sandbox/destroy.test.ts index 4a9724ab56c..930da4d702b 100644 --- a/src/lib/actions/sandbox/destroy.test.ts +++ b/src/lib/actions/sandbox/destroy.test.ts @@ -7,41 +7,12 @@ import { describe, expect, it, vi } from "vitest"; import { assertUnambiguousDestroyContainerIdentity, cleanupSandboxServices, - retireFinalOllamaRouteAfterSandboxRemoval, } from "./destroy"; const SANDBOX = "mybox"; const mainPidDir = path.resolve("/tmp", `nemoclaw-services-${SANDBOX}`); const googlechatPidDir = `${mainPidDir}-googlechat`; -describe("final Ollama route retirement", () => { - it("retires the route after removing the final Ollama sandbox", () => { - const clearIfUnused = vi.fn(() => true); - - expect( - retireFinalOllamaRouteAfterSandboxRemoval( - { provider: "ollama-local" }, - [{ provider: "nvidia-prod" }], - clearIfUnused, - ), - ).toBe(true); - expect(clearIfUnused).toHaveBeenCalledWith(["nvidia-prod"]); - }); - - it("retains the route while another Ollama sandbox remains", () => { - const clearIfUnused = vi.fn(() => false); - - expect( - retireFinalOllamaRouteAfterSandboxRemoval( - { provider: "ollama-local" }, - [{ provider: "ollama-local" }], - clearIfUnused, - ), - ).toBe(false); - expect(clearIfUnused).toHaveBeenCalledWith(["ollama-local"]); - }); -}); - describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => { it("fails closed before later cleanup when the Google Chat tunnel cannot stop", () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index d108801306a..51730f188b7 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -22,6 +22,7 @@ import { revokeHttpsPinRuntimeAdapterRoute, } from "../../inference/https-pin-runtime-adapter"; import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup"; +import type { OllamaUnloadResult } from "../../inference/ollama/proxy"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, normalizeRuntimeProviderIdentity, @@ -165,8 +166,8 @@ type RunOpenshell = (args: string[], opts?: Record) => { status export type CleanupSandboxServicesDeps = { getSandbox?: typeof registry.getSandbox; - stopAll?: (opts: { sandboxName: string }) => void; - unloadOllamaModels?: () => void; + stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void; + unloadOllamaModels?: () => OllamaUnloadResult | void; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; @@ -227,17 +228,17 @@ export function cleanupSandboxServices( deps.stopAll ?? ((opts: { sandboxName: string }) => { const services = require("../../tunnel/services") as { - stopAll: (opts: { sandboxName: string }) => void; + stopAll: (opts: { sandboxName: string }) => OllamaUnloadResult | void; }; - services.stopAll(opts); + return services.stopAll(opts); }); const unloadOllamaModels = deps.unloadOllamaModels ?? (() => { const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as { - unloadOllamaModels: () => void; + unloadOllamaModels: () => OllamaUnloadResult; }; - unload(); + return unload(); }); const runOpenshell = deps.runOpenshell ?? @@ -289,14 +290,26 @@ export function cleanupSandboxServices( if (stopHostServices) { // `stopAll()` already runs `unloadOllamaModels()` unconditionally — // see src/lib/tunnel/services.ts. Don't double-call here. - stopAll({ sandboxName: validatedSandboxName }); + const cleanup = stopAll({ sandboxName: validatedSandboxName }); + if (cleanup && !cleanup.ok) { + throw new Error( + `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). ` + + "The sandbox registry and saved route were retained; repair Ollama and retry destroy.", + ); + } } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this // branch a single-sandbox destroy would leave models loaded on the GPU. const sb = getSandbox(validatedSandboxName); if (sb?.provider?.includes("ollama")) { - unloadOllamaModels(); + const cleanup = unloadOllamaModels(); + if (cleanup && !cleanup.ok) { + throw new Error( + `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). ` + + "The sandbox registry and saved route were retained; repair Ollama and retry destroy.", + ); + } } } @@ -466,15 +479,6 @@ export function removeSandboxRegistryEntryWithReceipt( return removeSandboxWithReceipt(sandboxName); } -export function retireFinalOllamaRouteAfterSandboxRemoval( - removedSandbox: Pick | null, - remainingSandboxes: readonly Pick[], - clearIfUnused: (providers: readonly (string | null | undefined)[]) => boolean, -): boolean { - if (!removedSandbox?.provider?.includes("ollama")) return false; - return clearIfUnused(remainingSandboxes.map(({ provider }) => provider)); -} - function defaultDestroyWarn(message: string): void { console.warn(` ${YW}⚠${R} ${message}`); } @@ -974,10 +978,8 @@ async function destroySandboxUnlocked( providers: readonly (string | null | undefined)[], ): boolean; }; - retireFinalOllamaRouteAfterSandboxRemoval( - sandbox, - remainingSandboxes, - clearPersistedOllamaHostIfUnused, + clearPersistedOllamaHostIfUnused( + remainingSandboxes.map(({ provider }) => provider), ); } catch (error) { console.warn( diff --git a/src/lib/tunnel/services-gateway-ownership.test.ts b/src/lib/tunnel/services-gateway-ownership.test.ts index 42dd5aeee9c..e502f7ee28f 100644 --- a/src/lib/tunnel/services-gateway-ownership.test.ts +++ b/src/lib/tunnel/services-gateway-ownership.test.ts @@ -14,6 +14,8 @@ import * as gatewayStop from "./gateway-stop"; import * as sandboxGatewayStop from "./sandbox-gateway-stop"; import { stopAll } from "./services"; +const neutralOllamaCleanup = () => undefined; + vi.mock("../adapters/docker", () => ({ dockerCapture: vi.fn(), dockerForceRm: vi.fn(), @@ -247,7 +249,12 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, sandboxName: "alpha", releaseGatewayPort: true }); + stopAll({ + pidDir, + sandboxName: "alpha", + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -282,7 +289,7 @@ describe("stopAll gateway-stop wiring", () => { vi.spyOn(sandboxGatewayStop, "stopSandboxChannels").mockImplementation(() => {}); try { - stopAll({ pidDir, sandboxName: "alpha" }); + stopAll({ pidDir, sandboxName: "alpha", unloadOllamaModels: neutralOllamaCleanup }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -304,7 +311,11 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, releaseGatewayPort: true }); + stopAll({ + pidDir, + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -328,7 +339,11 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, releaseGatewayPort: true }); + stopAll({ + pidDir, + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -349,7 +364,11 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, releaseGatewayPort: true }); + stopAll({ + pidDir, + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index e3339313fc6..a03635b0ab5 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -494,7 +494,7 @@ export function showStatus(opts: ServiceOptions = {}): void { } } -export function stopAll(opts: ServiceOptions = {}): void { +export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { // Resolve the target sandbox once and reuse it for in-sandbox and host-side cleanup. const rawSandboxName = opts.sandboxName ?? @@ -526,9 +526,11 @@ export function stopAll(opts: ServiceOptions = {}): void { } let ollamaCleanupIncomplete = false; + let ollamaCleanup: OllamaUnloadResult | undefined; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; const cleanup = unloadOllamaModels(); + if (cleanup) ollamaCleanup = cleanup; if (cleanup && !cleanup.ok) { ollamaCleanupIncomplete = true; warn( @@ -573,12 +575,12 @@ export function stopAll(opts: ServiceOptions = {}): void { "Hint: rerun with NEMOCLAW_GATEWAY_PORT= to release that gateway, or 'openshell gateway list' to find it.", ); info("Host services stopped; managed gateway not released."); - return; + return ollamaCleanup; } if (gatewayOutcome === "unconfirmed") { info("Host services stopped; managed gateway release was not confirmed."); - return; + return ollamaCleanup; } if (ollamaCleanupIncomplete) { @@ -586,6 +588,7 @@ export function stopAll(opts: ServiceOptions = {}): void { } else { info("All services stopped."); } + return ollamaCleanup; } /** diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 2bdb5ab6e5c..3f9c45c3c86 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -56,6 +56,16 @@ function buildDeps(sandbox: SandboxLike): { } describe("cleanupSandboxServices Ollama unload (#2717)", () => { + const cleanupFailure = { + ok: false as const, + outcome: "discovery-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: [], + discoveries: [], + requests: [], + message: "could not connect", + }; + it("delegates GPU unload to stopAll() exactly once when stopHostServices=true", () => { const harness = buildDeps({ provider: "ollama-local" }); @@ -79,6 +89,26 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.unloadCalls).toBe(1); }); + it("preserves destroy recovery state when stopAll cannot release Ollama", () => { + const harness = buildDeps({ provider: "ollama-local" }); + vi.mocked(harness.deps.stopAll).mockReturnValue(cleanupFailure); + + expect(() => + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps), + ).toThrow(/saved route were retained.*retry destroy/); + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + }); + + it("preserves destroy recovery state when scoped Ollama release fails", () => { + const harness = buildDeps({ provider: "ollama-local" }); + vi.mocked(harness.deps.unloadOllamaModels).mockReturnValue(cleanupFailure); + + expect(() => + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps), + ).toThrow(/saved route were retained.*retry destroy/); + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + }); + it("skips unloadOllamaModels() entirely for non-Ollama providers", () => { const harness = buildDeps({ provider: "nvidia-prod" }); From b35166e8c30992051637173838bb0596dd6c6b62 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 07:51:33 -0700 Subject: [PATCH 13/47] test(inference): focus Ollama transport contracts --- .../agent/passthrough-ollama-recovery.test.ts | 7 +- .../agent/passthrough-ollama-recovery.ts | 5 +- .../local-windows-ollama-transport.test.ts | 148 ++++-------------- .../inference-providers/ollama-local.test.ts | 13 +- 4 files changed, 40 insertions(+), 133 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index d5ef2ca1eb9..9a9ed804d81 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -64,8 +64,8 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b'"); expect(stderr).toContain("timed out"); expect(stderr).toContain("at http://host.docker.internal:11434"); - expect(stderr).toContain("Repair that Ollama route and rerun this command"); - expect(stderr).toContain("continuing to OpenClaw dispatch"); + expect(stderr).toContain("OpenClaw dispatch will continue"); + expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); }); it.each([ @@ -88,7 +88,8 @@ describe("runOllamaRestartRecovery", () => { const stderr = writes.join(""); expect(stderr).toContain(message); expect(stderr).toContain("http://host.docker.internal:11434"); - expect(stderr).toContain("Repair that Ollama route and rerun this command"); + expect(stderr).toContain("OpenClaw dispatch will continue"); + expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); }); it.each([ diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 58f437ffc26..4687c5ca201 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -47,8 +47,9 @@ function reportRecovery( } proc.stderr.write( ` Ollama warm-up for '${model}' at ${result.endpoint} ${describeWarmFailure(result.reason)} ` + - `(${result.detail}). Repair that Ollama route and rerun this command; continuing to ` + - `OpenClaw dispatch.\n`, + `(${result.detail}). OpenClaw dispatch will continue. To retry the warm-up, restore ` + + `Ollama access to ${result.endpoint} and confirm that it serves '${model}', then rerun ` + + `this command.\n`, ); return; } diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 97a5e378a96..6d552cdc0dd 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -11,7 +11,7 @@ import { clearPersistedOllamaHostIfUnused, CONTAINER_REACHABILITY_IMAGE, findReachableOllamaHost, - getLocalProviderHealthCheck, + getOllamaApiCommand, getOllamaHostForCleanup, getOllamaModelOptions, getResolvedOllamaHost, @@ -33,6 +33,27 @@ describe("Windows-host Ollama transport", () => { resetOllamaRuntimeContextWindowAutoState(); }); + it("selects Docker Desktop only for the Windows-host transport owner", () => { + expect( + getOllamaApiCommand( + ["-sf", "http://host.docker.internal:11434/api/tags"], + OLLAMA_HOST_DOCKER_INTERNAL, + ), + ).toEqual([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "-sf", + "http://host.docker.internal:11434/api/tags", + ]); + expect(getOllamaApiCommand(["-sf", "http://127.0.0.1:11434/api/tags"], "127.0.0.1")).toEqual([ + "curl", + "-sf", + "http://127.0.0.1:11434/api/tags", + ]); + }); + it("restores the accepted route for cleanup in a fresh process", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-receipt-")); try { @@ -129,7 +150,6 @@ describe("Windows-host Ollama transport", () => { it("probes persisted Windows-host health through Docker Desktop", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-health-")); - const calls: (readonly string[])[] = []; try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); resetOllamaHostCache(); @@ -138,31 +158,18 @@ describe("Windows-host Ollama transport", () => { const result = probeLocalProviderHealth("ollama-local", { findReachableOllamaHostImpl: () => OLLAMA_HOST_DOCKER_INTERNAL, loadOllamaProxyTokenImpl: () => null, - ollamaRunCaptureExImpl: (command) => { - calls.push(command); - return { - stdout: JSON.stringify({ models: [] }), - stderr: "", - exitCode: 0, - timedOut: false, - }; - }, + ollamaRunCaptureExImpl: () => ({ + stdout: JSON.stringify({ models: [] }), + stderr: "", + exitCode: 0, + timedOut: false, + }), }); expect(result).toMatchObject({ ok: true, endpoint: "http://host.docker.internal:11434/api/tags", }); - expect(calls).toHaveLength(1); - expect(calls[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "http://host.docker.internal:11434/api/tags", - ]), - ); } finally { resetOllamaHostCache(); rmSync(stateRoot, { recursive: true, force: true }); @@ -174,68 +181,27 @@ describe("Windows-host Ollama transport", () => { const capture = vi.fn(() => JSON.stringify({ models: [{ name: "qwen3.5:9b" }] })); expect(getOllamaModelOptions(capture)).toEqual(["qwen3.5:9b"]); - expect(capture).toHaveBeenCalledWith( - [ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "-sf", - "--connect-timeout", - "3", - "--max-time", - "5", - "http://host.docker.internal:11434/api/tags", - ], - { ignoreError: true }, - ); }); it("validates a Windows-host model through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn((command: readonly string[]) => - command.at(-1) === "http://host.docker.internal:11434/api/show" - ? JSON.stringify({ capabilities: ["tools"] }) - : "", - ); - const captureEx = vi.fn((command: readonly string[]) => ({ - stdout: - command[0] === "docker" && - command.at(-1) === "http://host.docker.internal:11434/api/generate" - ? JSON.stringify({ done: true, response: "ready" }) - : "", + const capture = vi.fn(() => JSON.stringify({ capabilities: ["tools"] })); + const captureEx = vi.fn(() => ({ + stdout: JSON.stringify({ done: true, response: "ready" }), stderr: "", - exitCode: command[0] === "docker" ? 0 : 7, + exitCode: 0, timedOut: false, })); expect(validateOllamaModel("qwen3.5:9b", capture, () => false, captureEx)).toEqual({ ok: true, }); - expect(captureEx).toHaveBeenCalledOnce(); - expect(captureEx.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "http://host.docker.internal:11434/api/generate", - ]), - ); }); it("validates health and container reachability through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const capture = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - expect(getLocalProviderHealthCheck("ollama-local")).toEqual([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "-sf", - "http://host.docker.internal:11434/api/tags", - ]); expect( validateLocalProvider( "ollama-local", @@ -248,27 +214,6 @@ describe("Windows-host Ollama transport", () => { }), ), ).toEqual({ ok: true }); - - expect(capture).toHaveBeenCalledTimes(2); - expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "http://host.docker.internal:11434/api/tags", - ]), - ); - expect(capture.mock.calls[1]?.[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - "--add-host", - "host.openshell.internal:host-gateway", - "http://host.openshell.internal:11434/api/tags", - ]), - ); }); it("checks the Hermes context window through Docker Desktop (#10553)", () => { @@ -291,16 +236,6 @@ describe("Windows-host Ollama transport", () => { }), ).toEqual({ ok: true }); expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); - expect(capture).toHaveBeenCalledOnce(); - expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "http://host.docker.internal:11434/api/ps", - ]), - ); }); it("checks model capability metadata through Docker Desktop (#10553)", () => { @@ -313,16 +248,6 @@ describe("Windows-host Ollama transport", () => { source: "api", supportsTools: true, }); - expect(capture).toHaveBeenCalledOnce(); - expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "http://host.docker.internal:11434/api/show", - ]), - ); }); it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => { @@ -342,14 +267,5 @@ describe("Windows-host Ollama transport", () => { ok: false, message: expect.stringContaining("cannot verify the required 64000-token window"), }); - expect(capture.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "http://host.docker.internal:11434/api/ps", - ]), - ); }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index 2a84e8c67aa..caace774584 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -135,11 +135,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("dispatches Windows-host warm-up through Docker Desktop", async () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const runCommands: unknown[] = []; - const run = vi.fn((command: unknown) => { - runCommands.push(command); - return { status: 0 }; - }); + const run = vi.fn((_command: unknown) => ({ status: 0 })); await expect( setupOllamaLocalInference( @@ -157,13 +153,6 @@ describe("Ollama local provider sandbox-facing model gate", () => { ).resolves.toEqual({ done: false }); expect(run).toHaveBeenCalledOnce(); - expect(runCommands[0]).toEqual([ - "bash", - "-c", - expect.stringMatching( - /docker.*curlimages\/curl:8\.10\.1.*host\.docker\.internal:11434\/api\/generate/, - ), - ]); }); it("fails before provider registration when the cleanup route cannot be staged", async () => { From a15ea79717632e92256c80f6e57a0ac8b5ec800a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 09:34:34 -0700 Subject: [PATCH 14/47] fix: clear Windows-host Ollama review gaps --- .../local-windows-ollama-transport.test.ts | 179 ++++++++++++++---- src/lib/inference/local.ts | 113 ++++++++--- .../inference-providers/ollama-local.test.ts | 24 ++- .../inference-providers/ollama-local.ts | 6 +- src/lib/onboard/inference-providers/types.ts | 3 +- src/lib/onboard/setup-inference.ts | 47 ++++- .../onboard-inference-reconciliation.test.ts | 40 +++- 7 files changed, 332 insertions(+), 80 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 6d552cdc0dd..6dd8c24fb45 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -10,9 +10,9 @@ import { applyOllamaRuntimeContextWindow, clearPersistedOllamaHostIfUnused, CONTAINER_REACHABILITY_IMAGE, + createOllamaApiCapture, findReachableOllamaHost, getOllamaApiCommand, - getOllamaHostForCleanup, getOllamaModelOptions, getResolvedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, @@ -22,11 +22,25 @@ import { probeOllamaModelCapabilities, resetOllamaHostCache, resetOllamaRuntimeContextWindowAutoState, + runOllamaWarmup, setResolvedOllamaHost, validateLocalProvider, validateOllamaModel, } from "./local"; +function respondsOnlyThroughDockerDesktop(apiPath: string, response: string) { + return vi.fn((command: readonly string[]) => { + const expectedUrl = `http://host.docker.internal:11434${apiPath}`; + const usesExpectedTransport = + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command[3] === CONTAINER_REACHABILITY_IMAGE && + command.includes(expectedUrl); + return usesExpectedTransport ? response : ""; + }); +} + describe("Windows-host Ollama transport", () => { afterEach(() => { resetOllamaHostCache(); @@ -54,7 +68,7 @@ describe("Windows-host Ollama transport", () => { ]); }); - it("restores the accepted route for cleanup in a fresh process", () => { + it("restores the accepted route receipt in a fresh process", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-receipt-")); try { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); @@ -62,7 +76,6 @@ describe("Windows-host Ollama transport", () => { resetOllamaHostCache(); expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); - expect(getOllamaHostForCleanup(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); } finally { rmSync(stateRoot, { recursive: true, force: true }); } @@ -90,7 +103,6 @@ describe("Windows-host Ollama transport", () => { expect(clearPersistedOllamaHostIfUnused(["nvidia-prod"], stateRoot)).toBe(true); expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); - expect(getOllamaHostForCleanup(stateRoot)).toBe("127.0.0.1"); } finally { rmSync(stateRoot, { recursive: true, force: true }); } @@ -142,7 +154,6 @@ describe("Windows-host Ollama transport", () => { resetOllamaHostCache(); expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); - expect(getOllamaHostForCleanup(stateRoot)).toBe("127.0.0.1"); } finally { rmSync(stateRoot, { recursive: true, force: true }); } @@ -155,21 +166,30 @@ describe("Windows-host Ollama transport", () => { resetOllamaHostCache(); setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const captureEx = vi.fn((command: readonly string[]) => ({ + stdout: + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command[3] === CONTAINER_REACHABILITY_IMAGE && + command.includes("http://host.docker.internal:11434/api/tags") + ? JSON.stringify({ models: [] }) + : "", + stderr: "", + exitCode: 0, + timedOut: false, + })); const result = probeLocalProviderHealth("ollama-local", { findReachableOllamaHostImpl: () => OLLAMA_HOST_DOCKER_INTERNAL, loadOllamaProxyTokenImpl: () => null, - ollamaRunCaptureExImpl: () => ({ - stdout: JSON.stringify({ models: [] }), - stderr: "", - exitCode: 0, - timedOut: false, - }), + ollamaRunCaptureExImpl: captureEx, }); expect(result).toMatchObject({ ok: true, endpoint: "http://host.docker.internal:11434/api/tags", }); + expect(captureEx).toHaveBeenCalledOnce(); } finally { resetOllamaHostCache(); rmSync(stateRoot, { recursive: true, force: true }); @@ -178,52 +198,80 @@ describe("Windows-host Ollama transport", () => { it("reads the model inventory through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn(() => JSON.stringify({ models: [{ name: "qwen3.5:9b" }] })); + const capture = respondsOnlyThroughDockerDesktop( + "/api/tags", + JSON.stringify({ models: [{ name: "qwen3.5:9b" }] }), + ); expect(getOllamaModelOptions(capture)).toEqual(["qwen3.5:9b"]); + expect(capture).toHaveBeenCalledOnce(); }); it("validates a Windows-host model through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn(() => JSON.stringify({ capabilities: ["tools"] })); - const captureEx = vi.fn(() => ({ - stdout: JSON.stringify({ done: true, response: "ready" }), - stderr: "", - exitCode: 0, - timedOut: false, - })); + const capture = respondsOnlyThroughDockerDesktop( + "/api/show", + JSON.stringify({ capabilities: ["tools"] }), + ); + const captureEx = vi.fn((command: readonly string[]) => { + const expected = + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command[3] === CONTAINER_REACHABILITY_IMAGE && + command.includes("http://host.docker.internal:11434/api/generate"); + return { + stdout: expected ? JSON.stringify({ done: true, response: "ready" }) : "", + stderr: "", + exitCode: expected ? 0 : 1, + timedOut: false, + }; + }); expect(validateOllamaModel("qwen3.5:9b", capture, () => false, captureEx)).toEqual({ ok: true, }); + expect(captureEx).toHaveBeenCalledOnce(); }); it("validates health and container reachability through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const capture = vi.fn((command: readonly string[]) => { + const usesDockerDesktop = + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command.includes(CONTAINER_REACHABILITY_IMAGE); + const endpoint = command.find((argument) => argument.startsWith("http://")); + return usesDockerDesktop && + (endpoint === "http://host.docker.internal:11434/api/tags" || + endpoint === "http://host.openshell.internal:11434/api/tags" || + endpoint === "http://host.openshell.internal:11435/api/tags") + ? JSON.stringify({ models: [] }) + : ""; + }); - expect( - validateLocalProvider( - "ollama-local", - capture, - () => {}, - () => ({ - env: {}, - isolatedCredentialConfig: false, - cleanup: () => ({ ok: true }), - }), - ), - ).toEqual({ ok: true }); + const result = validateLocalProvider( + "ollama-local", + capture, + () => {}, + () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + ); + expect(result).toEqual({ ok: true }); + expect(capture).toHaveBeenCalled(); }); it("checks the Hermes context window through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn((command: readonly string[]) => - String(command.at(-1)).endsWith("/api/ps") - ? JSON.stringify({ - models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }], - }) - : "", + const capture = respondsOnlyThroughDockerDesktop( + "/api/ps", + JSON.stringify({ + models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }], + }), ); const env: NodeJS.ProcessEnv = {}; @@ -236,11 +284,13 @@ describe("Windows-host Ollama transport", () => { }), ).toEqual({ ok: true }); expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); + expect(capture).toHaveBeenCalledOnce(); }); it("checks model capability metadata through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn((_command: readonly string[]) => + const capture = respondsOnlyThroughDockerDesktop( + "/api/show", JSON.stringify({ capabilities: ["tools"] }), ); @@ -248,6 +298,57 @@ describe("Windows-host Ollama transport", () => { source: "api", supportsTools: true, }); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("isolates Docker credentials for Windows-host API requests", () => { + const cleanup = vi.fn(() => ({ ok: true as const })); + const capture = vi.fn((_command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const isolatedCapture = createOllamaApiCapture(capture, OLLAMA_HOST_DOCKER_INTERNAL, () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })); + + expect(isolatedCapture(["curl", "-sf", "http://host.docker.internal:11434/api/tags"])).toBe( + JSON.stringify({ models: [] }), + ); + expect(capture).toHaveBeenCalledWith( + expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + { env: { DOCKER_CONFIG: "/tmp/credential-free-docker" } }, + ); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("runs Windows-host warm-up with an isolated Docker client", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const cleanup = vi.fn(() => ({ ok: true as const })); + const run = vi.fn(); + + runOllamaWarmup("qwen3.5:9b", run, () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })); + + expect(run).toHaveBeenCalledWith( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/generate", + ]), + { + ignoreError: true, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }, + ); + expect(cleanup).toHaveBeenCalledOnce(); }); it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index ae3d5b43c3d..f21a17db58a 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -27,7 +27,7 @@ import { OLLAMA_PORT, OLLAMA_PROXY_PORT, VLLM_PORT } from "../core/ports"; import { retryUntil } from "../core/retry"; import { sleepSeconds } from "../core/wait"; import { containerCanReachHostLoopback, isWsl, type WslDetectionOptions } from "../platform"; -import { type CaptureResult, runCapture, runCaptureEx, shellQuote } from "../runner"; +import { type CaptureResult, run, runCapture, runCaptureEx, shellQuote } from "../runner"; import { buildSubprocessEnv } from "../subprocess-env"; import { @@ -128,7 +128,7 @@ export { MIN_OLLAMA_VERSION, } from "./ollama-version"; -export type RunCaptureExFn = (cmd: string[]) => CaptureResult; +export type RunCaptureExFn = (cmd: string[], opts?: { env?: NodeJS.ProcessEnv }) => CaptureResult; // Hosts that local-provider discovery may try when probing Ollama. The Windows // onboarding path separately checks host.docker.internal from Docker Desktop's @@ -196,18 +196,16 @@ export function findReachableOllamaHost( // would otherwise stall the synchronous onboard probe for the OS connect // timeout (~75-130s on Linux). Matches the convention used in // getLocalProviderHealthStatus probes. - const result = capture( - getOllamaApiCommand( - [ + const result = createOllamaApiCapture(capture, host)( + [ + "curl", "-sf", "--connect-timeout", "3", "--max-time", "5", `http://${host}:${OLLAMA_PORT}/api/tags`, - ], - host, - ), + ], { ignoreError: true }, ); if (result) { @@ -275,13 +273,6 @@ export function clearPersistedOllamaHostIfUnused( return true; } -/** Resolve cleanup transport after process-local onboarding state is gone. */ -export function getOllamaHostForCleanup( - stateRoot: string = resolveSharedLocalAdapterStateRoot(), -): string { - return _resolvedOllamaHost ?? loadPersistedOllamaHost(stateRoot) ?? OLLAMA_LOCALHOST; -} - /** Keep Windows-host Ollama requests in Docker Desktop's verified network context. */ export function getOllamaApiCommand( curlArgs: readonly string[], @@ -292,14 +283,55 @@ export function getOllamaApiCommand( : ["curl", ...curlArgs]; } +function withIsolatedOllamaDockerClient( + options: { env?: NodeJS.ProcessEnv } | undefined, + execute: (options: { env?: NodeJS.ProcessEnv } | undefined) => T, + prepareDockerEnvironment: PrepareDockerEnvironmentFn, + operation: string, +): T { + const prepared = prepareDockerEnvironment(); + try { + return execute({ + ...options, + env: mergeIsolatedDockerClientEnv(options?.env ?? {}, prepared), + }); + } finally { + warnIfDockerBuildEnvironmentCleanupFailed(prepared.cleanup(), operation); + } +} + export function createOllamaApiCapture( runCaptureImpl?: RunCaptureFn, host: string = getResolvedOllamaHost(), + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, ): RunCaptureFn { const capture = runCaptureImpl ?? runCapture; return (command, options) => { const [executable, ...args] = command; - return capture(executable === "curl" ? getOllamaApiCommand(args, host) : command, options); + const translated = executable === "curl" ? getOllamaApiCommand(args, host) : command; + if (translated[0] !== "docker") return capture(translated, options); + return withIsolatedOllamaDockerClient( + options, + (isolatedOptions) => capture(translated, isolatedOptions), + prepareDockerEnvironment, + "Windows-host Ollama API request", + ); + }; +} + +function createOllamaApiCaptureEx( + runCaptureExImpl: RunCaptureExFn = runCaptureEx, + host: string = getResolvedOllamaHost(), + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, +): RunCaptureExFn { + return (command, options) => { + if (command[0] !== "docker") return runCaptureExImpl(command, options); + return withIsolatedOllamaDockerClient( + options, + (isolatedOptions) => runCaptureExImpl(command, isolatedOptions), + prepareDockerEnvironment, + "Windows-host Ollama API request", + ); }; } @@ -439,7 +471,7 @@ function runOllamaLocalCurlProbe( runCaptureExImpl: RunCaptureExFn = runCaptureEx, ): CurlProbeResult { const command = getOllamaApiCommand(buildValidatedCurlCommandArgs(["-f", ...argv]), host); - const result = runCaptureExImpl(command); + const result = createOllamaApiCaptureEx(runCaptureExImpl, host)(command); const ok = result.exitCode === 0; const stderr = String(result.stderr ?? ""); return { @@ -929,9 +961,10 @@ export function isLocalProviderHostHealthy( const command = getLocalProviderHealthCheck(provider); if (!command) return false; const capture = runCaptureImpl ?? runCapture; + const hostCapture = provider === "ollama-local" ? createOllamaApiCapture(capture) : capture; return isLocalProviderProbeOutputHealthy( command.at(-1) ?? "", - capture(command, { ignoreError: true }), + hostCapture(command, { ignoreError: true }), ); } @@ -1302,10 +1335,11 @@ export function probeOllamaEndpointInventory( host: string, runCaptureImpl?: RunCaptureFn, ): string[] | null { - const capture = runCaptureImpl ?? runCapture; + const capture = createOllamaApiCapture(runCaptureImpl, host); const body = capture( - getOllamaApiCommand( - buildValidatedCurlCommandArgs([ + [ + "curl", + ...buildValidatedCurlCommandArgs([ "-sf", "--connect-timeout", "3", @@ -1313,8 +1347,7 @@ export function probeOllamaEndpointInventory( "5", `http://${host}:${OLLAMA_PORT}/api/tags`, ]), - host, - ), + ], { ignoreError: true }, ); return parseOllamaModelInventory(body); @@ -1413,7 +1446,8 @@ export function validateLocalProvider( return { ok: true }; } - const output = capture(command, { ignoreError: true }); + const hostCapture = provider === "ollama-local" ? createOllamaApiCapture(capture) : capture; + const output = hostCapture(command, { ignoreError: true }); if (!isLocalProviderProbeOutputHealthy(command.at(-1) ?? "", output)) { switch (provider) { case "vllm-local": @@ -1738,8 +1772,8 @@ export function getOllamaModelOptions( sleepMilliseconds: (milliseconds: number) => void = (milliseconds) => sleepSeconds(milliseconds / 1_000), ): string[] { - const capture = runCaptureImpl ?? runCapture; const host = getResolvedOllamaHost(); + const capture = createOllamaApiCapture(runCaptureImpl, host); const modelDiscoveryRetryDelaysMs = [500, 1_000] as const; // Docker Desktop owns Windows-host reachability because host.docker.internal // may not resolve from WSL. Keep model discovery on the verified transport. @@ -1918,6 +1952,33 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string ]; } +export function runOllamaWarmup( + model: string, + runImpl: ( + command: readonly string[], + options?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv }, + ) => unknown = run, + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, +): void { + const windowsHost = getResolvedOllamaHost() === OLLAMA_HOST_DOCKER_INTERNAL; + const command = windowsHost + ? getOllamaWarmupRequestCommand(model) + : getOllamaWarmupCommand(model); + const execute = (options?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv }) => { + runImpl(command, { ...options, ignoreError: true }); + }; + if (!windowsHost) { + execute(); + return; + } + withIsolatedOllamaDockerClient( + undefined, + execute, + prepareDockerEnvironment, + `Windows-host Ollama warm-up for '${model}'`, + ); +} + export function getOllamaProbeCommand( model: string, timeoutSeconds = 120, @@ -1955,7 +2016,7 @@ export function validateOllamaModel( options: { allowToolsIncompatible?: boolean } = {}, ): ValidationResult { const capture = runCaptureImpl ?? runCapture; - const captureEx = runCaptureExImpl ?? runCaptureEx; + const captureEx = createOllamaApiCaptureEx(runCaptureExImpl ?? runCaptureEx); const isSpark = isSparkImpl ?? (() => detectNvidiaPlatform() === "spark"); const sparkHost = isSpark(); const probeCmd = getOllamaProbeCommand(model); diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index caace774584..d1838bcb607 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -3,9 +3,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + CONTAINER_REACHABILITY_IMAGE, getOllamaWarmupCommand, OLLAMA_HOST_DOCKER_INTERNAL, resetOllamaHostCache, + runOllamaWarmup, setResolvedOllamaHost, } from "../../inference/local"; import { setupOllamaLocalInference } from "./ollama-local"; @@ -136,6 +138,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("dispatches Windows-host warm-up through Docker Desktop", async () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const run = vi.fn((_command: unknown) => ({ status: 0 })); + const cleanup = vi.fn(() => ({ ok: true as const })); await expect( setupOllamaLocalInference( @@ -146,13 +149,32 @@ describe("Ollama local provider sandbox-facing model gate", () => { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: (model, runImpl) => + runOllamaWarmup(model, runImpl, () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })), persistResolvedOllamaHost: vi.fn(), }, }), ), ).resolves.toEqual({ done: false }); - expect(run).toHaveBeenCalledOnce(); + expect(run).toHaveBeenCalledWith( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/generate", + ]), + { + ignoreError: true, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }, + ); + expect(cleanup).toHaveBeenCalledOnce(); }); it("fails before provider registration when the cleanup route cannot be staged", async () => { diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index 5968e16b1d9..fc937b1db05 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -164,7 +164,11 @@ export async function setupOllamaLocalInference( } } else { log(` Priming Ollama model: ${model}`); - run(getOllamaWarmupCommand(model), { ignoreError: true }); + if (localInference.runOllamaWarmup) { + localInference.runOllamaWarmup(model, run); + } else { + run(getOllamaWarmupCommand(model), { ignoreError: true }); + } const probe = localInference.validateOllamaModelWithToolsOverride( model, allowToolsIncompatible, diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 0b11fbbf0ca..5021b3c4206 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -213,7 +213,7 @@ export type HermesDeps = CommonDeps & { // loosely so callers can pass either shape without casting. export type RunFn = ( cmd: any, - opts?: { ignoreError?: boolean; suppressOutput?: boolean }, + opts?: { ignoreError?: boolean; suppressOutput?: boolean; env?: NodeJS.ProcessEnv }, ) => RunResult; export type VllmDeps = CommonDeps & { @@ -255,6 +255,7 @@ export type OllamaDeps = CommonDeps & { allowToolsIncompatible: boolean, ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; + runOllamaWarmup?(model: string, runImpl: RunFn): void; persistResolvedOllamaHost?(): (() => void) | void; clearPersistedOllamaHostIfUnused?(providers: readonly (string | null | undefined)[]): boolean; }; diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 94242cc8c0b..074e498f0a3 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -24,6 +24,7 @@ import { getOllamaProxyToken, persistAndProbeOllamaProxy, startOllamaAuthProxy, + type OllamaUnloadResult, withOllamaModelOwnershipLock, } from "../inference/ollama/proxy"; import { @@ -218,7 +219,7 @@ export type SetupInferenceDeps = ProviderBranchDeps & { // by hand, so every read below must stay optional-chained. getSandbox?: typeof import("../state/registry").getSandbox; listSandboxes?: typeof import("../state/registry").listSandboxes; - unloadOllamaModels?: (onlyModels: readonly string[]) => void; + unloadOllamaModels?: (onlyModels: readonly string[]) => OllamaUnloadResult | void; withOllamaModelOwnershipLock?: typeof withOllamaModelOwnershipLock; localInferenceTimeoutSecs: number; vllmLocalCredentialEnv: string; @@ -556,6 +557,7 @@ function releaseSupersededOllamaModel( // still owns its model. if (!previous || result.retry) return; let authorityRefusal: unknown; + let cleanupWarning: string | null = null; try { const withOwnershipLock = deps.withOllamaModelOwnershipLock ?? withOllamaModelOwnershipLock; withOwnershipLock(() => { @@ -572,16 +574,43 @@ function releaseSupersededOllamaModel( authorityRefusal = error; return; } - if (superseded) deps.unloadOllamaModels?.([superseded]); + if (superseded) { + try { + const cleanup = deps.unloadOllamaModels?.([superseded]); + if (cleanup && !cleanup.ok) { + const detail = cleanup.message + ? `: ${cleanup.message.replace(/\s+/g, " ").slice(0, 240)}` + : ""; + cleanupWarning = + ` Warning: Ollama did not release superseded model '${superseded}' from ` + + `${cleanup.endpoint} (outcome: ${cleanup.outcome}${detail}). The new inference ` + + `route remains active. Restore Ollama access at ${cleanup.endpoint}, then stop or ` + + `destroy the former sandbox to retry cleanup.`; + } + } catch (error) { + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .slice(0, 240); + cleanupWarning = + ` Warning: Ollama cleanup for superseded model '${superseded}' failed: ${detail}. ` + + `The new inference route remains active. Restore Ollama access, then stop or destroy ` + + `the former sandbox to retry cleanup.`; + } + } if (retireRoute) { - deps.localInference.clearPersistedOllamaHostIfUnused?.( - peers.map((peer) => peer.provider), - ); + deps.localInference.clearPersistedOllamaHostIfUnused?.(peers.map((peer) => peer.provider)); } }); - } catch { - /* Best-effort: a failed unload must not fail an onboarding that already committed its route. */ + } catch (error) { + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .slice(0, 240); + cleanupWarning = + ` Warning: NemoClaw could not finish superseded Ollama cleanup: ${detail}. The new ` + + `inference route remains active. Restore Ollama access, then stop or destroy the former ` + + `sandbox to retry cleanup.`; } + if (cleanupWarning) console.warn(cleanupWarning); if (authorityRefusal) throw authorityRefusal; } @@ -601,9 +630,7 @@ export function createSetupInference( hermesToolGateways: string[] = [], options: ProviderInferenceSetupOptions = {}, ): Promise { - const revalidateSandboxIdentity = sandboxName - ? options.revalidateSandboxIdentity - : undefined; + const revalidateSandboxIdentity = sandboxName ? options.revalidateSandboxIdentity : undefined; const gatewayName = options.gatewayName ?? deps.getGatewayName(); const endpointSource = options.endpointSource === undefined ? "onboard" : options.endpointSource; diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index e0c08bf94ed..77fa26e5465 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -9,7 +9,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createLocalInferenceRouteApplier } from "../../src/lib/onboard/local-inference-route.js"; -import type { SetupInference } from "../../src/lib/onboard/setup-inference.js"; +import type { SetupInference, SetupInferenceDeps } from "../../src/lib/onboard/setup-inference.js"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; import { bedrockRuntimeOnboard, @@ -1016,7 +1016,7 @@ describe("re-onboard Ollama GPU release (#9110)", () => { function releaseHarness(options: { getSandbox: () => typeof priorEntry | null; sandboxes: (typeof priorEntry)[]; - unloadOllamaModels: (onlyModels: readonly string[]) => void; + unloadOllamaModels: NonNullable; applyLocalInferenceRoute?: () => Promise; clearPersistedOllamaHostIfUnused?: ( providers: readonly (string | null | undefined)[], @@ -1095,6 +1095,10 @@ describe("re-onboard Ollama GPU release (#9110)", () => { let result: Awaited>; try { result = await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("synthetic unload failure")); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("stop or destroy the former sandbox"), + ); } finally { warn.mockRestore(); } @@ -1102,6 +1106,38 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).toHaveBeenCalledWith(["llama3"]); }); + it("reports structured cleanup failure after a successful provider switch", async () => { + const unloadOllamaModels = vi.fn(() => ({ + ok: false as const, + outcome: "unload-request-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: ["llama3"], + discoveries: [], + requests: [], + message: "connection refused", + })); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [priorEntry], + unloadOllamaModels, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + let result: Awaited>; + try { + result = await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("http://host.docker.internal:11434"), + ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("unload-request-failed")); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("stop or destroy the former sandbox"), + ); + } finally { + warn.mockRestore(); + } + expect(result).toEqual({ ok: true }); + }); + it("keeps the model when the re-onboard selects the same one (#9110)", async () => { const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); const prior = { ...priorEntry, model: "qwen3.5:9b" }; From 7125b6e2333524b2b136cc8d4ae90d7c9ee74a53 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 10:22:58 -0700 Subject: [PATCH 15/47] fix: isolate every Windows Ollama operation --- .../agent/ollama-restart-recovery.test.ts | 36 +++- .../sandbox/agent/ollama-restart-recovery.ts | 23 ++- .../local-windows-ollama-transport.test.ts | 38 +++- src/lib/inference/local.ts | 109 ++++++----- src/lib/inference/ollama/proxy.test.ts | 74 ++++++-- src/lib/inference/ollama/proxy.ts | 174 +++++++++++------- .../ollama/ollama-gpu-cleanup.test.ts | 67 ++++++- 7 files changed, 381 insertions(+), 140 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index fd6d45adec3..4e8bdf699a6 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { prepareOllamaApiExecution } from "../../../inference/local"; import { maybeWarmOllamaAfterDaemonRestart, type OllamaRestartRecoveryDeps, @@ -46,8 +47,23 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("uses the persisted direct bridge route for both the default probe and warm-up", () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const cleanup = vi.fn(() => ({ ok: true as const })); + const prepareDockerEnvironment = () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }); + const runCaptureImpl = vi.fn( + (_command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const runCaptureExImpl = vi.fn((_command: string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? successfulWarmResult() + : { stdout: "", exitCode: 1, timedOut: false }, + ); expect( maybeWarmOllamaAfterDaemonRestart( @@ -56,7 +72,16 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { runCaptureImpl, runCaptureExImpl }, + { + runCaptureImpl, + runCaptureExImpl, + prepareDockerEnvironment, + prepareOllamaApiExecution: (command, host, options) => + prepareOllamaApiExecution(command, host, { + ...options, + prepareDockerEnvironment, + }), + }, ), ).toEqual({ kind: "warmed", ok: true, timedOut: false }); @@ -73,6 +98,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { stream: false, think: false, }); + expect(runCaptureImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe("/tmp/credential-free-docker"); + expect(runCaptureExImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe( + "/tmp/credential-free-docker", + ); + expect(cleanup).toHaveBeenCalledTimes(2); }); it("maps an auth-proxy route back to host loopback", () => { diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 7b800076867..fd6c78ddd6e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -24,6 +24,7 @@ import { ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, + prepareOllamaApiExecution, probeOllamaEndpointInventory, type RunCaptureFn, type RunCaptureExFn, @@ -50,6 +51,8 @@ export interface OllamaRestartRecoveryDeps { runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; runCaptureImpl?: RunCaptureFn; + prepareDockerEnvironment?: Parameters[2]; + prepareOllamaApiExecution?: typeof prepareOllamaApiExecution; } export type OllamaRestartRecoveryFailureReason = @@ -218,7 +221,11 @@ export function maybeWarmOllamaAfterDaemonRestart( const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); const rawEndpoint = `http://${rawHost}:${OLLAMA_PORT}`; const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; - const rawCapture = createOllamaApiCapture(deps.runCaptureImpl, rawHost); + const rawCapture = createOllamaApiCapture( + deps.runCaptureImpl, + rawHost, + deps.prepareDockerEnvironment, + ); let status: OllamaRuntimeModelStatus; try { status = probe(model, () => rawHost, rawCapture); @@ -234,7 +241,19 @@ export function maybeWarmOllamaAfterDaemonRestart( const captureEx = deps.runCaptureExImpl ?? runCaptureEx; try { - const result = captureEx(buildWarmCommand(model, rawHost)); + const execution = (deps.prepareOllamaApiExecution ?? prepareOllamaApiExecution)( + buildWarmCommand(model, rawHost), + rawHost, + { operation: `Ollama restart warm-up for '${model}'` }, + ); + let result; + try { + result = captureEx(execution.command, { + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } if (result.timedOut) { return { kind: "warmed", diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 6dd8c24fb45..5484b2381bb 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -262,7 +262,43 @@ describe("Windows-host Ollama transport", () => { }), ); expect(result).toEqual({ ok: true }); - expect(capture).toHaveBeenCalled(); + const endpoints = capture.mock.calls.map(([command]) => + command.find((argument: string) => argument.startsWith("http://")), + ); + expect(endpoints).toContain("http://host.docker.internal:11434/api/tags"); + expect(endpoints).toContain("http://host.openshell.internal:11434/api/tags"); + }); + + it("rejects Windows-host health when the container route is unreachable", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((command: readonly string[]) => + command.includes("http://host.docker.internal:11434/api/tags") + ? JSON.stringify({ models: [] }) + : "", + ); + + const result = validateLocalProvider( + "ollama-local", + capture, + () => {}, + () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + ); + + expect(result).toMatchObject({ + ok: false, + message: expect.stringContaining("container reachability check failed"), + }); + const endpoints = capture.mock.calls.map(([command]) => + command.find((argument: string) => argument.startsWith("http://")), + ); + expect(endpoints).toContain("http://host.docker.internal:11434/api/tags"); + expect( + endpoints.some((endpoint) => endpoint?.startsWith("http://host.openshell.internal:")), + ).toBe(true); }); it("checks the Hermes context window through Docker Desktop (#10553)", () => { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index f21a17db58a..75c8c36ee9d 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -283,21 +283,41 @@ export function getOllamaApiCommand( : ["curl", ...curlArgs]; } -function withIsolatedOllamaDockerClient( - options: { env?: NodeJS.ProcessEnv } | undefined, - execute: (options: { env?: NodeJS.ProcessEnv } | undefined) => T, - prepareDockerEnvironment: PrepareDockerEnvironmentFn, - operation: string, -): T { - const prepared = prepareDockerEnvironment(); - try { - return execute({ - ...options, - env: mergeIsolatedDockerClientEnv(options?.env ?? {}, prepared), - }); - } finally { - warnIfDockerBuildEnvironmentCleanupFailed(prepared.cleanup(), operation); +export type PreparedOllamaApiExecution = { + readonly command: string[]; + readonly env?: NodeJS.ProcessEnv; + cleanup(): void; +}; + +/** Own command translation and Docker-client isolation for one Ollama API process. */ +export function prepareOllamaApiExecution( + command: readonly string[], + host: string = getResolvedOllamaHost(), + options: { + env?: NodeJS.ProcessEnv; + operation?: string; + prepareDockerEnvironment?: PrepareDockerEnvironmentFn; + } = {}, +): PreparedOllamaApiExecution { + const [executable, ...args] = command; + const translated = executable === "curl" ? getOllamaApiCommand(args, host) : [...command]; + if (translated[0] !== "docker") { + return { command: translated, env: options.env, cleanup: () => {} }; } + const prepared = (options.prepareDockerEnvironment ?? prepareIsolatedDockerEnvironment)(); + let cleaned = false; + return { + command: translated, + env: mergeIsolatedDockerClientEnv(options.env ?? {}, prepared), + cleanup: () => { + if (cleaned) return; + cleaned = true; + warnIfDockerBuildEnvironmentCleanupFailed( + prepared.cleanup(), + options.operation ?? "Windows-host Ollama API request", + ); + }, + }; } export function createOllamaApiCapture( @@ -307,31 +327,39 @@ export function createOllamaApiCapture( ): RunCaptureFn { const capture = runCaptureImpl ?? runCapture; return (command, options) => { - const [executable, ...args] = command; - const translated = executable === "curl" ? getOllamaApiCommand(args, host) : command; - if (translated[0] !== "docker") return capture(translated, options); - return withIsolatedOllamaDockerClient( - options, - (isolatedOptions) => capture(translated, isolatedOptions), + const execution = prepareOllamaApiExecution(command, host, { + env: options?.env, prepareDockerEnvironment, - "Windows-host Ollama API request", - ); + }); + try { + return capture(execution.command, { + ...options, + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } }; } -function createOllamaApiCaptureEx( +export function createOllamaApiCaptureEx( runCaptureExImpl: RunCaptureExFn = runCaptureEx, host: string = getResolvedOllamaHost(), prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, ): RunCaptureExFn { return (command, options) => { - if (command[0] !== "docker") return runCaptureExImpl(command, options); - return withIsolatedOllamaDockerClient( - options, - (isolatedOptions) => runCaptureExImpl(command, isolatedOptions), + const execution = prepareOllamaApiExecution(command, host, { + env: options?.env, prepareDockerEnvironment, - "Windows-host Ollama API request", - ); + }); + try { + return runCaptureExImpl(execution.command, { + ...options, + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } }; } @@ -1964,19 +1992,18 @@ export function runOllamaWarmup( const command = windowsHost ? getOllamaWarmupRequestCommand(model) : getOllamaWarmupCommand(model); - const execute = (options?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv }) => { - runImpl(command, { ...options, ignoreError: true }); - }; - if (!windowsHost) { - execute(); - return; - } - withIsolatedOllamaDockerClient( - undefined, - execute, + const execution = prepareOllamaApiExecution(command, getResolvedOllamaHost(), { prepareDockerEnvironment, - `Windows-host Ollama warm-up for '${model}'`, - ); + operation: `Windows-host Ollama warm-up for '${model}'`, + }); + try { + runImpl(execution.command, { + ignoreError: true, + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } } export function getOllamaProbeCommand( diff --git a/src/lib/inference/ollama/proxy.test.ts b/src/lib/inference/ollama/proxy.test.ts index 4a4b6d7489a..ba9f367599f 100644 --- a/src/lib/inference/ollama/proxy.test.ts +++ b/src/lib/inference/ollama/proxy.test.ts @@ -32,7 +32,7 @@ function loadProxyWithMocks(setup: MockSetup): { const childProcess = require(CHILD_PROCESS_DIST) as typeof import("node:child_process"); const runner = require(RUNNER_DIST); const originalGetOllamaModelOptions = local.getOllamaModelOptions; - const originalGetOllamaWarmupCommand = local.getOllamaWarmupCommand; + const originalRunOllamaWarmup = local.runOllamaWarmup; const originalPrompt = creds.prompt; const originalProbeOllamaModelCapabilities = local.probeOllamaModelCapabilities; const originalRun = runner.run; @@ -68,9 +68,9 @@ function loadProxyWithMocks(setup: MockSetup): { capabilities: ["tools"], supportsTools: true, }); - local.getOllamaWarmupCommand = (model: string) => { + local.runOllamaWarmup = (model: string, runImpl: typeof runner.run) => { warmupModels.push(model); - return ["warmup", model]; + runImpl(["warmup", model], { ignoreError: true }); }; local.validateOllamaModel = (...args: unknown[]) => { validateCalls.push(args); @@ -95,7 +95,7 @@ function loadProxyWithMocks(setup: MockSetup): { restore() { delete require.cache[PROXY_DIST]; local.getOllamaModelOptions = originalGetOllamaModelOptions; - local.getOllamaWarmupCommand = originalGetOllamaWarmupCommand; + local.runOllamaWarmup = originalRunOllamaWarmup; creds.prompt = originalPrompt; local.probeOllamaModelCapabilities = originalProbeOllamaModelCapabilities; runner.run = originalRun; @@ -378,13 +378,17 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { host: string; hasLocalCli: boolean; httpCloseCode?: number; + isolatedDockerConfig?: string; }) { const local = require(LOCAL_DIST); const runner = require(RUNNER_DIST); const childProcess = require(CHILD_PROCESS_DIST) as typeof import("node:child_process"); const originalRunCapture = runner.runCapture; + const originalPrepareOllamaApiExecution = local.prepareOllamaApiExecution; const cliCommands: string[][] = []; const httpCommands: string[][] = []; + const httpEnvs: NodeJS.ProcessEnv[] = []; + let cleanupCalls = 0; runner.runCapture = () => (setup.hasLocalCli ? "/usr/bin/ollama" : ""); @@ -394,23 +398,42 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { cliCommands.push([String(file), ...(((args as string[]) ?? []) as string[]).map(String)]); return { status: 0, signal: null, output: [], pid: 1, stdout: "", stderr: "" } as never; }); - const spawn = vi.spyOn(childProcess, "spawn").mockImplementation((file: unknown, args) => { - httpCommands.push([String(file), ...(((args as string[]) ?? []) as string[]).map(String)]); - const child = new EventEmitter() as EventEmitter & { - stdout: PassThrough; - stderr: PassThrough; - }; - child.stdout = new PassThrough(); - child.stderr = new PassThrough(); - process.nextTick(() => { - const closeCode = setup.httpCloseCode ?? 0; - const output = closeCode === 0 ? '{"status":"success"}\n' : ""; - child.stdout.end(output, () => { - setImmediate(() => child.emit("close", closeCode)); + local.prepareOllamaApiExecution = ( + command: readonly string[], + host: string, + options: NonNullable[2]>, + ) => + originalPrepareOllamaApiExecution(command, host, { + ...options, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: setup.isolatedDockerConfig ?? "/tmp/test-docker-config" }, + isolatedCredentialConfig: true, + cleanup: () => { + cleanupCalls += 1; + return { ok: true }; + }, + }), + }); + const spawn = vi + .spyOn(childProcess, "spawn") + .mockImplementation((file: unknown, args, options) => { + httpCommands.push([String(file), ...(((args as string[]) ?? []) as string[]).map(String)]); + httpEnvs.push((options?.env ?? {}) as NodeJS.ProcessEnv); + const child = new EventEmitter() as EventEmitter & { + stdout: PassThrough; + stderr: PassThrough; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + process.nextTick(() => { + const closeCode = setup.httpCloseCode ?? 0; + const output = closeCode === 0 ? '{"status":"success"}\n' : ""; + child.stdout.end(output, () => { + setImmediate(() => child.emit("close", closeCode)); + }); }); + return child as never; }); - return child as never; - }); local.setResolvedOllamaHost(setup.host); delete require.cache[PROXY_DIST]; @@ -419,9 +442,14 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { proxy, cliCommands, httpCommands, + httpEnvs, + get cleanupCalls() { + return cleanupCalls; + }, restore() { delete require.cache[PROXY_DIST]; runner.runCapture = originalRunCapture; + local.prepareOllamaApiExecution = originalPrepareOllamaApiExecution; spawnSync.mockRestore(); spawn.mockRestore(); local.setResolvedOllamaHost(null); @@ -440,7 +468,11 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { it("pulls through Docker when the daemon resolves on the Windows host (#10553)", async () => { vi.spyOn(console, "log").mockImplementation(() => {}); - active = loadProxyForDispatch({ host: "host.docker.internal", hasLocalCli: true }); + active = loadProxyForDispatch({ + host: "host.docker.internal", + hasLocalCli: true, + isolatedDockerConfig: "/tmp/credential-free-docker", + }); const result = await active.proxy.pullOllamaModel("qwen3.5:9b"); @@ -463,6 +495,8 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { stream: true, }); expect(active.cliCommands.map((command) => command[0])).not.toContain("bash"); + expect(active.httpEnvs[0]?.DOCKER_CONFIG).toBe("/tmp/credential-free-docker"); + expect(active.cleanupCalls).toBe(1); }); it("pulls over HTTP when a loopback daemon has no local ollama binary (#7472)", async () => { diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 0c408a0b3c9..375e51946bd 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -22,24 +22,26 @@ const { redirectInheritedChildStdoutToStderr, }: typeof import("../../cli/stdout-guard") = require("../../cli/stdout-guard"); const { OLLAMA_PORT, OLLAMA_PROXY_PORT } = require("../../core/ports"); -const { isNonInteractiveEnv }: typeof import("../../core/non-interactive") = - require("../../core/non-interactive"); +const { + isNonInteractiveEnv, +}: typeof import("../../core/non-interactive") = require("../../core/non-interactive"); const { sleepMs, waitForPort } = require("../../core/wait"); -const { ensurePulledOllamaModel }: typeof import("./model-discovery") = - require("./model-discovery"); +const { + ensurePulledOllamaModel, +}: typeof import("./model-discovery") = require("./model-discovery"); const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./model-discovery"); const { getBootstrapOllamaModelOptions, findReachableOllamaHost, clearPersistedOllamaHostIfUnused, - getOllamaApiCommand, getOllamaModelOptions, - getOllamaWarmupCommand, getResolvedOllamaHost, loadPersistedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, + prepareOllamaApiExecution, probeOllamaModelCapabilities, persistResolvedOllamaHost, + runOllamaWarmup, selectDefaultOllamaModel, validateOllamaModel, } = require("../local"); @@ -539,7 +541,9 @@ function attemptStartOllamaAuthProxyWithTokenUnlocked( printProxyPortConflict(owners); } else { console.error(` Error: Ollama auth proxy exited during startup on :${OLLAMA_PROXY_PORT}.`); - console.error(" Containers will not be able to reach the inference endpoint without the proxy."); + console.error( + " Containers will not be able to reach the inference endpoint without the proxy.", + ); console.error(` Check the proxy port owner: lsof -ti :${OLLAMA_PROXY_PORT}`); } } @@ -1048,34 +1052,47 @@ function pullOllamaModelViaHttp(model: string): Promise { // The endpoint is restricted to the local Ollama hosts NemoClaw probes and // the model id is normalized before being serialized as JSON request data. - const [executable, ...args] = getOllamaApiCommand( - [ - "-sN", - "--connect-timeout", - "10", - "--max-time", - String(TIMEOUT_MS / 1000), - "-X", - "POST", - "-H", - "Content-Type: application/json", - "-d", - // codeql[js/file-access-to-http]: local-only Ollama API with a normalized model id. - body, - url, - ], - host, - ); - const proc = spawn( - executable, - args, - { + let execution; + let proc; + try { + execution = prepareOllamaApiExecution( + [ + "curl", + "-sN", + "--connect-timeout", + "10", + "--max-time", + String(TIMEOUT_MS / 1000), + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + // codeql[js/file-access-to-http]: local-only Ollama API with a normalized model id. + body, + url, + ], + host, + { + env: buildSubprocessEnv(), + operation: `Windows-host Ollama model pull for '${model}'`, + }, + ); + const [executable, ...args] = execution.command; + proc = spawn(executable, args, { stdio: ["ignore", "pipe", "pipe"], // #2616: inject NO_PROXY=localhost so the streamed pull against the // local Ollama daemon doesn't tunnel through the user's host proxy. - env: buildSubprocessEnv(), - }, - ); + env: execution.env, + }); + } catch (error) { + execution?.cleanup(); + console.error( + ` Docker request failed to start: ${error instanceof Error ? error.message : String(error)}`, + ); + resolve(false); + return; + } const readline = require("readline"); const rl = readline.createInterface({ input: proc.stdout }); @@ -1155,6 +1172,7 @@ function pullOllamaModelViaHttp(model: string): Promise { }); proc.on("error", (err: Error) => { + execution.cleanup(); finishLine(); console.error(` Pull failed to start: ${err.message}`); resolve(false); @@ -1164,6 +1182,7 @@ function pullOllamaModelViaHttp(model: string): Promise { // child's stdio streams are fully drained, ensuring readline has emitted // the final 'line' event for the trailing `success` JSON. proc.on("close", (code: number | null) => { + execution.cleanup(); finishLine(); if (sawError) { resolve(false); @@ -1173,7 +1192,9 @@ function pullOllamaModelViaHttp(model: string): Promise { // curl exit 28 covers both the connection timeout and the complete // request limit. Elapsed time distinguishes the operator actions. if (code === 28) { - console.error(httpPullTimeoutErrorHint(performance.now() - startedAtMs, TIMEOUT_MS, host)); + console.error( + httpPullTimeoutErrorHint(performance.now() - startedAtMs, TIMEOUT_MS, host), + ); } else { console.error(` Model pull exited with code ${String(code)} (network error).`); console.error(" Already-downloaded layers are kept; re-running the pull resumes them."); @@ -1320,7 +1341,7 @@ async function prepareOllamaModel( } console.log(` Loading Ollama model: ${model}`); - run(getOllamaWarmupCommand(model), { ignoreError: true }); + runOllamaWarmup(model, run); const allowToolsIncompatible = capCheck.allowToolsIncompatible === true; const result = validateOllamaModel(model, undefined, undefined, undefined, { allowToolsIncompatible, @@ -1370,6 +1391,7 @@ type OllamaUnloadOptions = { readonly maxAttempts?: number; readonly sleep?: (milliseconds: number) => void; readonly spawnSync?: typeof spawnSync; + readonly prepareOllamaApiExecution?: typeof prepareOllamaApiExecution; }; function boundedCurlError(result): string | undefined { @@ -1378,7 +1400,9 @@ function boundedCurlError(result): string | undefined { } function transientCurlFailure(status: number | null): boolean { - return status === 6 || status === 7 || status === 18 || status === 28 || status === 52 || status === 56; + return ( + status === 6 || status === 7 || status === 18 || status === 28 || status === 52 || status === 56 + ); } function defaultReleaseSleep(milliseconds: number): void { @@ -1392,21 +1416,25 @@ function discoverResidentOllamaModels( releaseHost: string, releaseEndpoint: string, spawnSyncImpl: typeof spawnSync, + prepareExecution: typeof prepareOllamaApiExecution, ): OllamaModelDiscoveryEvidence { const endpoint = `${releaseEndpoint}/api/ps`; - const [command, ...args] = getOllamaApiCommand( - ["-sS", "--fail-with-body", "--max-time", "3", endpoint], - releaseHost, - ); let result; try { - result = spawnSyncImpl( - command, - args, - // #2616: env-sanitize so an ambient HTTP proxy cannot intercept the - // loopback-only Ollama ownership and release checks. - { encoding: "utf8", env: buildSubprocessEnv() }, + const execution = prepareExecution( + ["curl", "-sS", "--fail-with-body", "--max-time", "3", endpoint], + releaseHost, + { + env: buildSubprocessEnv(), + operation: "Ollama resident-model discovery", + }, ); + const [command, ...args] = execution.command; + try { + result = spawnSyncImpl(command, args, { encoding: "utf8", env: execution.env }); + } finally { + execution.cleanup(); + } } catch (error) { return { attempt, @@ -1494,8 +1522,7 @@ function unloadOllamaModels( } else { const persistedHost = loadPersistedOllamaHost(options.ollamaHostStateRoot); releaseHost = - persistedHost ?? - findReachableOllamaHost(undefined, {}, options.ollamaHostStateRoot); + persistedHost ?? findReachableOllamaHost(undefined, {}, options.ollamaHostStateRoot); if (releaseHost && !persistedHost) { persistResolvedOllamaHost(releaseHost, options.ollamaHostStateRoot); } @@ -1512,6 +1539,7 @@ function unloadOllamaModels( } const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost!); const spawnSyncImpl = options.spawnSync ?? spawnSync; + const prepareExecution = options.prepareOllamaApiExecution ?? prepareOllamaApiExecution; const sleepImpl = options.sleep ?? defaultReleaseSleep; const maxAttempts = Math.max(1, options.maxAttempts ?? OLLAMA_RELEASE_MAX_ATTEMPTS); const discoveries: OllamaModelDiscoveryEvidence[] = []; @@ -1525,6 +1553,7 @@ function unloadOllamaModels( releaseHost, releaseEndpoint, spawnSyncImpl, + prepareExecution, ); discoveries.push(discovery); if (discovery.error) { @@ -1559,31 +1588,37 @@ function unloadOllamaModels( let retryRequest = false; for (const model of lastMatchedModels) { const endpoint = `${releaseEndpoint}/api/generate`; - const [command, ...args] = getOllamaApiCommand( - [ - "-sS", - "--fail-with-body", - "-o", - "/dev/null", - "--max-time", - "3", - "-X", - "POST", - "-H", - "Content-Type: application/json", - "-d", - JSON.stringify({ model, keep_alive: 0 }), - endpoint, - ], - releaseHost, - ); let result; try { - result = spawnSyncImpl( - command, - args, - { encoding: "utf8", env: buildSubprocessEnv() }, + const execution = prepareExecution( + [ + "curl", + "-sS", + "--fail-with-body", + "-o", + "/dev/null", + "--max-time", + "3", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + JSON.stringify({ model, keep_alive: 0 }), + endpoint, + ], + releaseHost, + { + env: buildSubprocessEnv(), + operation: `Ollama model release for '${model}'`, + }, ); + const [command, ...args] = execution.command; + try { + result = spawnSyncImpl(command, args, { encoding: "utf8", env: execution.env }); + } finally { + execution.cleanup(); + } } catch (error) { result = { status: null, @@ -1628,6 +1663,7 @@ function unloadOllamaModels( releaseHost, releaseEndpoint, spawnSyncImpl, + prepareExecution, ); discoveries.push(verification); if (verification.error) { diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index 0723699eb95..4678e6cc53f 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -5,16 +5,21 @@ import type { SpawnSyncReturns } from "node:child_process"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { OLLAMA_HOST_DOCKER_INTERNAL, persistResolvedOllamaHost, + prepareOllamaApiExecution, resetOllamaHostCache, } from "../../../src/lib/inference/local.js"; import { unloadOllamaModels as unloadOllamaModelsImpl } from "../../../src/lib/inference/ollama/proxy.js"; -type SpawnCall = { command: string; args: readonly string[] }; +type SpawnCall = { + command: string; + args: readonly string[]; + options?: { env?: NodeJS.ProcessEnv }; +}; type SpawnSync = (typeof import("node:child_process"))["spawnSync"]; type OllamaModules = { unloadOllamaModels: (typeof import("../../../src/lib/inference/ollama/proxy.ts"))["unloadOllamaModels"]; @@ -48,8 +53,12 @@ function withMockedSpawnSync( ollamaHost = "127.0.0.1", ): T | Promise { const calls: SpawnCall[] = []; - const spawnSync = ((command: string, args: readonly string[]) => { - const call = { command, args }; + const spawnSync = (( + command: string, + args: readonly string[], + options?: { env?: NodeJS.ProcessEnv }, + ) => { + const call = { command, args, options }; calls.push(call); return responder(call); }) as SpawnSync; @@ -149,6 +158,56 @@ describe("Ollama GPU cleanup", () => { ); }); + it("isolates Docker credentials for discovery, release, and verification", () => { + const calls: SpawnCall[] = []; + const cleanup = vi.fn(() => ({ ok: true as const })); + const respond = respondWithLoadedModels("llama3.2:1b"); + const spawnSync = (( + command: string, + args: readonly string[], + options?: { env?: NodeJS.ProcessEnv }, + ) => { + const call = { command, args, options }; + calls.push(call); + return options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? respond(call) + : fail("ambient Docker config used"); + }) as SpawnSync; + + const result = unloadOllamaModelsImpl(["llama3.2:1b"], { + getResolvedOllamaHost: () => OLLAMA_HOST_DOCKER_INTERNAL, + sleep: () => {}, + spawnSync, + prepareOllamaApiExecution: ( + command: Parameters[0], + host: Parameters[1], + options: NonNullable[2]>, + ) => + prepareOllamaApiExecution(command, host, { + ...options, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + }), + }); + + expect(result).toMatchObject({ ok: true, outcome: "released" }); + expect(calls).toHaveLength(3); + expect(calls.map(({ options }) => options?.env?.DOCKER_CONFIG)).toEqual([ + "/tmp/credential-free-docker", + "/tmp/credential-free-docker", + "/tmp/credential-free-docker", + ]); + expect(calls.map(({ args }) => args.at(-1))).toEqual([ + "http://host.docker.internal:11434/api/ps", + "http://host.docker.internal:11434/api/generate", + "http://host.docker.internal:11434/api/ps", + ]); + expect(cleanup).toHaveBeenCalledTimes(3); + }); + it("unloads every running model through the Ollama API", async () => { await withMockedSpawnSync( respondWithLoadedModels("llama3.1:8b", "qwen:7b"), From 6ba00d75e57b7300d732ab8d78d73cef2e20e4fd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 10:44:26 -0700 Subject: [PATCH 16/47] test: require exact Windows Ollama URLs --- .../inference/local-windows-ollama-transport.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 5484b2381bb..b6ca9d3cc47 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -36,7 +36,7 @@ function respondsOnlyThroughDockerDesktop(apiPath: string, response: string) { command[1] === "run" && command[2] === "--rm" && command[3] === CONTAINER_REACHABILITY_IMAGE && - command.includes(expectedUrl); + command.some((argument) => argument === expectedUrl); return usesExpectedTransport ? response : ""; }); } @@ -126,7 +126,9 @@ describe("Windows-host Ollama transport", () => { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); resetOllamaHostCache(); const capture = vi.fn((command: readonly string[]) => - command.includes("http://127.0.0.1:11434/api/tags") ? JSON.stringify({ models: [] }) : "", + command.some((argument) => argument === "http://127.0.0.1:11434/api/tags") + ? JSON.stringify({ models: [] }) + : "", ); expect(findReachableOllamaHost(capture, { isWsl: true }, stateRoot)).toBe("127.0.0.1"); @@ -172,7 +174,7 @@ describe("Windows-host Ollama transport", () => { command[1] === "run" && command[2] === "--rm" && command[3] === CONTAINER_REACHABILITY_IMAGE && - command.includes("http://host.docker.internal:11434/api/tags") + command.some((argument) => argument === "http://host.docker.internal:11434/api/tags") ? JSON.stringify({ models: [] }) : "", stderr: "", @@ -219,7 +221,7 @@ describe("Windows-host Ollama transport", () => { command[1] === "run" && command[2] === "--rm" && command[3] === CONTAINER_REACHABILITY_IMAGE && - command.includes("http://host.docker.internal:11434/api/generate"); + command.some((argument) => argument === "http://host.docker.internal:11434/api/generate"); return { stdout: expected ? JSON.stringify({ done: true, response: "ready" }) : "", stderr: "", @@ -272,7 +274,7 @@ describe("Windows-host Ollama transport", () => { it("rejects Windows-host health when the container route is unreachable", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); const capture = vi.fn((command: readonly string[]) => - command.includes("http://host.docker.internal:11434/api/tags") + command.some((argument) => argument === "http://host.docker.internal:11434/api/tags") ? JSON.stringify({ models: [] }) : "", ); From 40c869ac1d0938950801a5ab60c775b9b92fd4a9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 11:04:34 -0700 Subject: [PATCH 17/47] fix: preserve peer Ollama model ownership --- .../agent/passthrough-ollama-recovery.test.ts | 2 +- src/lib/actions/sandbox/destroy.ts | 56 +++++++++++-------- src/lib/inference/context-window.test.ts | 41 +++++++++++++- src/lib/inference/context-window.ts | 6 +- .../destroy-cleanup-sandbox-services.test.ts | 44 +++++++++++++-- 5 files changed, 118 insertions(+), 31 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 9a9ed804d81..8888031343b 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -136,7 +136,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("Ollama was unreachable during the restart check"); }); - it("contains an unexpected recovery exception", () => { + it("continues OpenClaw dispatch when Ollama recovery throws", () => { const { writes, proc } = makeProcMock(); expect(() => diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 51730f188b7..d7de94ff4db 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -166,14 +166,24 @@ type RunOpenshell = (args: string[], opts?: Record) => { status export type CleanupSandboxServicesDeps = { getSandbox?: typeof registry.getSandbox; + listSandboxes?: typeof registry.listSandboxes; stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void; - unloadOllamaModels?: () => OllamaUnloadResult | void; + unloadOllamaModels?: (onlyModels?: readonly string[]) => OllamaUnloadResult | void; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; googlechatWebhookTunnelPidDir?: (servicePidDir: string) => string; }; +function sameOllamaModelRef(left: string, right: string): boolean { + const normalize = (model: string) => { + const ref = model.trim(); + const lastSegment = ref.slice(ref.lastIndexOf("/") + 1); + return ref && !lastSegment.includes(":") ? `${ref}:latest` : ref; + }; + return normalize(left) === normalize(right); +} + type ShieldsTimerNeutralizeResult = { warnings?: string[]; }; @@ -224,6 +234,7 @@ export function cleanupSandboxServices( const validatedSandboxName = validateName(sandboxName, "sandbox name"); const servicesPidDir = path.resolve("/tmp", `nemoclaw-services-${validatedSandboxName}`); const getSandbox = deps.getSandbox ?? registry.getSandbox; + const listSandboxes = deps.listSandboxes ?? registry.listSandboxes; const stopAll = deps.stopAll ?? ((opts: { sandboxName: string }) => { @@ -234,11 +245,11 @@ export function cleanupSandboxServices( }); const unloadOllamaModels = deps.unloadOllamaModels ?? - (() => { + ((onlyModels?: readonly string[]) => { const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as { - unloadOllamaModels: () => OllamaUnloadResult; + unloadOllamaModels: (onlyModels?: readonly string[]) => OllamaUnloadResult; }; - return unload(); + return unload(onlyModels); }); const runOpenshell = deps.runOpenshell ?? @@ -287,31 +298,34 @@ export function cleanupSandboxServices( ); } + let ollamaCleanup: OllamaUnloadResult | void = undefined; if (stopHostServices) { // `stopAll()` already runs `unloadOllamaModels()` unconditionally — // see src/lib/tunnel/services.ts. Don't double-call here. - const cleanup = stopAll({ sandboxName: validatedSandboxName }); - if (cleanup && !cleanup.ok) { - throw new Error( - `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). ` + - "The sandbox registry and saved route were retained; repair Ollama and retry destroy.", - ); - } + ollamaCleanup = stopAll({ sandboxName: validatedSandboxName }); } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this // branch a single-sandbox destroy would leave models loaded on the GPU. const sb = getSandbox(validatedSandboxName); - if (sb?.provider?.includes("ollama")) { - const cleanup = unloadOllamaModels(); - if (cleanup && !cleanup.ok) { - throw new Error( - `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). ` + - "The sandbox registry and saved route were retained; repair Ollama and retry destroy.", - ); - } + const model = String(sb?.model ?? "").trim(); + if (sb?.provider?.includes("ollama") && model) { + const peers = listSandboxes().sandboxes.filter( + (candidate) => + candidate.name !== validatedSandboxName && candidate.provider?.includes("ollama"), + ); + const sharedModel = peers.some( + (candidate) => candidate.model && sameOllamaModelRef(model, candidate.model), + ); + if (!sharedModel) ollamaCleanup = unloadOllamaModels([model]); } } + if (ollamaCleanup && !ollamaCleanup.ok) { + throw new Error( + `Ollama model cleanup failed at ${ollamaCleanup.endpoint} (${ollamaCleanup.outcome}: ${ollamaCleanup.message ?? "no detail"}). ` + + "The sandbox registry and saved route were retained; repair Ollama and retry destroy.", + ); + } try { rmSync(servicesPidDir, { @@ -978,9 +992,7 @@ async function destroySandboxUnlocked( providers: readonly (string | null | undefined)[], ): boolean; }; - clearPersistedOllamaHostIfUnused( - remainingSandboxes.map(({ provider }) => provider), - ); + clearPersistedOllamaHostIfUnused(remainingSandboxes.map(({ provider }) => provider)); } catch (error) { console.warn( ` ${YW}⚠${R} Failed to retire the final local Ollama route receipt: ${redactDestroyError(error)}`, diff --git a/src/lib/inference/context-window.test.ts b/src/lib/inference/context-window.test.ts index e2e46d3d17c..8a1127dd5e1 100644 --- a/src/lib/inference/context-window.test.ts +++ b/src/lib/inference/context-window.test.ts @@ -6,12 +6,17 @@ import { afterEach, describe, expect, it, vi } from "vitest"; // Most tests inject deps, so these mocks replace the real inference stack under // vitest. The default-deps suite below calls through to them. vi.mock("./local", () => ({ + createOllamaApiCaptureEx: vi.fn((capture) => capture), getOllamaProbeCommand: vi.fn(() => ["curl", "ollama-probe"]), resolveOllamaRuntimeContextWindow: vi.fn(() => null), })); vi.mock("./vllm-runtime-context", () => ({ resolveVllmContextWindowFromModels: vi.fn() })); -import { getOllamaProbeCommand, resolveOllamaRuntimeContextWindow } from "./local"; +import { + createOllamaApiCaptureEx, + getOllamaProbeCommand, + resolveOllamaRuntimeContextWindow, +} from "./local"; import { type ContextWindowDeps, resolveContextWindowForModel } from "./context-window"; // The default dependencies reach ../runner through a lazy CJS require, so swap the @@ -19,7 +24,7 @@ import { type ContextWindowDeps, resolveContextWindowForModel } from "./context- type CaptureStub = { stdout: string; exitCode: number | null; timedOut: boolean }; const runner = require("../runner") as { - runCaptureEx: (cmd: readonly string[]) => CaptureStub; + runCaptureEx: (cmd: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => CaptureStub; }; function captured(timedOut = false): CaptureStub { @@ -101,6 +106,7 @@ describe("resolveContextWindowForModel default dependencies (#8974)", () => { afterEach(() => { runner.runCaptureEx = originalRunCaptureEx; + vi.mocked(createOllamaApiCaptureEx).mockImplementation((capture) => capture!); }); it("ollama-local: runs the blocking probe command, not a backgrounded warm-up", () => { @@ -134,6 +140,37 @@ describe("resolveContextWindowForModel default dependencies (#8974)", () => { expect(getOllamaProbeCommand).toHaveBeenLastCalledWith("qwen3.5:9b", 300); }); + it("ollama-local: isolates Docker credentials for the initial and retry probes", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + const cleanup = vi.fn(); + const environments: Array = []; + let attempts = 0; + runner.runCaptureEx = (_command, options) => { + environments.push(options?.env); + attempts += 1; + return captured(attempts === 1); + }; + vi.mocked(createOllamaApiCaptureEx).mockImplementation((capture) => (command, options) => { + try { + return capture!(command, { + ...options, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }); + } finally { + cleanup(); + } + }); + vi.mocked(getOllamaProbeCommand).mockReturnValue(["docker", "run", "ollama-probe"]); + vi.mocked(resolveOllamaRuntimeContextWindow).mockReturnValue(16384); + + expect(resolveContextWindowForModel("ollama-local", "qwen3.5:9b")).toBe(16384); + expect(environments).toEqual([ + { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + ]); + expect(cleanup).toHaveBeenCalledTimes(2); + }); + it("ollama-local: does not retry when the first probe fails without timing out", () => { vi.spyOn(console, "log").mockImplementation(() => {}); let attempts = 0; diff --git a/src/lib/inference/context-window.ts b/src/lib/inference/context-window.ts index d0179f16380..ee2ef459c47 100644 --- a/src/lib/inference/context-window.ts +++ b/src/lib/inference/context-window.ts @@ -12,6 +12,7 @@ import { DEFAULT_CONTEXT_WINDOW } from "./config"; import { + createOllamaApiCaptureEx, getLocalProviderHealthEndpoint, getManagedVllmProviderBinding, getOllamaProbeCommand, @@ -44,6 +45,7 @@ const defaultContextWindowDeps: ContextWindowDeps = { // Lazy require: ../runner is CJS and a top-level require fails to resolve // under the test runner. Runs only for the real (non-injected) deps. const { runCaptureEx } = require("../runner") as { runCaptureEx: RunCaptureExFn }; + const captureEx = createOllamaApiCaptureEx(runCaptureEx); console.log(` Priming Ollama model: ${model}`); // Blocking probe, the command onboarding also waits for. A backgrounded // warm-up returns before the daemon has the model resident, and `/api/ps` @@ -51,8 +53,8 @@ const defaultContextWindowDeps: ContextWindowDeps = { // model can exceed the 120 s default on unified-memory and tight-VRAM // hosts, so retry once at 300 s as onboarding does. // A connection-refused result keeps `timedOut` false and skips the retry. - if (runCaptureEx(getOllamaProbeCommand(model)).timedOut) { - runCaptureEx(getOllamaProbeCommand(model, 300)); + if (captureEx(getOllamaProbeCommand(model)).timedOut) { + captureEx(getOllamaProbeCommand(model, 300)); } }, // currentContextWindow = null → always probe (we recompute on every switch diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 3f9c45c3c86..cc8757c479e 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -14,13 +14,17 @@ import type { CleanupSandboxServicesDeps } from "../../../src/lib/actions/sandbo import { cleanupSandboxServices } from "../../../src/lib/actions/sandbox/destroy.js"; import { SANDBOX_PROVIDER_SUFFIXES } from "../../../src/lib/onboard/sandbox-provider-cleanup.js"; -type SandboxLike = { provider?: string | null } | null; +type SandboxLike = { name?: string; model?: string | null; provider?: string | null } | null; -function buildDeps(sandbox: SandboxLike): { +function buildDeps( + sandbox: SandboxLike, + peers: Exclude[] = [], +): { deps: Required< Pick< CleanupSandboxServicesDeps, | "getSandbox" + | "listSandboxes" | "stopAll" | "unloadOllamaModels" | "runOpenshell" @@ -31,21 +35,32 @@ function buildDeps(sandbox: SandboxLike): { >; stopAllCalls: Array<{ sandboxName: string }>; unloadCalls: number; + unloadArgs: Array; } { const stopAllCalls: Array<{ sandboxName: string }> = []; + const target = sandbox + ? { name: "regression-2717", model: "target-model:latest", ...sandbox } + : null; let unloadCalls = 0; + const unloadArgs: Array = []; return { stopAllCalls, + unloadArgs, get unloadCalls() { return unloadCalls; }, deps: { - getSandbox: vi.fn(() => sandbox as never), + getSandbox: vi.fn(() => target as never), + listSandboxes: vi.fn(() => ({ + sandboxes: [...(target ? [target] : []), ...peers] as never, + defaultSandbox: null, + })), stopAll: vi.fn((opts: { sandboxName: string }) => { stopAllCalls.push(opts); }), - unloadOllamaModels: vi.fn(() => { + unloadOllamaModels: vi.fn((onlyModels?: readonly string[]) => { unloadCalls += 1; + unloadArgs.push(onlyModels); }), runOpenshell: vi.fn(() => ({ status: 0 })), rmSync: vi.fn(), @@ -86,9 +101,30 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.deps.stopAll).not.toHaveBeenCalled(); expect(harness.deps.unloadOllamaModels).toHaveBeenCalledTimes(1); + expect(harness.unloadArgs).toEqual([["target-model:latest"]]); expect(harness.unloadCalls).toBe(1); }); + it("releases only the destroyed sandbox model when another Ollama sandbox uses a different model", () => { + const harness = buildDeps({ provider: "ollama-local", model: "target-model:latest" }, [ + { name: "peer", provider: "ollama-local", model: "peer-model:latest" }, + ]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.unloadArgs).toEqual([["target-model:latest"]]); + }); + + it("keeps a model that another Ollama sandbox shares", () => { + const harness = buildDeps({ provider: "ollama-local", model: "shared-model" }, [ + { name: "peer", provider: "ollama-local", model: "shared-model:latest" }, + ]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); + }); + it("preserves destroy recovery state when stopAll cannot release Ollama", () => { const harness = buildDeps({ provider: "ollama-local" }); vi.mocked(harness.deps.stopAll).mockReturnValue(cleanupFailure); From 13073167f59b966e3462893dbdf7c9f58f25561c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 11:27:55 -0700 Subject: [PATCH 18/47] refactor: retire Ollama routes through owner --- src/lib/actions/inference-set.test-support.ts | 9 ++++++--- src/lib/actions/sandbox/destroy.ts | 15 ++++++++------- src/lib/domain/sandbox/destroy.ts | 5 +++++ src/lib/inference/config.ts | 2 +- src/lib/inference/local.ts | 3 ++- src/lib/inference/ollama-model-registry.ts | 2 ++ src/lib/inference/ollama/proxy.ts | 2 -- src/lib/state/onboard-session.ts | 1 + 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 98f575c18de..261c6267bce 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { vi } from "vitest"; -import type { ValidationResult } from "../inference/local"; import type { AgentConfigTarget } from "../sandbox/config"; import type { ConfigObject, ConfigValue } from "../security/credential-filter"; import type { Session } from "../state/onboard-session"; @@ -10,6 +9,8 @@ import type { SandboxEntry } from "../state/registry"; import type { InferenceSetDeps } from "./inference-set"; import type { EnsureHttpsPinRuntimeAdapterFn } from "./inference-set-route-containment"; +type LocalValidationResult = ReturnType; + export const OPENCLAW_TARGET: AgentConfigTarget = { agentName: "openclaw", configPath: "/sandbox/.openclaw/openclaw.json", @@ -165,7 +166,7 @@ export function createDeps(options: { session?: Session | null; openshellStatus?: number; captureOpenshell?: InferenceSetDeps["captureOpenshell"]; - localValidation?: ValidationResult; + localValidation?: LocalValidationResult; localReachable?: boolean; contextWindow?: number | null; shieldsMutable?: boolean; @@ -232,7 +233,9 @@ export function createDeps(options: { }), appendAuditEntry: vi.fn(), log: vi.fn(), - validateLocalProvider: vi.fn((): ValidationResult => options.localValidation ?? { ok: true }), + validateLocalProvider: vi.fn( + (): LocalValidationResult => options.localValidation ?? { ok: true }, + ), ensureLocalProviderReachable: vi.fn(() => options.localReachable ?? true), resolveContextWindowForModel: vi.fn((_provider: string, _model: string) => options.contextWindow === undefined ? null : options.contextWindow, diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index d7de94ff4db..fa176a02003 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -6,13 +6,13 @@ import path from "node:path"; import { CLI_NAME } from "../../cli/branding"; import { G, R, YW } from "../../cli/terminal-style"; -import { isNonInteractiveEnv } from "../../core/non-interactive"; import { prompt as askPrompt } from "../../credentials/store"; import { type DestroySandboxOptions, normalizeDestroySandboxOptions, } from "../../domain/lifecycle/options"; import { + isDestroyNonInteractiveEnv, resolveDestroyGatewayCleanupDecision, shouldStopHostServicesAfterDestroy, } from "../../domain/sandbox/destroy"; @@ -39,7 +39,6 @@ import { validateName } from "../../runner"; import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import * as onboardSession from "../../state/onboard-session"; -import type { RetainedSandboxRecoveryRecord } from "../../state/onboard-session/retained-sandbox-recovery"; import { resolveNemoclawStateDir } from "../../state/paths"; import * as registry from "../../state/registry"; import { @@ -90,8 +89,8 @@ type RemoveSandboxRegistryEntryWithReceiptDeps = { function selectRetainedSandboxRecoveryAuthority( sandboxName: string, sandbox: registry.SandboxEntry | null, - records: readonly RetainedSandboxRecoveryRecord[], -): RetainedSandboxRecoveryRecord | null { + records: readonly onboardSession.RetainedSandboxRecoveryRecord[], +): onboardSession.RetainedSandboxRecoveryRecord | null { const candidates = records.filter( (record) => record.sandboxName === sandboxName && record.sandboxIdentityFingerprint !== null, ); @@ -116,7 +115,9 @@ function selectRetainedSandboxRecoveryAuthority( return observedMatches.length === 1 ? observedMatches[0]! : null; } - const matchesRegistryAuthority = (record: RetainedSandboxRecoveryRecord): boolean => { + const matchesRegistryAuthority = ( + record: onboardSession.RetainedSandboxRecoveryRecord, + ): boolean => { const pending = sandbox.pendingCreateIdentity; if (pending) { return ( @@ -202,7 +203,7 @@ type RemoveShieldsStateDeps = { async function resolveCleanupGatewayDecision(options: DestroySandboxOptions): Promise { const decision = resolveDestroyGatewayCleanupDecision(options, { - nonInteractive: isNonInteractiveEnv(), + nonInteractive: isDestroyNonInteractiveEnv(), platform: process.platform, }); if (decision === "cleanup") return true; @@ -987,7 +988,7 @@ async function destroySandboxUnlocked( if (sandbox?.provider?.includes("ollama")) { try { const remainingSandboxes = registry.listSandboxes().sandboxes; - const { clearPersistedOllamaHostIfUnused } = require("../../inference/ollama/proxy") as { + const { clearPersistedOllamaHostIfUnused } = require("../../inference/local") as { clearPersistedOllamaHostIfUnused( providers: readonly (string | null | undefined)[], ): boolean; diff --git a/src/lib/domain/sandbox/destroy.ts b/src/lib/domain/sandbox/destroy.ts index e41b430f28b..c3faa255bc5 100644 --- a/src/lib/domain/sandbox/destroy.ts +++ b/src/lib/domain/sandbox/destroy.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { parseLiveSandboxEntries } from "../../runtime-recovery"; +import { isNonInteractiveEnv } from "../../core/non-interactive"; import { resolveSandboxContainerOwner } from "./container-owner"; const ANSI_RE = /\x1b\[[0-9;]*m/g; @@ -97,6 +98,10 @@ export function shouldStopHostServicesAfterDestroy(input: { ); } +export function isDestroyNonInteractiveEnv(): boolean { + return isNonInteractiveEnv(); +} + export function shouldCleanupGatewayAfterDestroy(input: { deleteSucceededOrAlreadyGone: boolean; removedRegistryEntry: boolean; diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index 7c9f48fa6ed..e57ec9d095c 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -8,7 +8,7 @@ import { isSafeModelId, shouldSkipResponsesProbe } from "../validation"; import { isSafeLlamaCppServedModelAlias, LLAMA_CPP_CREDENTIAL_ENV } from "./llama-cpp/contract"; -import { DEFAULT_OLLAMA_MODEL } from "./local"; +import { DEFAULT_OLLAMA_MODEL_TAG as DEFAULT_OLLAMA_MODEL } from "./ollama-model-registry"; import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "./ollama/contract"; import { OPENROUTER_CREDENTIAL_ENV, OPENROUTER_PROVIDER_NAME } from "./openrouter"; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 75c8c36ee9d..e2fca1cbb91 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -39,6 +39,7 @@ import { import { detectNvidiaPlatform } from "./nim"; import { anyRegistryModelFits, + DEFAULT_OLLAMA_MODEL_TAG, effectiveGpuMemoryMB, fittableOllamaModelTags, largestFittableOllamaModelTag, @@ -112,7 +113,7 @@ function assertRegistryTag(tag: string): string { } export const SMALL_OLLAMA_MODEL = SMALLEST_OLLAMA_MODEL_TAG; -export const DEFAULT_OLLAMA_MODEL = assertRegistryTag("nemotron-3-nano:30b"); +export const DEFAULT_OLLAMA_MODEL = assertRegistryTag(DEFAULT_OLLAMA_MODEL_TAG); export const QWEN3_6_OLLAMA_MODEL = assertRegistryTag("qwen3.6:35b"); export type RunCaptureFn = ( diff --git a/src/lib/inference/ollama-model-registry.ts b/src/lib/inference/ollama-model-registry.ts index 933c2a92156..2a334f7e242 100644 --- a/src/lib/inference/ollama-model-registry.ts +++ b/src/lib/inference/ollama-model-registry.ts @@ -26,6 +26,8 @@ import type { GpuInfo } from "./local"; +export const DEFAULT_OLLAMA_MODEL_TAG = "nemotron-3-nano:30b"; + export interface OllamaModelEntry { tag: string; requiredMemoryMB: number; diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 375e51946bd..25adf6443ba 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -33,7 +33,6 @@ const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./ const { getBootstrapOllamaModelOptions, findReachableOllamaHost, - clearPersistedOllamaHostIfUnused, getOllamaModelOptions, getResolvedOllamaHost, loadPersistedOllamaHost, @@ -1709,7 +1708,6 @@ function unloadOllamaModels( export { checkOllamaModelToolSupport, ensureOllamaAuthProxy, - clearPersistedOllamaHostIfUnused, getOllamaProxyToken, getOllamaPullTimeoutMs, isProxyHealthy, diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 4a760b5132e..8398480f592 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -75,6 +75,7 @@ import { hasUnsafeHostMountTerminalText } from "./registry/host-mount"; import { nemoclawStateRoot } from "./state-root"; export { normalizePersistedSandboxHostMounts } from "./registry/host-mount"; +export type { RetainedSandboxRecoveryRecord } from "./onboard-session/retained-sandbox-recovery"; export const SESSION_VERSION = 1; export const MACHINE_SNAPSHOT_VERSION = 1; From ed3c43aef7b5317c08482b0ec5b636814b36b808 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 11:47:05 -0700 Subject: [PATCH 19/47] fix: persist scoped Ollama cleanup retries --- src/lib/actions/sandbox/destroy.ts | 62 +++++++++++-- .../inference/ollama/model-ownership.test.ts | 41 ++++++++- src/lib/inference/ollama/model-ownership.ts | 83 ++++++++++++++++++ src/lib/inference/ollama/proxy.ts | 6 ++ src/lib/onboard/inference-providers/types.ts | 3 + src/lib/onboard/setup-inference.ts | 80 ++++++++++++++--- src/lib/tunnel/services.test.ts | 10 ++- src/lib/tunnel/services.ts | 13 ++- .../onboard-inference-reconciliation.test.ts | 87 +++++++++++++++---- .../destroy-cleanup-sandbox-services.test.ts | 30 +++++++ 10 files changed, 372 insertions(+), 43 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index fa176a02003..47cd9ab26c7 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -170,6 +170,12 @@ export type CleanupSandboxServicesDeps = { listSandboxes?: typeof registry.listSandboxes; stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void; unloadOllamaModels?: (onlyModels?: readonly string[]) => OllamaUnloadResult | void; + loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; + clearPendingOllamaModelCleanup?: ( + sandboxName: string, + releasedModels?: readonly string[], + ) => void; + withOllamaModelOwnershipLock?: (operation: () => T) => T; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; @@ -252,6 +258,30 @@ export function cleanupSandboxServices( }; return unload(onlyModels); }); + const loadPendingOllamaModelCleanup = + deps.loadPendingOllamaModelCleanup ?? + ((name: string) => { + const local = require("../../inference/ollama/proxy") as { + loadPendingOllamaModelCleanup(sandboxName: string): readonly string[]; + }; + return local.loadPendingOllamaModelCleanup(name); + }); + const clearPendingOllamaModelCleanup = + deps.clearPendingOllamaModelCleanup ?? + ((name: string, releasedModels?: readonly string[]) => { + const local = require("../../inference/ollama/proxy") as { + clearPendingOllamaModelCleanup(sandboxName: string, models?: readonly string[]): void; + }; + local.clearPendingOllamaModelCleanup(name, releasedModels); + }); + const withOllamaModelOwnershipLock = + deps.withOllamaModelOwnershipLock ?? + ((operation: () => T): T => { + const proxy = require("../../inference/ollama/proxy") as { + withOllamaModelOwnershipLock(operation: () => T): T; + }; + return proxy.withOllamaModelOwnershipLock(operation); + }); const runOpenshell = deps.runOpenshell ?? ((args: string[], opts?: Record) => { @@ -308,23 +338,39 @@ export function cleanupSandboxServices( // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this // branch a single-sandbox destroy would leave models loaded on the GPU. - const sb = getSandbox(validatedSandboxName); - const model = String(sb?.model ?? "").trim(); - if (sb?.provider?.includes("ollama") && model) { + withOllamaModelOwnershipLock(() => { + const sb = getSandbox(validatedSandboxName); const peers = listSandboxes().sandboxes.filter( (candidate) => candidate.name !== validatedSandboxName && candidate.provider?.includes("ollama"), ); - const sharedModel = peers.some( - (candidate) => candidate.model && sameOllamaModelRef(model, candidate.model), + const pending = loadPendingOllamaModelCleanup(validatedSandboxName); + const currentModel = String(sb?.model ?? "").trim(); + const candidates = [ + ...pending, + ...(sb?.provider?.includes("ollama") && currentModel ? [currentModel] : []), + ].filter( + (model, index, models) => + models.findIndex((candidate) => sameOllamaModelRef(candidate, model)) === index && + !peers.some((candidate) => candidate.model && sameOllamaModelRef(model, candidate.model)), ); - if (!sharedModel) ollamaCleanup = unloadOllamaModels([model]); - } + if (candidates.length === 0) return; + ollamaCleanup = unloadOllamaModels(candidates); + if (!ollamaCleanup || ollamaCleanup.ok) { + clearPendingOllamaModelCleanup(validatedSandboxName, candidates); + } + }); } if (ollamaCleanup && !ollamaCleanup.ok) { + const recoveryAction = + ollamaCleanup.outcome === "discovery-failed" + ? `restore access to ${ollamaCleanup.endpoint}` + : ollamaCleanup.outcome === "still-resident" + ? `stop the recorded model at ${ollamaCleanup.endpoint}` + : `allow the model unload request at ${ollamaCleanup.endpoint}`; throw new Error( `Ollama model cleanup failed at ${ollamaCleanup.endpoint} (${ollamaCleanup.outcome}: ${ollamaCleanup.message ?? "no detail"}). ` + - "The sandbox registry and saved route were retained; repair Ollama and retry destroy.", + `The sandbox registry and saved route were retained; ${recoveryAction}, then retry destroy.`, ); } diff --git a/src/lib/inference/ollama/model-ownership.test.ts b/src/lib/inference/ollama/model-ownership.test.ts index 84454236390..60cb341b687 100644 --- a/src/lib/inference/ollama/model-ownership.test.ts +++ b/src/lib/inference/ollama/model-ownership.test.ts @@ -1,12 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + clearPendingOllamaModelCleanup, decideOllamaModelOwnership, exclusivelyHeldOllamaModel, + loadPendingOllamaModelCleanup, type OllamaModelHolder, + persistPendingOllamaModelCleanup, supersededOllamaModel, } from "./model-ownership"; @@ -95,9 +101,11 @@ describe("decideOllamaModelOwnership", () => { it.each([[undefined], [""], [" "]])( "returns missing-model for registry model %j (#10074)", (model) => { - expect(decideOllamaModelOwnership(holder({ model }), [holder({ model })], new Set())).toEqual({ - kind: "missing-model", - }); + expect(decideOllamaModelOwnership(holder({ model }), [holder({ model })], new Set())).toEqual( + { + kind: "missing-model", + }, + ); }, ); @@ -121,3 +129,30 @@ describe("exclusivelyHeldOllamaModel", () => { expect(exclusivelyHeldOllamaModel(holder(), [holder(), peer])).toBe("llama3"); }); }); + +describe("pending Ollama model cleanup", () => { + it("persists exact sandbox-scoped models until verified release", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-pending-ollama-cleanup-")); + try { + persistPendingOllamaModelCleanup("test-box", ["llama3", "llama3:latest"], stateRoot); + persistPendingOllamaModelCleanup("test-box", ["qwen3.5:9b"], stateRoot); + + expect(loadPendingOllamaModelCleanup("test-box", stateRoot)).toEqual([ + "llama3", + "qwen3.5:9b", + ]); + clearPendingOllamaModelCleanup("test-box", ["llama3:latest"], stateRoot); + expect(loadPendingOllamaModelCleanup("test-box", stateRoot)).toEqual(["qwen3.5:9b"]); + clearPendingOllamaModelCleanup("test-box", undefined, stateRoot); + expect(loadPendingOllamaModelCleanup("test-box", stateRoot)).toEqual([]); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("rejects an unsafe sandbox name before state access", () => { + expect(() => loadPendingOllamaModelCleanup("../peer")).toThrow( + "Invalid sandbox name for pending Ollama cleanup", + ); + }); +}); diff --git a/src/lib/inference/ollama/model-ownership.ts b/src/lib/inference/ollama/model-ownership.ts index 392edfbb21f..22bb1803e11 100644 --- a/src/lib/inference/ollama/model-ownership.ts +++ b/src/lib/inference/ollama/model-ownership.ts @@ -1,9 +1,92 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import path from "node:path"; + import type { SandboxEntry } from "../../state/registry"; +import { + readLocalAdapterJsonFile, + removeLocalAdapterFile, + resolveSharedLocalAdapterStateRoot, + writeLocalAdapterJsonFile, +} from "../local-adapter-lifecycle"; import { ollamaModelRefsMatch } from "./model-discovery"; +const PENDING_CLEANUP_DIRECTORY = "ollama-pending-model-cleanup"; +const SAFE_SANDBOX_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; + +function pendingCleanupPath(sandboxName: string, stateRoot: string): string { + if (!SAFE_SANDBOX_NAME.test(sandboxName)) { + throw new Error(`Invalid sandbox name for pending Ollama cleanup: ${sandboxName}`); + } + return path.join(stateRoot, PENDING_CLEANUP_DIRECTORY, `${sandboxName}.json`); +} + +function validPendingModel(value: unknown): value is string { + return ( + typeof value === "string" && + value === value.trim() && + value.length > 0 && + Buffer.byteLength(value, "utf8") <= 512 && + !/[\u0000\r\n]/u.test(value) + ); +} + +export function loadPendingOllamaModelCleanup( + sandboxName: string, + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): readonly string[] { + const record = readLocalAdapterJsonFile(pendingCleanupPath(sandboxName, stateRoot)); + return record?.schemaVersion === 1 && Array.isArray(record.models) + ? record.models.filter(validPendingModel) + : []; +} + +export function persistPendingOllamaModelCleanup( + sandboxName: string, + models: readonly string[], + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + const pending = [...loadPendingOllamaModelCleanup(sandboxName, stateRoot)]; + for (const model of models) { + const normalized = model.trim(); + if (!validPendingModel(normalized)) continue; + if (!pending.some((existing) => ollamaModelRefsMatch(existing, normalized))) { + pending.push(normalized); + } + } + if (pending.length === 0) return; + writeLocalAdapterJsonFile(pendingCleanupPath(sandboxName, stateRoot), { + schemaVersion: 1, + sandboxName, + models: pending, + }); +} + +export function clearPendingOllamaModelCleanup( + sandboxName: string, + releasedModels?: readonly string[], + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + const receiptPath = pendingCleanupPath(sandboxName, stateRoot); + if (!releasedModels) { + removeLocalAdapterFile(receiptPath); + return; + } + const remaining = loadPendingOllamaModelCleanup(sandboxName, stateRoot).filter( + (pending) => !releasedModels.some((released) => ollamaModelRefsMatch(pending, released)), + ); + if (remaining.length === 0) { + removeLocalAdapterFile(receiptPath); + return; + } + writeLocalAdapterJsonFile(receiptPath, { + schemaVersion: 1, + sandboxName, + models: remaining, + }); +} + /** The registry fields an Ollama GPU-release decision reads. */ export type OllamaModelHolder = Pick; diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 25adf6443ba..4d8b329adac 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -30,6 +30,10 @@ const { ensurePulledOllamaModel, }: typeof import("./model-discovery") = require("./model-discovery"); const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./model-discovery"); +const { + clearPendingOllamaModelCleanup, + loadPendingOllamaModelCleanup, +}: typeof import("./model-ownership") = require("./model-ownership"); const { getBootstrapOllamaModelOptions, findReachableOllamaHost, @@ -1707,11 +1711,13 @@ function unloadOllamaModels( export { checkOllamaModelToolSupport, + clearPendingOllamaModelCleanup, ensureOllamaAuthProxy, getOllamaProxyToken, getOllamaPullTimeoutMs, isProxyHealthy, killStaleProxy, + loadPendingOllamaModelCleanup, noAuthProxy, persistAndProbeOllamaProxy, persistProxyToken, diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 5021b3c4206..1e6ff017e16 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -256,6 +256,9 @@ export type OllamaDeps = CommonDeps & { ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; runOllamaWarmup?(model: string, runImpl: RunFn): void; + loadPendingOllamaModelCleanup?(sandboxName: string): readonly string[]; + persistPendingOllamaModelCleanup?(sandboxName: string, models: readonly string[]): void; + clearPendingOllamaModelCleanup?(sandboxName: string, releasedModels?: readonly string[]): void; persistResolvedOllamaHost?(): (() => void) | void; clearPersistedOllamaHostIfUnused?(providers: readonly (string | null | undefined)[]): boolean; }; diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 074e498f0a3..efb2caf0bb2 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -19,7 +19,13 @@ import { withModelRouterPortLifecycleLock, } from "../inference/gateway-route-mutation-lock"; import { getManagedVllmProviderBinding } from "../inference/local"; -import { type OllamaModelHolder, supersededOllamaModel } from "../inference/ollama/model-ownership"; +import { + clearPendingOllamaModelCleanup, + loadPendingOllamaModelCleanup, + type OllamaModelHolder, + persistPendingOllamaModelCleanup, + supersededOllamaModel, +} from "../inference/ollama/model-ownership"; import { getOllamaProxyToken, persistAndProbeOllamaProxy, @@ -558,57 +564,105 @@ function releaseSupersededOllamaModel( if (!previous || result.retry) return; let authorityRefusal: unknown; let cleanupWarning: string | null = null; + let attemptedModels: readonly string[] = []; + const loadPending = + deps.localInference.loadPendingOllamaModelCleanup ?? loadPendingOllamaModelCleanup; + const persistPending = + deps.localInference.persistPendingOllamaModelCleanup ?? persistPendingOllamaModelCleanup; + const clearPending = + deps.localInference.clearPendingOllamaModelCleanup ?? clearPendingOllamaModelCleanup; + const persistRetry = (): string | null => { + if (attemptedModels.length === 0) return null; + try { + persistPending(previous.name, attemptedModels); + return null; + } catch (error) { + return (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .slice(0, 240); + } + }; try { const withOwnershipLock = deps.withOllamaModelOwnershipLock ?? withOllamaModelOwnershipLock; withOwnershipLock(() => { const peers = deps.listSandboxes?.().sandboxes ?? []; const superseded = supersededOllamaModel(previous, nextModel, peers); + const pending = loadPending(previous.name); + const retryablePending = pending.filter((model) => + supersededOllamaModel( + { name: previous.name, provider: "ollama-local", model }, + nextModel, + peers, + ), + ); + attemptedModels = [...new Set([...(superseded ? [superseded] : []), ...retryablePending])]; const retireRoute = !!previous.provider?.includes("ollama") && !nextProvider.includes("ollama") && !peers.some((peer) => peer.provider?.includes("ollama")); - if (!superseded && !retireRoute) return; + if (attemptedModels.length === 0 && !retireRoute) return; try { revalidateSandboxIdentity?.("release the superseded Ollama model"); } catch (error) { authorityRefusal = error; return; } - if (superseded) { + if (attemptedModels.length > 0 && deps.unloadOllamaModels) { try { - const cleanup = deps.unloadOllamaModels?.([superseded]); + const cleanup = deps.unloadOllamaModels(attemptedModels); if (cleanup && !cleanup.ok) { + const persistenceFailure = persistRetry(); const detail = cleanup.message ? `: ${cleanup.message.replace(/\s+/g, " ").slice(0, 240)}` : ""; + const recoveryAction = + cleanup.outcome === "discovery-failed" + ? `Restore access to ${cleanup.endpoint}` + : cleanup.outcome === "still-resident" + ? `Stop the recorded model at ${cleanup.endpoint}` + : `Allow the model unload request at ${cleanup.endpoint}`; cleanupWarning = - ` Warning: Ollama did not release superseded model '${superseded}' from ` + + ` Warning: Ollama did not release recorded model cleanup for '${previous.name}' from ` + `${cleanup.endpoint} (outcome: ${cleanup.outcome}${detail}). The new inference ` + - `route remains active. Restore Ollama access at ${cleanup.endpoint}, then stop or ` + - `destroy the former sandbox to retry cleanup.`; + `route remains active. ${recoveryAction}, then re-run onboarding, stop, or destroy ` + + `'${previous.name}' to retry only: ${attemptedModels.join(", ")}.` + + (persistenceFailure + ? ` Cleanup retry state could not be recorded: ${persistenceFailure}.` + : ""); + } else { + clearPending(previous.name, attemptedModels); } } catch (error) { + const persistenceFailure = persistRetry(); const detail = (error instanceof Error ? error.message : String(error)) .replace(/\s+/g, " ") .slice(0, 240); cleanupWarning = - ` Warning: Ollama cleanup for superseded model '${superseded}' failed: ${detail}. ` + - `The new inference route remains active. Restore Ollama access, then stop or destroy ` + - `the former sandbox to retry cleanup.`; + ` Warning: Ollama cleanup for '${previous.name}' failed: ${detail}. The new inference ` + + `route remains active. Re-run onboarding, stop, or destroy '${previous.name}' to ` + + `retry only the recorded models: ${attemptedModels.join(", ")}.` + + (persistenceFailure + ? ` Cleanup retry state could not be recorded: ${persistenceFailure}.` + : ""); } } - if (retireRoute) { + const pendingAfterCleanup = loadPending(previous.name); + if (retireRoute && !cleanupWarning && pendingAfterCleanup.length === 0) { deps.localInference.clearPersistedOllamaHostIfUnused?.(peers.map((peer) => peer.provider)); } }); } catch (error) { + const persistenceFailure = persistRetry(); const detail = (error instanceof Error ? error.message : String(error)) .replace(/\s+/g, " ") .slice(0, 240); cleanupWarning = ` Warning: NemoClaw could not finish superseded Ollama cleanup: ${detail}. The new ` + - `inference route remains active. Restore Ollama access, then stop or destroy the former ` + - `sandbox to retry cleanup.`; + `inference route remains active. Re-run onboarding, stop, or destroy '${previous.name}' to ` + + `retry only the recorded models: ${attemptedModels.join(", ") || "none"}.` + + (persistenceFailure + ? ` Cleanup retry state could not be recorded: ${persistenceFailure}.` + : ""); } if (cleanupWarning) console.warn(cleanupWarning); if (authorityRefusal) throw authorityRefusal; diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index b83df9bd62b..c26a3c04891 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -604,8 +604,14 @@ describe("stopAll", () => { it("runs injected Ollama cleanup before reporting services stopped", () => { const cleanup = vi.fn(); + const clearPendingOllamaModelCleanup = vi.fn(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir, unloadOllamaModels: cleanup }); + stopAll({ + pidDir, + sandboxName: "test-box", + unloadOllamaModels: cleanup, + clearPendingOllamaModelCleanup, + }); const stoppedCallIndex = logSpy.mock.calls.findIndex(([message]) => String(message).includes("All services stopped"), ); @@ -613,6 +619,7 @@ describe("stopAll", () => { logSpy.mockRestore(); expect(cleanup).toHaveBeenCalledOnce(); + expect(clearPendingOllamaModelCleanup).toHaveBeenCalledWith("test-box"); expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(stoppedCallOrder ?? 0); }); @@ -634,6 +641,7 @@ describe("stopAll", () => { expect(output).toContain("Ollama model cleanup failed at http://host.docker.internal:11434"); expect(output).toContain("saved local route was retained"); + expect(output).toContain("restore access to http://host.docker.internal:11434"); expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); expect(output).not.toContain("All services stopped"); }); diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index a03635b0ab5..99556a4cb7a 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -20,6 +20,7 @@ import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding" import { isObjectRecord } from "../core/json-types"; import { DASHBOARD_PORT } from "../core/ports"; import { + clearPendingOllamaModelCleanup as clearDefaultPendingOllamaModelCleanup, unloadOllamaModels as unloadDefaultOllamaModels, type OllamaUnloadResult, } from "../inference/ollama/proxy"; @@ -49,6 +50,8 @@ export interface ServiceOptions { processControl?: ProcessControl; /** Injectable Ollama model cleanup for tests. */ unloadOllamaModels?: () => OllamaUnloadResult | void; + /** Injectable retirement of sandbox-scoped Ollama cleanup recovery. */ + clearPendingOllamaModelCleanup?: (sandboxName: string) => void; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ cloudflareTunnelToken?: string; /** Also release the managed host gateway port (legacy full-stop only). */ @@ -534,8 +537,16 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { if (cleanup && !cleanup.ok) { ollamaCleanupIncomplete = true; warn( - `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; repair Ollama and retry this command.`, + `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; ${ + cleanup.outcome === "discovery-failed" + ? `restore access to ${cleanup.endpoint}` + : cleanup.outcome === "still-resident" + ? `stop the recorded model at ${cleanup.endpoint}` + : `allow the model unload request at ${cleanup.endpoint}` + }, then retry this command.`, ); + } else if (sandboxName) { + (opts.clearPendingOllamaModelCleanup ?? clearDefaultPendingOllamaModelCleanup)(sandboxName); } } catch (error) { ollamaCleanupIncomplete = true; diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index 77fa26e5465..2941c2ecdff 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -1015,12 +1015,18 @@ describe("re-onboard Ollama GPU release (#9110)", () => { function releaseHarness(options: { getSandbox: () => typeof priorEntry | null; - sandboxes: (typeof priorEntry)[]; + sandboxes: (typeof priorEntry)[] | (() => (typeof priorEntry)[]); unloadOllamaModels: NonNullable; applyLocalInferenceRoute?: () => Promise; clearPersistedOllamaHostIfUnused?: ( providers: readonly (string | null | undefined)[], ) => boolean; + loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; + persistPendingOllamaModelCleanup?: (sandboxName: string, models: readonly string[]) => void; + clearPendingOllamaModelCleanup?: ( + sandboxName: string, + releasedModels?: readonly string[], + ) => void; }) { return createDirectSetupInferenceHarness({ runOpenshell: (args) => @@ -1035,17 +1041,20 @@ describe("re-onboard Ollama GPU release (#9110)", () => { persistAndProbeOllamaProxy: async () => {}, applyLocalInferenceRoute: options.applyLocalInferenceRoute, getSandbox: options.getSandbox, - listSandboxes: () => ({ sandboxes: options.sandboxes, defaultSandbox: null }), + listSandboxes: () => ({ + sandboxes: + typeof options.sandboxes === "function" ? options.sandboxes() : options.sandboxes, + defaultSandbox: null, + }), unloadOllamaModels: options.unloadOllamaModels, - ...(options.clearPersistedOllamaHostIfUnused - ? { - localInference: { - validateOllamaModelWithToolsOverride: () => ({ ok: true }), - validateSandboxFacingOllamaModel: () => ({ ok: true }), - clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, - }, - } - : {}), + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, + loadPendingOllamaModelCleanup: options.loadPendingOllamaModelCleanup ?? (() => []), + persistPendingOllamaModelCleanup: options.persistPendingOllamaModelCleanup ?? (() => {}), + clearPendingOllamaModelCleanup: options.clearPendingOllamaModelCleanup ?? (() => {}), + }, }, }); } @@ -1096,9 +1105,7 @@ describe("re-onboard Ollama GPU release (#9110)", () => { try { result = await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); expect(warn).toHaveBeenCalledWith(expect.stringContaining("synthetic unload failure")); - expect(warn).toHaveBeenCalledWith( - expect.stringContaining("stop or destroy the former sandbox"), - ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("retry only the recorded models")); } finally { warn.mockRestore(); } @@ -1129,15 +1136,61 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect.stringContaining("http://host.docker.internal:11434"), ); expect(warn).toHaveBeenCalledWith(expect.stringContaining("unload-request-failed")); - expect(warn).toHaveBeenCalledWith( - expect.stringContaining("stop or destroy the former sandbox"), - ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("Allow the model unload request")); } finally { warn.mockRestore(); } expect(result).toEqual({ ok: true }); }); + it("persists a failed superseded cleanup and retries only that model on re-onboard", async () => { + let current = priorEntry; + let pending: readonly string[] = []; + const persistPendingOllamaModelCleanup = vi.fn((_sandboxName, models: readonly string[]) => { + pending = models; + }); + const clearPendingOllamaModelCleanup = vi.fn( + (_sandboxName, releasedModels?: readonly string[]) => { + pending = releasedModels ? pending.filter((model) => !releasedModels.includes(model)) : []; + }, + ); + const unloadOllamaModels = vi + .fn>() + .mockReturnValueOnce({ + ok: false, + outcome: "unload-request-failed", + endpoint: "http://host.docker.internal:11434", + selectedModels: ["llama3"], + discoveries: [], + requests: [], + message: "connection refused", + }) + .mockReturnValueOnce(undefined); + const harness = releaseHarness({ + getSandbox: () => current, + sandboxes: () => [current], + unloadOllamaModels, + loadPendingOllamaModelCleanup: () => pending, + persistPendingOllamaModelCleanup, + clearPendingOllamaModelCleanup, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + expect(pending).toEqual(["llama3"]); + current = { ...priorEntry, model: "qwen3.5:9b" }; + + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + } finally { + warn.mockRestore(); + } + + expect(unloadOllamaModels).toHaveBeenNthCalledWith(1, ["llama3"]); + expect(unloadOllamaModels).toHaveBeenNthCalledWith(2, ["llama3"]); + expect(clearPendingOllamaModelCleanup).toHaveBeenCalledWith("test-box", ["llama3"]); + expect(pending).toEqual([]); + }); + it("keeps the model when the re-onboard selects the same one (#9110)", async () => { const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); const prior = { ...priorEntry, model: "qwen3.5:9b" }; diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index cc8757c479e..29f4bc3e032 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -27,6 +27,9 @@ function buildDeps( | "listSandboxes" | "stopAll" | "unloadOllamaModels" + | "loadPendingOllamaModelCleanup" + | "clearPendingOllamaModelCleanup" + | "withOllamaModelOwnershipLock" | "runOpenshell" | "rmSync" | "stopGooglechatWebhookTunnel" @@ -62,6 +65,9 @@ function buildDeps( unloadCalls += 1; unloadArgs.push(onlyModels); }), + loadPendingOllamaModelCleanup: vi.fn(() => []), + clearPendingOllamaModelCleanup: vi.fn(), + withOllamaModelOwnershipLock: (operation) => operation(), runOpenshell: vi.fn(() => ({ status: 0 })), rmSync: vi.fn(), stopGooglechatWebhookTunnel: vi.fn(() => "/tmp/nemoclaw-services-regression-2717-googlechat"), @@ -125,6 +131,30 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); }); + it("retries a pending superseded model after the sandbox route changes", () => { + const harness = buildDeps({ provider: "nvidia-prod", model: "new-model" }); + vi.mocked(harness.deps.loadPendingOllamaModelCleanup).mockReturnValue(["old-model"]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.unloadArgs).toEqual([["old-model"]]); + expect(harness.deps.clearPendingOllamaModelCleanup).toHaveBeenCalledWith("regression-2717", [ + "old-model", + ]); + }); + + it("keeps a pending superseded model that an Ollama peer shares", () => { + const harness = buildDeps({ provider: "nvidia-prod", model: "new-model" }, [ + { name: "peer", provider: "ollama-local", model: "old-model:latest" }, + ]); + vi.mocked(harness.deps.loadPendingOllamaModelCleanup).mockReturnValue(["old-model"]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); + expect(harness.deps.clearPendingOllamaModelCleanup).not.toHaveBeenCalled(); + }); + it("preserves destroy recovery state when stopAll cannot release Ollama", () => { const harness = buildDeps({ provider: "ollama-local" }); vi.mocked(harness.deps.stopAll).mockReturnValue(cleanupFailure); From 9382e36776006faabcc7214c4eca9f8539ce48af Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 12:11:46 -0700 Subject: [PATCH 20/47] fix: propagate Ollama cleanup exceptions --- src/lib/actions/sandbox/destroy.ts | 24 ++++++++++--------- .../local-windows-ollama-transport.test.ts | 2 +- src/lib/inference/ollama/proxy.ts | 1 + src/lib/tunnel/services.test.ts | 24 +++++++++++++++++++ src/lib/tunnel/services.ts | 12 +++++++++- .../destroy-cleanup-sandbox-services.test.ts | 3 +++ 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 47cd9ab26c7..a535188226d 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -176,21 +176,13 @@ export type CleanupSandboxServicesDeps = { releasedModels?: readonly string[], ) => void; withOllamaModelOwnershipLock?: (operation: () => T) => T; + ollamaModelRefsMatch?: (left: string, right: string) => boolean; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; googlechatWebhookTunnelPidDir?: (servicePidDir: string) => string; }; -function sameOllamaModelRef(left: string, right: string): boolean { - const normalize = (model: string) => { - const ref = model.trim(); - const lastSegment = ref.slice(ref.lastIndexOf("/") + 1); - return ref && !lastSegment.includes(":") ? `${ref}:latest` : ref; - }; - return normalize(left) === normalize(right); -} - type ShieldsTimerNeutralizeResult = { warnings?: string[]; }; @@ -282,6 +274,14 @@ export function cleanupSandboxServices( }; return proxy.withOllamaModelOwnershipLock(operation); }); + const ollamaModelRefsMatch = + deps.ollamaModelRefsMatch ?? + ((left: string, right: string) => { + const proxy = require("../../inference/ollama/proxy") as { + ollamaModelRefsMatch(leftModel: string, rightModel: string): boolean; + }; + return proxy.ollamaModelRefsMatch(left, right); + }); const runOpenshell = deps.runOpenshell ?? ((args: string[], opts?: Record) => { @@ -351,8 +351,10 @@ export function cleanupSandboxServices( ...(sb?.provider?.includes("ollama") && currentModel ? [currentModel] : []), ].filter( (model, index, models) => - models.findIndex((candidate) => sameOllamaModelRef(candidate, model)) === index && - !peers.some((candidate) => candidate.model && sameOllamaModelRef(model, candidate.model)), + models.findIndex((candidate) => ollamaModelRefsMatch(candidate, model)) === index && + !peers.some( + (candidate) => candidate.model && ollamaModelRefsMatch(model, candidate.model), + ), ); if (candidates.length === 0) return; ollamaCleanup = unloadOllamaModels(candidates); diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index b6ca9d3cc47..85b3250f47e 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -161,7 +161,7 @@ describe("Windows-host Ollama transport", () => { } }); - it("probes persisted Windows-host health through Docker Desktop", () => { + it("probes a resolved Windows-host route through Docker Desktop", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-health-")); try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 4d8b329adac..6088b464bf5 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -1719,6 +1719,7 @@ export { killStaleProxy, loadPendingOllamaModelCleanup, noAuthProxy, + ollamaModelRefsMatch, persistAndProbeOllamaProxy, persistProxyToken, prepareOllamaModel, diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index c26a3c04891..84687074bbf 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -645,6 +645,30 @@ describe("stopAll", () => { expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); expect(output).not.toContain("All services stopped"); }); + + it("returns a failed cleanup result when Ollama cleanup throws", () => { + const clearPendingOllamaModelCleanup = vi.fn(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + const result = stopAll({ + pidDir, + sandboxName: "test-box", + unloadOllamaModels: () => { + throw new Error("synthetic cleanup exception"); + }, + clearPendingOllamaModelCleanup, + }); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(result).toMatchObject({ + ok: false, + outcome: "discovery-failed", + message: "synthetic cleanup exception", + }); + expect(output).toContain("restore access to the saved local Ollama endpoint"); + expect(clearPendingOllamaModelCleanup).not.toHaveBeenCalled(); + }); }); // #6212: after cloudflared yields a public URL, startAll must register that diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 99556a4cb7a..a1bd8daff39 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -550,8 +550,18 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { } } catch (error) { ollamaCleanupIncomplete = true; + const detail = error instanceof Error ? error.message : String(error); + ollamaCleanup = { + ok: false, + outcome: "discovery-failed", + endpoint: "the saved local Ollama endpoint", + selectedModels: [], + discoveries: [], + requests: [], + message: detail, + }; warn( - `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, + `Ollama model cleanup failed unexpectedly: ${detail}. The saved local route was retained; restore access to the saved local Ollama endpoint, then retry this command.`, ); } diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 29f4bc3e032..8d6e37a43d5 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -12,6 +12,7 @@ import { describe, expect, it, vi } from "vitest"; import type { CleanupSandboxServicesDeps } from "../../../src/lib/actions/sandbox/destroy.js"; import { cleanupSandboxServices } from "../../../src/lib/actions/sandbox/destroy.js"; +import { ollamaModelRefsMatch } from "../../../src/lib/inference/ollama/model-discovery.js"; import { SANDBOX_PROVIDER_SUFFIXES } from "../../../src/lib/onboard/sandbox-provider-cleanup.js"; type SandboxLike = { name?: string; model?: string | null; provider?: string | null } | null; @@ -30,6 +31,7 @@ function buildDeps( | "loadPendingOllamaModelCleanup" | "clearPendingOllamaModelCleanup" | "withOllamaModelOwnershipLock" + | "ollamaModelRefsMatch" | "runOpenshell" | "rmSync" | "stopGooglechatWebhookTunnel" @@ -68,6 +70,7 @@ function buildDeps( loadPendingOllamaModelCleanup: vi.fn(() => []), clearPendingOllamaModelCleanup: vi.fn(), withOllamaModelOwnershipLock: (operation) => operation(), + ollamaModelRefsMatch, runOpenshell: vi.fn(() => ({ status: 0 })), rmSync: vi.fn(), stopGooglechatWebhookTunnel: vi.fn(() => "/tmp/nemoclaw-services-regression-2717-googlechat"), From 715c315c10f189aaeb26523035c6daecdb2bc750 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 12:24:08 -0700 Subject: [PATCH 21/47] fix(inference): preserve Ollama cleanup recovery Signed-off-by: Prekshi Vyas --- .../rebuild-local-provider-recreate.test.ts | 184 +++++++++--------- .../inference-providers/ollama-local.test.ts | 68 ++++--- .../inference-providers/ollama-local.ts | 6 +- src/lib/onboard/inference-providers/types.ts | 2 +- src/lib/onboard/setup-inference.test.ts | 1 + src/lib/tunnel/services.test.ts | 18 ++ src/lib/tunnel/services.ts | 22 ++- ...board-host-local-inference-routing.test.ts | 1 + .../onboard-inference-reconciliation.test.ts | 1 + .../destroy-cleanup-sandbox-services.test.ts | 13 ++ 10 files changed, 186 insertions(+), 130 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index cd02dc35e5d..b0840a1a118 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -93,6 +93,7 @@ const localProviderScenarios = [ localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => () => {}, }, OLLAMA_PROXY_CREDENTIAL_ENV: "NEMOCLAW_OLLAMA_PROXY_TOKEN", ...unusedCommonInferenceDeps, @@ -139,101 +140,96 @@ function makeRouteApplier() { installRebuildFlowTestHooks({ acceptThirdPartySoftware: true }); describe("rebuild local-provider recreation", () => { - it.each( - localProviderScenarios, - )("recreates a missing $provider gateway provider through the resumed local setup path", async ({ - provider, - model, - baseUrl, - credentialEnv, - setup, - }) => { - let sourceDeleted = false; - let harness!: RebuildFlowHarness; - let setupResult: SetupResult | undefined; - harness = createRebuildFlowHarness({ - sandboxEntry: { provider, model, credentialEnv: null }, - onboard: async (session) => { - const callsBeforeSetup = harness.runOpenshellSpy.mock.calls.map( - (call) => call[0] as string[], - ); - expect(callsBeforeSetup).not.toContainEqual(["provider", "get", provider]); - expect(session.provider).toBe(provider); - expect(session.model).toBe(model); - expect(session.steps.provider_selection.status).toBe("pending"); - expect(session.steps.inference.status).toBe("pending"); + it.each(localProviderScenarios)( + "recreates a missing $provider gateway provider through the resumed local setup path", + async ({ provider, model, baseUrl, credentialEnv, setup }) => { + let sourceDeleted = false; + let harness!: RebuildFlowHarness; + let setupResult: SetupResult | undefined; + harness = createRebuildFlowHarness({ + sandboxEntry: { provider, model, credentialEnv: null }, + onboard: async (session) => { + const callsBeforeSetup = harness.runOpenshellSpy.mock.calls.map( + (call) => call[0] as string[], + ); + expect(callsBeforeSetup).not.toContainEqual(["provider", "get", provider]); + expect(session.provider).toBe(provider); + expect(session.model).toBe(model); + expect(session.steps.provider_selection.status).toBe("pending"); + expect(session.steps.inference.status).toBe("pending"); - setupResult = await setup(makeRouteApplier()); - }, - }); - harness.session.provider = provider; - harness.session.model = model; - harness.runOpenshellSpy.mockImplementation((args: string[]) => { - sourceDeleted ||= args.join(" ") === "sandbox delete -g nemoclaw alpha"; - return args[0] === "sandbox" && args[1] === "get" - ? { - status: 1, - stdout: "", - stderr: "sandbox alpha not found", - } - : { - status: args[0] === "provider" && args[1] === "get" ? 1 : 0, - stdout: "", - stderr: "", - }; - }); - const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; - harness.captureOpenshellSpy.mockImplementation((args: unknown) => { - const argv = Array.isArray(args) ? args.map(String) : []; - return argv.join(" ") === "sandbox get -g nemoclaw alpha" && !sourceDeleted - ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } - : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; - }); + setupResult = await setup(makeRouteApplier()); + }, + }); + harness.session.provider = provider; + harness.session.model = model; + harness.runOpenshellSpy.mockImplementation((args: string[]) => { + sourceDeleted ||= args.join(" ") === "sandbox delete -g nemoclaw alpha"; + return args[0] === "sandbox" && args[1] === "get" + ? { + status: 1, + stdout: "", + stderr: "sandbox alpha not found", + } + : { + status: args[0] === "provider" && args[1] === "get" ? 1 : 0, + stdout: "", + stderr: "", + }; + }); + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + harness.captureOpenshellSpy.mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args.map(String) : []; + return argv.join(" ") === "sandbox get -g nemoclaw alpha" && !sourceDeleted + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } + : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; + }); - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); - const calls = harness.runOpenshellSpy.mock.calls.map((call) => call[0] as string[]); - const deleteCall = calls.findIndex( - (args) => args.join(" ") === "sandbox delete -g nemoclaw alpha", - ); - const providerLookup = calls.findIndex( - (args) => args[0] === "provider" && args[1] === "get" && args[2] === provider, - ); - expect(setupResult).toEqual({ done: false }); - expect(harness.onboardSpy).toHaveBeenCalledWith( - expect.objectContaining({ resume: true, nonInteractive: true, recreateSandbox: true }), - ); - expect(deleteCall).toBeGreaterThanOrEqual(0); - expect(providerLookup).toBeGreaterThan(deleteCall); - expect(calls).toContainEqual(["provider", "get", provider]); - expect(calls).toContainEqual([ - "provider", - "create", - "--name", - provider, - "--type", - "openai", - "--credential", - credentialEnv, - "--config", - `OPENAI_BASE_URL=${baseUrl}`, - ]); - expect(calls).toContainEqual([ - "inference", - "set", - "--no-verify", - "--provider", - provider, - "--model", - model, - "--timeout", - "30", - ]); - expect(calls.some((args) => args[0] === "provider" && args[1] === "update")).toBe(false); - expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith("alpha", harness.backupPath, { - targetAgentType: "openclaw", - }); - }); + const calls = harness.runOpenshellSpy.mock.calls.map((call) => call[0] as string[]); + const deleteCall = calls.findIndex( + (args) => args.join(" ") === "sandbox delete -g nemoclaw alpha", + ); + const providerLookup = calls.findIndex( + (args) => args[0] === "provider" && args[1] === "get" && args[2] === provider, + ); + expect(setupResult).toEqual({ done: false }); + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ resume: true, nonInteractive: true, recreateSandbox: true }), + ); + expect(deleteCall).toBeGreaterThanOrEqual(0); + expect(providerLookup).toBeGreaterThan(deleteCall); + expect(calls).toContainEqual(["provider", "get", provider]); + expect(calls).toContainEqual([ + "provider", + "create", + "--name", + provider, + "--type", + "openai", + "--credential", + credentialEnv, + "--config", + `OPENAI_BASE_URL=${baseUrl}`, + ]); + expect(calls).toContainEqual([ + "inference", + "set", + "--no-verify", + "--provider", + provider, + "--model", + model, + "--timeout", + "30", + ]); + expect(calls.some((args) => args[0] === "provider" && args[1] === "update")).toBe(false); + expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith("alpha", harness.backupPath, { + targetAgentType: "openclaw", + }); + }, + ); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index d1838bcb607..d32dc177971 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -1,11 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { CONTAINER_REACHABILITY_IMAGE, getOllamaWarmupCommand, + loadPersistedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, + persistResolvedOllamaHost, resetOllamaHostCache, runOllamaWarmup, setResolvedOllamaHost, @@ -22,7 +27,12 @@ const SANDBOX_ENDPOINT_MISMATCH = "sandbox reaches through http://host.openshell.internal:11434 does not serve it " + "(reported models: qwen3.5:2b, gemma4:26b)."; -function deps(overrides: Partial = {}): OllamaDeps { +type OllamaDepsOverrides = Omit, "localInference"> & { + localInference?: Partial; +}; + +function deps(overrides: OllamaDepsOverrides = {}): OllamaDeps { + const { localInference, ...rest } = overrides; return { runOpenshell: vi.fn(() => ({ status: 0 })), upsertProvider: vi.fn(() => ({ ok: true })), @@ -48,9 +58,11 @@ function deps(overrides: Partial = {}): OllamaDeps { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => () => {}, + ...localInference, }, OLLAMA_PROXY_CREDENTIAL_ENV: CREDENTIAL_ENV, - ...overrides, + ...rest, }; } @@ -109,30 +121,32 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("records the route when the sandbox endpoint serves the model", async () => { const upsertProvider = vi.fn(() => ({ ok: true })); - const persistResolvedOllamaHost = vi.fn(); - - await expect( - setupOllamaLocalInference( - { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, - deps({ - upsertProvider, - localInference: { - validateOllamaModelWithToolsOverride: () => ({ ok: true }), - validateSandboxFacingOllamaModel: () => ({ ok: true }), - persistResolvedOllamaHost, - }, - }), - ), - ).resolves.toEqual({ done: false }); - - expect(upsertProvider).toHaveBeenCalledWith( - "ollama-local", - "openai", - CREDENTIAL_ENV, - "http://host.openshell.internal:11434/v1", - { [CREDENTIAL_ENV]: "ollama" }, - ); - expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-provider-route-")); + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + try { + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + upsertProvider, + localInference: { + persistResolvedOllamaHost: () => persistResolvedOllamaHost(undefined, stateRoot), + }, + }), + ), + ).resolves.toEqual({ done: false }); + + expect(upsertProvider).toHaveBeenCalledWith( + "ollama-local", + "openai", + CREDENTIAL_ENV, + "http://host.openshell.internal:11434/v1", + { [CREDENTIAL_ENV]: "ollama" }, + ); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } }); it("dispatches Windows-host warm-up through Docker Desktop", async () => { @@ -155,7 +169,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { isolatedCredentialConfig: true, cleanup, })), - persistResolvedOllamaHost: vi.fn(), + persistResolvedOllamaHost: vi.fn(() => () => {}), }, }), ), diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index fc937b1db05..4a1a539aac1 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -95,9 +95,9 @@ export async function setupOllamaLocalInference( await persistAndProbeOllamaProxy(proxyToken); } } - let rollbackPersistedOllamaHost: (() => void) | undefined; + let rollbackPersistedOllamaHost: () => void; try { - rollbackPersistedOllamaHost = localInference.persistResolvedOllamaHost?.() ?? undefined; + rollbackPersistedOllamaHost = localInference.persistResolvedOllamaHost(); } catch (persistError) { error( ` Could not stage the selected local Ollama route for later stop/destroy cleanup: ${ @@ -108,7 +108,7 @@ export async function setupOllamaLocalInference( } const rollbackCleanupRoute = (): boolean => { try { - rollbackPersistedOllamaHost?.(); + rollbackPersistedOllamaHost(); return true; } catch (rollbackError) { error( diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 1e6ff017e16..645914845e3 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -259,7 +259,7 @@ export type OllamaDeps = CommonDeps & { loadPendingOllamaModelCleanup?(sandboxName: string): readonly string[]; persistPendingOllamaModelCleanup?(sandboxName: string, models: readonly string[]): void; clearPendingOllamaModelCleanup?(sandboxName: string, releasedModels?: readonly string[]): void; - persistResolvedOllamaHost?(): (() => void) | void; + persistResolvedOllamaHost(): () => void; clearPersistedOllamaHostIfUnused?(providers: readonly (string | null | undefined)[]): boolean; }; /** Exact provider-owned proof used instead of legacy host warmup/probes. */ diff --git a/src/lib/onboard/setup-inference.test.ts b/src/lib/onboard/setup-inference.test.ts index d7a3deeba7b..d300ae310fd 100644 --- a/src/lib/onboard/setup-inference.test.ts +++ b/src/lib/onboard/setup-inference.test.ts @@ -313,6 +313,7 @@ describe("createProviderReviewDeps", () => { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => () => {}, }, OLLAMA_PROXY_CREDENTIAL_ENV: "NEMOCLAW_OLLAMA_PROXY_TOKEN", }, diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index c26a3c04891..0f84d88ddae 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -645,6 +645,24 @@ describe("stopAll", () => { expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); expect(output).not.toContain("All services stopped"); }); + + it("propagates an unexpected Ollama cleanup failure after stopping services (#10553)", () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(() => + stopAll({ + pidDir, + unloadOllamaModels: () => { + throw new Error("transport failed\nwith unbounded detail"); + }, + }), + ).toThrow("Ollama model cleanup failed unexpectedly: transport failed with unbounded detail"); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); + expect(output).not.toContain("All services stopped"); + }); }); // #6212: after cloudflared yields a public URL, startAll must register that diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 99556a4cb7a..ca1065f4e27 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -530,6 +530,7 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { let ollamaCleanupIncomplete = false; let ollamaCleanup: OllamaUnloadResult | undefined; + let ollamaCleanupError: Error | undefined; try { const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; const cleanup = unloadOllamaModels(); @@ -550,10 +551,21 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { } } catch (error) { ollamaCleanupIncomplete = true; - warn( - `Ollama model cleanup failed unexpectedly: ${error instanceof Error ? error.message : String(error)}. Retry this command after repairing Ollama.`, + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .trim() + .slice(0, 300); + ollamaCleanupError = new Error( + `Ollama model cleanup failed unexpectedly: ${detail || "unknown error"}. ` + + "Retry this command after repairing Ollama.", + { cause: error }, ); + warn(ollamaCleanupError.message); } + const finishOllamaCleanup = (): OllamaUnloadResult | void => { + if (ollamaCleanupError) throw ollamaCleanupError; + return ollamaCleanup; + }; // Stop host-side services only when their state directory is explicit or // derived from a trusted sandbox name. An invalid requested sandbox must not @@ -586,12 +598,12 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { "Hint: rerun with NEMOCLAW_GATEWAY_PORT= to release that gateway, or 'openshell gateway list' to find it.", ); info("Host services stopped; managed gateway not released."); - return ollamaCleanup; + return finishOllamaCleanup(); } if (gatewayOutcome === "unconfirmed") { info("Host services stopped; managed gateway release was not confirmed."); - return ollamaCleanup; + return finishOllamaCleanup(); } if (ollamaCleanupIncomplete) { @@ -599,7 +611,7 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { } else { info("All services stopped."); } - return ollamaCleanup; + return finishOllamaCleanup(); } /** diff --git a/test/onboarding/onboard-host-local-inference-routing.test.ts b/test/onboarding/onboard-host-local-inference-routing.test.ts index a8a3196671d..9d989103284 100644 --- a/test/onboarding/onboard-host-local-inference-routing.test.ts +++ b/test/onboarding/onboard-host-local-inference-routing.test.ts @@ -499,6 +499,7 @@ describe("onboard host-local inference routing", () => { localInference: { validateOllamaModelWithToolsOverride: legacyOllamaProof, validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => () => {}, }, verifyInferenceRoute: verify, verifyOnboardInferenceSmoke: smoke, diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index 2941c2ecdff..99f34516e0c 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -1050,6 +1050,7 @@ describe("re-onboard Ollama GPU release (#9110)", () => { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => () => {}, clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, loadPendingOllamaModelCleanup: options.loadPendingOllamaModelCleanup ?? (() => []), persistPendingOllamaModelCleanup: options.persistPendingOllamaModelCleanup ?? (() => {}), diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 29f4bc3e032..19aa9ef089a 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -165,6 +165,19 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.deps.rmSync).not.toHaveBeenCalled(); }); + it("preserves destroy recovery state when stopAll throws unexpectedly (#10553)", () => { + const harness = buildDeps({ provider: "ollama-local" }); + vi.mocked(harness.deps.stopAll).mockImplementation(() => { + throw new Error("unexpected cleanup failure"); + }); + + expect(() => + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps), + ).toThrow("unexpected cleanup failure"); + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + expect(harness.deps.runOpenshell).not.toHaveBeenCalled(); + }); + it("preserves destroy recovery state when scoped Ollama release fails", () => { const harness = buildDeps({ provider: "ollama-local" }); vi.mocked(harness.deps.unloadOllamaModels).mockReturnValue(cleanupFailure); From 6eb158b0fbb7aec9311173b95c67ae9faec7cbb4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 12:51:37 -0700 Subject: [PATCH 22/47] test: allow policy rollback under coverage --- src/lib/actions/sandbox/policy-channel-conflict.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/policy-channel-conflict.test.ts b/src/lib/actions/sandbox/policy-channel-conflict.test.ts index 640fe2eca9b..bf6f2209ba3 100644 --- a/src/lib/actions/sandbox/policy-channel-conflict.test.ts +++ b/src/lib/actions/sandbox/policy-channel-conflict.test.ts @@ -590,7 +590,7 @@ describe("addSandboxChannel cross-sandbox conflict check (#4305)", () => { ); expect(removePresetMock).toHaveBeenCalledWith("alpha", "telegram"); - }); + }, 15_000); // Scenario 5b it("different hash on the other sandbox is NOT a conflict (no warning, add proceeds)", async () => { From e7e858215e3b2cc8de8a2f308d96f86ef71926d4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 13:09:35 -0700 Subject: [PATCH 23/47] fix: preserve scoped Ollama cleanup recovery --- .../local-windows-ollama-transport.test.ts | 6 +-- src/lib/onboard/setup-inference.ts | 37 ++++++++++--------- .../onboard-inference-reconciliation.test.ts | 35 ++++++++++++++++++ 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 85b3250f47e..5a43d4e8fdc 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -33,9 +33,9 @@ function respondsOnlyThroughDockerDesktop(apiPath: string, response: string) { const expectedUrl = `http://host.docker.internal:11434${apiPath}`; const usesExpectedTransport = command[0] === "docker" && - command[1] === "run" && - command[2] === "--rm" && - command[3] === CONTAINER_REACHABILITY_IMAGE && + command.includes("run") && + command.includes("--rm") && + command.includes(CONTAINER_REACHABILITY_IMAGE) && command.some((argument) => argument === expectedUrl); return usesExpectedTransport ? response : ""; }); diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index efb2caf0bb2..0c82dfa41b5 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -565,6 +565,7 @@ function releaseSupersededOllamaModel( let authorityRefusal: unknown; let cleanupWarning: string | null = null; let attemptedModels: readonly string[] = []; + let pendingRecordFailure: string | null = null; const loadPending = deps.localInference.loadPendingOllamaModelCleanup ?? loadPendingOllamaModelCleanup; const persistPending = @@ -608,10 +609,13 @@ function releaseSupersededOllamaModel( return; } if (attemptedModels.length > 0 && deps.unloadOllamaModels) { + // The committed route no longer names the old model. Record it before + // release so later lifecycle commands retain a scoped retry target. + pendingRecordFailure = persistRetry(); try { const cleanup = deps.unloadOllamaModels(attemptedModels); if (cleanup && !cleanup.ok) { - const persistenceFailure = persistRetry(); + if (pendingRecordFailure) pendingRecordFailure = persistRetry(); const detail = cleanup.message ? `: ${cleanup.message.replace(/\s+/g, " ").slice(0, 240)}` : ""; @@ -624,26 +628,24 @@ function releaseSupersededOllamaModel( cleanupWarning = ` Warning: Ollama did not release recorded model cleanup for '${previous.name}' from ` + `${cleanup.endpoint} (outcome: ${cleanup.outcome}${detail}). The new inference ` + - `route remains active. ${recoveryAction}, then re-run onboarding, stop, or destroy ` + - `'${previous.name}' to retry only: ${attemptedModels.join(", ")}.` + - (persistenceFailure - ? ` Cleanup retry state could not be recorded: ${persistenceFailure}.` - : ""); + `route remains active. ${recoveryAction}. ` + + (pendingRecordFailure + ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ")} at ${cleanup.endpoint}.` + : `Re-run onboarding, stop, or destroy '${previous.name}' to retry only: ${attemptedModels.join(", ")}.`); } else { clearPending(previous.name, attemptedModels); } } catch (error) { - const persistenceFailure = persistRetry(); + if (pendingRecordFailure) pendingRecordFailure = persistRetry(); const detail = (error instanceof Error ? error.message : String(error)) .replace(/\s+/g, " ") .slice(0, 240); cleanupWarning = ` Warning: Ollama cleanup for '${previous.name}' failed: ${detail}. The new inference ` + - `route remains active. Re-run onboarding, stop, or destroy '${previous.name}' to ` + - `retry only the recorded models: ${attemptedModels.join(", ")}.` + - (persistenceFailure - ? ` Cleanup retry state could not be recorded: ${persistenceFailure}.` - : ""); + `route remains active. ` + + (pendingRecordFailure + ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ")} from the saved local Ollama endpoint.` + : `Re-run onboarding, stop, or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ")}.`); } } const pendingAfterCleanup = loadPending(previous.name); @@ -652,17 +654,16 @@ function releaseSupersededOllamaModel( } }); } catch (error) { - const persistenceFailure = persistRetry(); + if (!pendingRecordFailure) pendingRecordFailure = persistRetry(); const detail = (error instanceof Error ? error.message : String(error)) .replace(/\s+/g, " ") .slice(0, 240); cleanupWarning = ` Warning: NemoClaw could not finish superseded Ollama cleanup: ${detail}. The new ` + - `inference route remains active. Re-run onboarding, stop, or destroy '${previous.name}' to ` + - `retry only the recorded models: ${attemptedModels.join(", ") || "none"}.` + - (persistenceFailure - ? ` Cleanup retry state could not be recorded: ${persistenceFailure}.` - : ""); + `inference route remains active. ` + + (pendingRecordFailure + ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ") || "the superseded model"} from the saved local Ollama endpoint.` + : `Re-run onboarding, stop, or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ") || "none"}.`); } if (cleanupWarning) console.warn(cleanupWarning); if (authorityRefusal) throw authorityRefusal; diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index 2941c2ecdff..c530eb66813 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -1191,6 +1191,41 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(pending).toEqual([]); }); + it("names manual cleanup when a superseded-model retry record cannot be written", async () => { + const persistPendingOllamaModelCleanup = vi.fn(() => { + throw new Error("state directory is unavailable"); + }); + const unloadOllamaModels = vi.fn(() => ({ + ok: false as const, + outcome: "unload-request-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: ["llama3"], + discoveries: [], + requests: [], + message: "connection refused", + })); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [priorEntry], + unloadOllamaModels, + persistPendingOllamaModelCleanup, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + const warning = warn.mock.calls.map(([message]) => String(message)).join("\n"); + expect(warning).toContain("Manually release only llama3"); + expect(warning).toContain("http://host.docker.internal:11434"); + expect(warning).not.toContain("Re-run onboarding, stop, or destroy"); + } finally { + warn.mockRestore(); + } + + expect(persistPendingOllamaModelCleanup.mock.invocationCallOrder[0]).toBeLessThan( + unloadOllamaModels.mock.invocationCallOrder[0] ?? 0, + ); + }); + it("keeps the model when the re-onboard selects the same one (#9110)", async () => { const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); const prior = { ...priorEntry, model: "qwen3.5:9b" }; From b2929f54f21ea16b38dc946c47c218268fbc17d2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 13:27:11 -0700 Subject: [PATCH 24/47] test: stabilize readiness deadline budget --- src/lib/onboard/sandbox-readiness-tracing.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 005345b4e40..cead6a01d25 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -80,6 +80,7 @@ describe("createSandboxReadyWaiter", () => { target: TARGET, isLinuxDockerDriverGatewayEnabled: () => true, sleep, + now: () => 0, }); await expect(waitForSandboxReady(NAME, 2, 3)).resolves.toEqual({ From 11855ea4633611cf0726d6b2706813d8a0237109 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 13:39:46 -0700 Subject: [PATCH 25/47] test: use readiness clock fix from base --- src/lib/onboard/sandbox-readiness-tracing.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index cead6a01d25..005345b4e40 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -80,7 +80,6 @@ describe("createSandboxReadyWaiter", () => { target: TARGET, isLinuxDockerDriverGatewayEnabled: () => true, sleep, - now: () => 0, }); await expect(waitForSandboxReady(NAME, 2, 3)).resolves.toEqual({ From 735203b9ddf331ba2a1beee12b4faf7417beb50f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 13:46:06 -0700 Subject: [PATCH 26/47] fix: retain Ollama cleanup when host is unavailable --- src/lib/inference/ollama/proxy.ts | 5 +++-- test/inference/ollama/ollama-gpu-cleanup.test.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 6088b464bf5..4cd8f01bb50 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -1532,12 +1532,13 @@ function unloadOllamaModels( } if (!releaseHost) { return { - ok: true, - outcome: "not-resident", + ok: false, + outcome: "discovery-failed", endpoint: buildLocalOllamaEndpoint(), selectedModels: selectedModels ?? [], discoveries: [], requests: [], + message: "No reachable local Ollama endpoint was found for cleanup", }; } const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost!); diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index 4678e6cc53f..58e6d867954 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -158,6 +158,22 @@ describe("Ollama GPU cleanup", () => { ); }); + it("retains cleanup recovery when no local Ollama endpoint is reachable", () => { + const result = unloadOllamaModelsImpl(["llama3.2:1b"], { + getResolvedOllamaHost: () => null, + }); + + expect(result).toMatchObject({ + ok: false, + outcome: "discovery-failed", + endpoint: "http://127.0.0.1:11434", + selectedModels: ["llama3.2:1b"], + discoveries: [], + requests: [], + message: "No reachable local Ollama endpoint was found for cleanup", + }); + }); + it("isolates Docker credentials for discovery, release, and verification", () => { const calls: SpawnCall[] = []; const cleanup = vi.fn(() => ({ ok: true as const })); From 7e605208cbe51b81d9ef22f5a2419f65d36cdfe0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 14:13:09 -0700 Subject: [PATCH 27/47] fix: scope final Ollama cleanup to model owners --- src/lib/actions/sandbox/destroy.ts | 24 +++++-- src/lib/tunnel/services.test.ts | 20 ++++++ src/lib/tunnel/services.ts | 62 ++++++++++--------- .../destroy-cleanup-sandbox-services.test.ts | 40 +++++++++--- 4 files changed, 103 insertions(+), 43 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index a535188226d..4e692a11496 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -168,7 +168,10 @@ type RunOpenshell = (args: string[], opts?: Record) => { status export type CleanupSandboxServicesDeps = { getSandbox?: typeof registry.getSandbox; listSandboxes?: typeof registry.listSandboxes; - stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void; + stopAll?: (opts: { + sandboxName: string; + cleanupOllamaModels?: boolean; + }) => OllamaUnloadResult | void; unloadOllamaModels?: (onlyModels?: readonly string[]) => OllamaUnloadResult | void; loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; clearPendingOllamaModelCleanup?: ( @@ -236,9 +239,12 @@ export function cleanupSandboxServices( const listSandboxes = deps.listSandboxes ?? registry.listSandboxes; const stopAll = deps.stopAll ?? - ((opts: { sandboxName: string }) => { + ((opts: { sandboxName: string; cleanupOllamaModels?: boolean }) => { const services = require("../../tunnel/services") as { - stopAll: (opts: { sandboxName: string }) => OllamaUnloadResult | void; + stopAll: (opts: { + sandboxName: string; + cleanupOllamaModels?: boolean; + }) => OllamaUnloadResult | void; }; return services.stopAll(opts); }); @@ -331,9 +337,15 @@ export function cleanupSandboxServices( let ollamaCleanup: OllamaUnloadResult | void = undefined; if (stopHostServices) { - // `stopAll()` already runs `unloadOllamaModels()` unconditionally — - // see src/lib/tunnel/services.ts. Don't double-call here. - ollamaCleanup = stopAll({ sandboxName: validatedSandboxName }); + // `stopAll()` owns the host-wide unload when this sandbox has an Ollama + // route or retained cleanup work. Don't probe an unrelated daemon for a + // sandbox with no Ollama ownership, and don't double-call cleanup here. + const cleanupOllamaModels = withOllamaModelOwnershipLock(() => { + const sandbox = getSandbox(validatedSandboxName); + const pending = loadPendingOllamaModelCleanup(validatedSandboxName); + return Boolean(sandbox?.provider?.includes("ollama") || pending.length > 0); + }); + ollamaCleanup = stopAll({ sandboxName: validatedSandboxName, cleanupOllamaModels }); } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index 84687074bbf..396ae2d0b56 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -623,6 +623,26 @@ describe("stopAll", () => { expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(stoppedCallOrder ?? 0); }); + it("skips Ollama cleanup when the scoped caller proves no model ownership", () => { + const cleanup = vi.fn(); + const clearPendingOllamaModelCleanup = vi.fn(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + stopAll({ + pidDir, + sandboxName: "test-box", + cleanupOllamaModels: false, + unloadOllamaModels: cleanup, + clearPendingOllamaModelCleanup, + }); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(cleanup).not.toHaveBeenCalled(); + expect(clearPendingOllamaModelCleanup).not.toHaveBeenCalled(); + expect(output).toContain("All services stopped"); + }); + it("reports Ollama cleanup failure and retains its recovery route", () => { const failure = { ok: false as const, diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index a1bd8daff39..99a54532f5b 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -50,6 +50,8 @@ export interface ServiceOptions { processControl?: ProcessControl; /** Injectable Ollama model cleanup for tests. */ unloadOllamaModels?: () => OllamaUnloadResult | void; + /** Whether this scoped stop owns Ollama models that require cleanup. Defaults to true. */ + cleanupOllamaModels?: boolean; /** Injectable retirement of sandbox-scoped Ollama cleanup recovery. */ clearPendingOllamaModelCleanup?: (sandboxName: string) => void; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ @@ -530,39 +532,41 @@ export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { let ollamaCleanupIncomplete = false; let ollamaCleanup: OllamaUnloadResult | undefined; - try { - const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; - const cleanup = unloadOllamaModels(); - if (cleanup) ollamaCleanup = cleanup; - if (cleanup && !cleanup.ok) { + if (opts.cleanupOllamaModels !== false) { + try { + const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; + const cleanup = unloadOllamaModels(); + if (cleanup) ollamaCleanup = cleanup; + if (cleanup && !cleanup.ok) { + ollamaCleanupIncomplete = true; + warn( + `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; ${ + cleanup.outcome === "discovery-failed" + ? `restore access to ${cleanup.endpoint}` + : cleanup.outcome === "still-resident" + ? `stop the recorded model at ${cleanup.endpoint}` + : `allow the model unload request at ${cleanup.endpoint}` + }, then retry this command.`, + ); + } else if (sandboxName) { + (opts.clearPendingOllamaModelCleanup ?? clearDefaultPendingOllamaModelCleanup)(sandboxName); + } + } catch (error) { ollamaCleanupIncomplete = true; + const detail = error instanceof Error ? error.message : String(error); + ollamaCleanup = { + ok: false, + outcome: "discovery-failed", + endpoint: "the saved local Ollama endpoint", + selectedModels: [], + discoveries: [], + requests: [], + message: detail, + }; warn( - `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; ${ - cleanup.outcome === "discovery-failed" - ? `restore access to ${cleanup.endpoint}` - : cleanup.outcome === "still-resident" - ? `stop the recorded model at ${cleanup.endpoint}` - : `allow the model unload request at ${cleanup.endpoint}` - }, then retry this command.`, + `Ollama model cleanup failed unexpectedly: ${detail}. The saved local route was retained; restore access to the saved local Ollama endpoint, then retry this command.`, ); - } else if (sandboxName) { - (opts.clearPendingOllamaModelCleanup ?? clearDefaultPendingOllamaModelCleanup)(sandboxName); } - } catch (error) { - ollamaCleanupIncomplete = true; - const detail = error instanceof Error ? error.message : String(error); - ollamaCleanup = { - ok: false, - outcome: "discovery-failed", - endpoint: "the saved local Ollama endpoint", - selectedModels: [], - discoveries: [], - requests: [], - message: detail, - }; - warn( - `Ollama model cleanup failed unexpectedly: ${detail}. The saved local route was retained; restore access to the saved local Ollama endpoint, then retry this command.`, - ); } // Stop host-side services only when their state directory is explicit or diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 8d6e37a43d5..dc06b74578d 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -2,10 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // // Regression guard for #2717: cleanupSandboxServices must invoke -// `unloadOllamaModels()` exactly once across both branches of the destroy -// flow — never zero (orphans GPU memory) and never twice (the original -// duplicate-call bug). Mirrors the structural argument captured in the -// inline comments in `src/lib/actions/sandbox/destroy.ts`. +// `unloadOllamaModels()` exactly once across both branches when the sandbox +// owns Ollama cleanup work, and never for an unrelated provider. This avoids +// both orphaned GPU memory and the original duplicate-call bug. import path from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -38,11 +37,11 @@ function buildDeps( | "googlechatWebhookTunnelPidDir" > >; - stopAllCalls: Array<{ sandboxName: string }>; + stopAllCalls: Array<{ sandboxName: string; cleanupOllamaModels?: boolean }>; unloadCalls: number; unloadArgs: Array; } { - const stopAllCalls: Array<{ sandboxName: string }> = []; + const stopAllCalls: Array<{ sandboxName: string; cleanupOllamaModels?: boolean }> = []; const target = sandbox ? { name: "regression-2717", model: "target-model:latest", ...sandbox } : null; @@ -60,7 +59,7 @@ function buildDeps( sandboxes: [...(target ? [target] : []), ...peers] as never, defaultSandbox: null, })), - stopAll: vi.fn((opts: { sandboxName: string }) => { + stopAll: vi.fn((opts: { sandboxName: string; cleanupOllamaModels?: boolean }) => { stopAllCalls.push(opts); }), unloadOllamaModels: vi.fn((onlyModels?: readonly string[]) => { @@ -96,13 +95,38 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); expect(harness.deps.stopAll).toHaveBeenCalledTimes(1); - expect(harness.stopAllCalls[0]).toEqual({ sandboxName: "regression-2717" }); + expect(harness.stopAllCalls[0]).toEqual({ + sandboxName: "regression-2717", + cleanupOllamaModels: true, + }); // stopAll() invokes unloadOllamaModels() internally — see services.ts. // cleanupSandboxServices itself must not call it again. expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); expect(harness.unloadCalls).toBe(0); }); + it("skips host-wide Ollama discovery for a final sandbox with no Ollama ownership", () => { + const harness = buildDeps({ provider: "nvidia-prod" }); + + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + + expect(harness.stopAllCalls).toEqual([ + { sandboxName: "regression-2717", cleanupOllamaModels: false }, + ]); + expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); + }); + + it("keeps host-wide Ollama cleanup enabled for retained model recovery", () => { + const harness = buildDeps({ provider: "nvidia-prod" }); + vi.mocked(harness.deps.loadPendingOllamaModelCleanup).mockReturnValue(["old-model"]); + + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + + expect(harness.stopAllCalls).toEqual([ + { sandboxName: "regression-2717", cleanupOllamaModels: true }, + ]); + }); + it("calls unloadOllamaModels() exactly once for an Ollama sandbox when stopHostServices=false", () => { const harness = buildDeps({ provider: "ollama-local" }); From 7df5c027c31276beacff021dc86d9abdf2ee2051 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 14:38:19 -0700 Subject: [PATCH 28/47] fix: isolate Windows Ollama readiness transport --- .../local-windows-ollama-transport.test.ts | 13 +++- src/lib/inference/ollama/windows.test.ts | 60 +++++++++++++++++-- src/lib/inference/ollama/windows.ts | 24 ++++++-- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 5a43d4e8fdc..ab7ca4d242b 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -391,7 +391,8 @@ describe("Windows-host Ollama transport", () => { it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = vi.fn((_command: readonly string[]) => + const capture = respondsOnlyThroughDockerDesktop( + "/api/ps", JSON.stringify({ models: [{ name: "qwen3.5:9b", context_length: "invalid" }] }), ); @@ -406,5 +407,15 @@ describe("Windows-host Ollama transport", () => { ok: false, message: expect.stringContaining("cannot verify the required 64000-token window"), }); + expect(capture).toHaveBeenCalledWith( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/ps", + ]), + expect.any(Object), + ); }); }); diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 5b02de34c20..187dc8f8c03 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -106,7 +106,10 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks( + run, + runCapture, + ); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); @@ -122,8 +125,10 @@ describe("Windows Ollama helper", () => { expect(atomicsWaitSpy).toHaveBeenNthCalledWith(1, expect.any(Int32Array), 0, 0, 1000); expect(atomicsWaitSpy).toHaveBeenNthCalledWith(2, expect.any(Int32Array), 0, 0, 1000); expect(atomicsWaitSpy).toHaveBeenNthCalledWith(3, expect.any(Int32Array), 0, 0, 2000); - // The retired subprocess-sleep path must not be exercised. - expect(spawnSyncSpy).not.toHaveBeenCalled(); + // Credential isolation may inspect the Docker context, but the retired + // subprocess-sleep path must remain unused. + expect(spawnSyncSpy).toHaveBeenCalledTimes(1); + expect(spawnSyncSpy.mock.calls[0]?.slice(0, 2)).toEqual(["docker", ["context", "show"]]); } finally { restore(); logSpy.mockRestore(); @@ -152,14 +157,59 @@ describe("Windows Ollama helper", () => { "5", "http://host.docker.internal:11434/api/tags", ], - { ignoreError: true }, + expect.objectContaining({ ignoreError: true }), ); }); + it("isolates Docker credentials while waiting for the Windows-host daemon", () => { + const run = vi.fn(); + const cleanup = vi.fn(() => ({ ok: true as const })); + const runCapture = vi.fn((command: string | string[], options?: { env?: NodeJS.ProcessEnv }) => + Array.isArray(command) && + command[0] === "docker" && + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL && + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const localInference = require(LOCAL_INFERENCE_PATH); + localInference.resetOllamaHostCache(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + + try { + expect( + windows.awaitWindowsOllamaReady({ + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + }), + ).toBe(true); + expect(localInference.getResolvedOllamaHost()).toBe("host.docker.internal"); + expect(runCapture).toHaveBeenCalledWith( + expect.arrayContaining(["docker", "run", "--rm", WINDOWS_OLLAMA_TAGS_URL]), + expect.objectContaining({ + ignoreError: true, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }), + ); + expect(cleanup).toHaveBeenCalledOnce(); + } finally { + localInference.resetOllamaHostCache(); + restore(); + logSpy.mockRestore(); + } + }); + it("skips the blocking wait for non-positive delays", () => { const run = vi.fn(); const runCapture = vi.fn(); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks( + run, + runCapture, + ); try { windows.sleep(0); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 58dcd9a61fa..dda0d5d587a 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -6,9 +6,9 @@ // Detection lives in onboard.ts; this module owns the action side. const { spawn } = require("child_process"); -const { dockerCapture } = require("../../adapters/docker/command"); const { run, runCapture } = require("../../runner"); const { + createOllamaApiCapture, getWindowsHostOllamaDockerReachabilityArgs, isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, @@ -120,13 +120,27 @@ function killWindowsOllamaProcesses(): void { ); } -function awaitWindowsOllamaReady(): boolean { +function awaitWindowsOllamaReady(opts: { prepareDockerEnvironment?: () => unknown } = {}): boolean { console.log(" Waiting for Ollama to respond on host.docker.internal..."); + const capture = createOllamaApiCapture( + runCapture, + OLLAMA_HOST_DOCKER_INTERNAL, + opts.prepareDockerEnvironment, + ); for (let attempt = 0; attempt < 15; attempt++) { sleep(2); - const probe = dockerCapture(getWindowsHostOllamaDockerReachabilityArgs(), { - ignoreError: true, - }); + const probe = capture( + [ + "curl", + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + `http://${OLLAMA_HOST_DOCKER_INTERNAL}:${OLLAMA_PORT}/api/tags`, + ], + { ignoreError: true }, + ); if (isValidOllamaTagsResponseBody(probe)) { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); return true; From b1c6ac33361da1800eecf67dcaf7a0278bdf9fed Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 14:47:04 -0700 Subject: [PATCH 29/47] refactor: centralize Windows Ollama model discovery --- src/lib/inference/local.ts | 8 ++++---- src/lib/inference/ollama/windows.test.ts | 21 +-------------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index e2fca1cbb91..810b112034c 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -1806,8 +1806,9 @@ export function getOllamaModelOptions( const modelDiscoveryRetryDelaysMs = [500, 1_000] as const; // Docker Desktop owns Windows-host reachability because host.docker.internal // may not resolve from WSL. Keep model discovery on the verified transport. - const tagsCommand = getOllamaApiCommand( - buildValidatedCurlCommandArgs([ + const tagsCommand = [ + "curl", + ...buildValidatedCurlCommandArgs([ "-sf", "--connect-timeout", "3", @@ -1815,8 +1816,7 @@ export function getOllamaModelOptions( "5", `http://${host}:${OLLAMA_PORT}/api/tags`, ]), - host, - ); + ]; const readTags = () => { const tagsOutput = capture(tagsCommand, { ignoreError: true }); return parseOllamaModelInventory(String(tagsOutput || "")); diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 187dc8f8c03..d9c284e5c9c 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -106,29 +106,10 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks( - run, - runCapture, - ); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); - // The blocking wait settles for 1s after the kill, pauses 1s between - // launch attempts, then polls readiness with a 2s delay. - expect(atomicsWaitSpy).toHaveBeenCalledTimes(3); - // The wait must target the module's shared backing store (a plain - // ArrayBuffer would be rejected by Atomics.wait). - atomicsWaitSpy.mock.calls.forEach(([array]) => { - expect(array).toBeInstanceOf(Int32Array); - expect(array.buffer).toBeInstanceOf(SharedArrayBuffer); - }); - expect(atomicsWaitSpy).toHaveBeenNthCalledWith(1, expect.any(Int32Array), 0, 0, 1000); - expect(atomicsWaitSpy).toHaveBeenNthCalledWith(2, expect.any(Int32Array), 0, 0, 1000); - expect(atomicsWaitSpy).toHaveBeenNthCalledWith(3, expect.any(Int32Array), 0, 0, 2000); - // Credential isolation may inspect the Docker context, but the retired - // subprocess-sleep path must remain unused. - expect(spawnSyncSpy).toHaveBeenCalledTimes(1); - expect(spawnSyncSpy.mock.calls[0]?.slice(0, 2)).toEqual(["docker", ["context", "show"]]); } finally { restore(); logSpy.mockRestore(); From 9a48167a28259781a135f9d68aeaeb38a83b90d2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 15:02:13 -0700 Subject: [PATCH 30/47] fix: keep Ollama lifecycle transitions atomic --- src/lib/actions/sandbox/destroy.ts | 8 ++++--- .../inference-providers/ollama-local.test.ts | 22 +++++++++++++++++++ .../inference-providers/ollama-local.ts | 21 ++++++++++-------- src/lib/tunnel/services.test.ts | 2 +- .../destroy-cleanup-sandbox-services.test.ts | 21 ++++++++++++++++++ 5 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 4e692a11496..3ea41ebb5b9 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -340,12 +340,14 @@ export function cleanupSandboxServices( // `stopAll()` owns the host-wide unload when this sandbox has an Ollama // route or retained cleanup work. Don't probe an unrelated daemon for a // sandbox with no Ollama ownership, and don't double-call cleanup here. - const cleanupOllamaModels = withOllamaModelOwnershipLock(() => { + ollamaCleanup = withOllamaModelOwnershipLock(() => { const sandbox = getSandbox(validatedSandboxName); const pending = loadPendingOllamaModelCleanup(validatedSandboxName); - return Boolean(sandbox?.provider?.includes("ollama") || pending.length > 0); + const cleanupOllamaModels = Boolean( + sandbox?.provider?.includes("ollama") || pending.length > 0, + ); + return stopAll({ sandboxName: validatedSandboxName, cleanupOllamaModels }); }); - ollamaCleanup = stopAll({ sandboxName: validatedSandboxName, cleanupOllamaModels }); } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index d32dc177971..81a6405bd05 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -331,4 +331,26 @@ describe("Ollama local provider sandbox-facing model gate", () => { expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); }); + + it("restores the prior cleanup route when model warm-up throws", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + localInference: { + runOllamaWarmup: () => { + throw new Error("warm-up transport failed"); + }, + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => rollbackPersistedOllamaHost, + }, + }), + ), + ).rejects.toThrow("warm-up transport failed"); + + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index 4a1a539aac1..d1e5836d5e8 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -163,16 +163,19 @@ export async function setupOllamaLocalInference( return exitProcess(1); } } else { - log(` Priming Ollama model: ${model}`); - if (localInference.runOllamaWarmup) { - localInference.runOllamaWarmup(model, run); - } else { - run(getOllamaWarmupCommand(model), { ignoreError: true }); + let probe: ReturnType; + try { + log(` Priming Ollama model: ${model}`); + if (localInference.runOllamaWarmup) { + localInference.runOllamaWarmup(model, run); + } else { + run(getOllamaWarmupCommand(model), { ignoreError: true }); + } + probe = localInference.validateOllamaModelWithToolsOverride(model, allowToolsIncompatible); + } catch (probeError) { + rollbackCleanupRoute(); + throw probeError; } - const probe = localInference.validateOllamaModelWithToolsOverride( - model, - allowToolsIncompatible, - ); if (!probe.ok) { rollbackCleanupRoute(); error(` ${probe.message}`); diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index 871a67e9eb4..913e04f68e8 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -596,7 +596,7 @@ describe("stopAll", () => { it("logs stop messages", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir }); + stopAll({ pidDir, unloadOllamaModels: () => undefined }); const output = logSpy.mock.calls.map((c) => c[0]).join("\n"); expect(output).toContain("All services stopped"); logSpy.mockRestore(); diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index cedd72855a3..a50ea79ffc7 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -127,6 +127,27 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { ]); }); + it("holds model ownership while final host-wide cleanup runs", () => { + const harness = buildDeps({ provider: "ollama-local" }); + let ownershipHeld = false; + harness.deps.withOllamaModelOwnershipLock = vi.fn((operation) => { + ownershipHeld = true; + try { + return operation(); + } finally { + ownershipHeld = false; + } + }); + vi.mocked(harness.deps.stopAll).mockImplementation(() => { + expect(ownershipHeld).toBe(true); + }); + + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + + expect(harness.deps.stopAll).toHaveBeenCalledOnce(); + expect(ownershipHeld).toBe(false); + }); + it("calls unloadOllamaModels() exactly once for an Ollama sandbox when stopHostServices=false", () => { const harness = buildDeps({ provider: "ollama-local" }); From 7051d83650ed11d767c217f5819190761992b254 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 15:16:44 -0700 Subject: [PATCH 31/47] test: remove duplicate readiness clock --- src/lib/onboard/sandbox-readiness-tracing.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 0701b040248..aa49d4ae27f 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -81,7 +81,6 @@ describe("createSandboxReadyWaiter", () => { isLinuxDockerDriverGatewayEnabled: () => true, now: () => 0, sleep, - now: () => 0, }); await expect(waitForSandboxReady(NAME, 2, 3)).resolves.toEqual({ From f059cd073dfb4e51859e23cb1c83854961b552cf Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 15:37:42 -0700 Subject: [PATCH 32/47] test: prove Ollama unload runs under ownership lock --- src/lib/actions/sandbox/destroy.ts | 14 +++++- .../destroy-cleanup-sandbox-services.test.ts | 46 +++++++++++++------ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 3ea41ebb5b9..7d0ecd27c3e 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -171,6 +171,7 @@ export type CleanupSandboxServicesDeps = { stopAll?: (opts: { sandboxName: string; cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; }) => OllamaUnloadResult | void; unloadOllamaModels?: (onlyModels?: readonly string[]) => OllamaUnloadResult | void; loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; @@ -239,11 +240,16 @@ export function cleanupSandboxServices( const listSandboxes = deps.listSandboxes ?? registry.listSandboxes; const stopAll = deps.stopAll ?? - ((opts: { sandboxName: string; cleanupOllamaModels?: boolean }) => { + ((opts: { + sandboxName: string; + cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; + }) => { const services = require("../../tunnel/services") as { stopAll: (opts: { sandboxName: string; cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; }) => OllamaUnloadResult | void; }; return services.stopAll(opts); @@ -346,7 +352,11 @@ export function cleanupSandboxServices( const cleanupOllamaModels = Boolean( sandbox?.provider?.includes("ollama") || pending.length > 0, ); - return stopAll({ sandboxName: validatedSandboxName, cleanupOllamaModels }); + return stopAll({ + sandboxName: validatedSandboxName, + cleanupOllamaModels, + unloadOllamaModels: () => unloadOllamaModels(), + }); }); } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index a50ea79ffc7..78bfa327f3a 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -12,9 +12,15 @@ import { describe, expect, it, vi } from "vitest"; import type { CleanupSandboxServicesDeps } from "../../../src/lib/actions/sandbox/destroy.js"; import { cleanupSandboxServices } from "../../../src/lib/actions/sandbox/destroy.js"; import { ollamaModelRefsMatch } from "../../../src/lib/inference/ollama/model-discovery.js"; +import type { OllamaUnloadResult } from "../../../src/lib/inference/ollama/proxy.js"; import { SANDBOX_PROVIDER_SUFFIXES } from "../../../src/lib/onboard/sandbox-provider-cleanup.js"; type SandboxLike = { name?: string; model?: string | null; provider?: string | null } | null; +type StopAllOptions = { + sandboxName: string; + cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; +}; function buildDeps( sandbox: SandboxLike, @@ -37,11 +43,11 @@ function buildDeps( | "googlechatWebhookTunnelPidDir" > >; - stopAllCalls: Array<{ sandboxName: string; cleanupOllamaModels?: boolean }>; + stopAllCalls: StopAllOptions[]; unloadCalls: number; unloadArgs: Array; } { - const stopAllCalls: Array<{ sandboxName: string; cleanupOllamaModels?: boolean }> = []; + const stopAllCalls: StopAllOptions[] = []; const target = sandbox ? { name: "regression-2717", model: "target-model:latest", ...sandbox } : null; @@ -59,8 +65,9 @@ function buildDeps( sandboxes: [...(target ? [target] : []), ...peers] as never, defaultSandbox: null, })), - stopAll: vi.fn((opts: { sandboxName: string; cleanupOllamaModels?: boolean }) => { + stopAll: vi.fn((opts: StopAllOptions) => { stopAllCalls.push(opts); + return opts.cleanupOllamaModels === false ? undefined : opts.unloadOllamaModels?.(); }), unloadOllamaModels: vi.fn((onlyModels?: readonly string[]) => { unloadCalls += 1; @@ -95,14 +102,15 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); expect(harness.deps.stopAll).toHaveBeenCalledTimes(1); - expect(harness.stopAllCalls[0]).toEqual({ - sandboxName: "regression-2717", - cleanupOllamaModels: true, - }); - // stopAll() invokes unloadOllamaModels() internally — see services.ts. - // cleanupSandboxServices itself must not call it again. - expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); - expect(harness.unloadCalls).toBe(0); + expect(harness.stopAllCalls[0]).toEqual( + expect.objectContaining({ + sandboxName: "regression-2717", + cleanupOllamaModels: true, + unloadOllamaModels: expect.any(Function), + }), + ); + expect(harness.deps.unloadOllamaModels).toHaveBeenCalledOnce(); + expect(harness.unloadCalls).toBe(1); }); it("skips host-wide Ollama discovery for a final sandbox with no Ollama ownership", () => { @@ -111,7 +119,11 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); expect(harness.stopAllCalls).toEqual([ - { sandboxName: "regression-2717", cleanupOllamaModels: false }, + expect.objectContaining({ + sandboxName: "regression-2717", + cleanupOllamaModels: false, + unloadOllamaModels: expect.any(Function), + }), ]); expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); }); @@ -123,8 +135,13 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); expect(harness.stopAllCalls).toEqual([ - { sandboxName: "regression-2717", cleanupOllamaModels: true }, + expect.objectContaining({ + sandboxName: "regression-2717", + cleanupOllamaModels: true, + unloadOllamaModels: expect.any(Function), + }), ]); + expect(harness.deps.unloadOllamaModels).toHaveBeenCalledOnce(); }); it("holds model ownership while final host-wide cleanup runs", () => { @@ -138,13 +155,14 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { ownershipHeld = false; } }); - vi.mocked(harness.deps.stopAll).mockImplementation(() => { + vi.mocked(harness.deps.unloadOllamaModels).mockImplementation(() => { expect(ownershipHeld).toBe(true); }); cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); expect(harness.deps.stopAll).toHaveBeenCalledOnce(); + expect(harness.deps.unloadOllamaModels).toHaveBeenCalledOnce(); expect(ownershipHeld).toBe(false); }); From fac042be6ac6d4e46edfd9182c24d56ba498a6aa Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 15:43:06 -0700 Subject: [PATCH 33/47] fix: preserve destroy recovery guidance Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/destroy.ts | 35 +++++++++++++------ .../destroy-cleanup-sandbox-services.test.ts | 19 +++++++--- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 7d0ecd27c3e..d4f1258e4ea 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -346,18 +346,31 @@ export function cleanupSandboxServices( // `stopAll()` owns the host-wide unload when this sandbox has an Ollama // route or retained cleanup work. Don't probe an unrelated daemon for a // sandbox with no Ollama ownership, and don't double-call cleanup here. - ollamaCleanup = withOllamaModelOwnershipLock(() => { - const sandbox = getSandbox(validatedSandboxName); - const pending = loadPendingOllamaModelCleanup(validatedSandboxName); - const cleanupOllamaModels = Boolean( - sandbox?.provider?.includes("ollama") || pending.length > 0, - ); - return stopAll({ - sandboxName: validatedSandboxName, - cleanupOllamaModels, - unloadOllamaModels: () => unloadOllamaModels(), + try { + ollamaCleanup = withOllamaModelOwnershipLock(() => { + const sandbox = getSandbox(validatedSandboxName); + const pending = loadPendingOllamaModelCleanup(validatedSandboxName); + const cleanupOllamaModels = Boolean( + sandbox?.provider?.includes("ollama") || pending.length > 0, + ); + return stopAll({ + sandboxName: validatedSandboxName, + cleanupOllamaModels, + unloadOllamaModels: () => unloadOllamaModels(), + }); }); - }); + } catch (error) { + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .trim() + .slice(0, 300); + throw new Error( + `Host-service cleanup failed after sandbox '${validatedSandboxName}' was deleted: ${detail || "unknown error"}. ` + + `The local registry and cleanup state for '${validatedSandboxName}' were retained for recovery; ` + + `restore the reported dependency, then retry \`nemoclaw ${validatedSandboxName} destroy\`.`, + { cause: error }, + ); + } } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 78bfa327f3a..9eda677a291 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -233,13 +233,24 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { it("preserves destroy recovery state when stopAll throws unexpectedly (#10553)", () => { const harness = buildDeps({ provider: "ollama-local" }); + const stopError = new Error(`unexpected cleanup failure ${"detail ".repeat(100)}`); vi.mocked(harness.deps.stopAll).mockImplementation(() => { - throw new Error("unexpected cleanup failure"); + throw stopError; }); - expect(() => - cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps), - ).toThrow("unexpected cleanup failure"); + let thrown: unknown; + try { + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(Error); + expect(thrown).toMatchObject({ cause: stopError }); + expect((thrown as Error).message).toMatch( + /sandbox 'regression-2717' was deleted.*local registry and cleanup state.*nemoclaw regression-2717 destroy/, + ); + expect((thrown as Error).message.length).toBeLessThan(700); expect(harness.deps.rmSync).not.toHaveBeenCalled(); expect(harness.deps.runOpenshell).not.toHaveBeenCalled(); }); From a101ffb1528556e1674da4d911bd4458cd8c44c1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 16:01:29 -0700 Subject: [PATCH 34/47] fix: complete Windows Ollama transport migration --- .../rebuild-local-provider-recreate.test.ts | 1 + .../http/container-curl-probe.test.ts | 6 ++++++ src/lib/adapters/http/container-curl-probe.ts | 3 ++- src/lib/inference/local.ts | 7 ++++++- src/lib/inference/ollama/proxy.test.ts | 2 +- src/lib/inference/ollama/windows.test.ts | 20 +------------------ .../onboard-host-docker-internal.test.ts | 4 +++- .../inference-providers/ollama-local.test.ts | 1 + .../inference-providers/ollama-local.ts | 7 +------ src/lib/onboard/inference-providers/types.ts | 2 +- src/lib/onboard/provider-host-state.test.ts | 2 +- src/lib/onboard/setup-inference.test.ts | 1 + src/lib/tunnel/services.ts | 2 +- test/e2e/live/ollama-auth-proxy.test.ts | 4 ++-- .../ollama/ollama-gpu-cleanup.test.ts | 16 +++++++++++++-- .../ollama/ollama-pull-timeout.test.ts | 6 +++++- ...board-host-local-inference-routing.test.ts | 1 + .../onboard-inference-reconciliation.test.ts | 1 + 18 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index b0840a1a118..d8cd4675cd2 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -93,6 +93,7 @@ const localProviderScenarios = [ localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, persistResolvedOllamaHost: () => () => {}, }, OLLAMA_PROXY_CREDENTIAL_ENV: "NEMOCLAW_OLLAMA_PROXY_TOKEN", diff --git a/src/lib/adapters/http/container-curl-probe.test.ts b/src/lib/adapters/http/container-curl-probe.test.ts index 11da9590efb..5a28e294e1b 100644 --- a/src/lib/adapters/http/container-curl-probe.test.ts +++ b/src/lib/adapters/http/container-curl-probe.test.ts @@ -23,6 +23,12 @@ function successfulSpawn(stdout = "200"): SpawnSyncReturns { } describe("container curl probe", () => { + it("uses the accepted immutable curl image", () => { + expect(CONTAINER_REACHABILITY_IMAGE).toBe( + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + ); + }); + it("writes the response body and returns the HTTP status without a WSL bind mount (#9116)", () => { const responseBody = '{"choices":[{"message":{"tool_calls":[{}]}}]}'; const spawn = vi.fn( diff --git a/src/lib/adapters/http/container-curl-probe.ts b/src/lib/adapters/http/container-curl-probe.ts index 90955adca3c..483e2154343 100644 --- a/src/lib/adapters/http/container-curl-probe.ts +++ b/src/lib/adapters/http/container-curl-probe.ts @@ -11,7 +11,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -export const CONTAINER_REACHABILITY_IMAGE = "curlimages/curl:8.10.1"; +export const CONTAINER_REACHABILITY_IMAGE = + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661"; const MAX_CONTAINER_CURL_STDOUT_BYTES = 8 * 1024 * 1024 + 256; const HTTP_STATUS_MARKER_PREFIX = "\n__NEMOCLAW_CONTAINER_HTTP_STATUS_"; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 64696ea6bd8..4925fb5c67b 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -224,7 +224,12 @@ export function getResolvedOllamaHost(): string { return _resolvedOllamaHost ?? OLLAMA_LOCALHOST; } -/** Persist the accepted local Ollama route for later CLI processes. */ +/** + * Persist the accepted host-global Ollama route for later CLI processes. + * `ollama-local` is one gateway provider backed by one host auth proxy, so all + * sandboxes using that provider share the same daemon target. Discovery probes + * this receipt first and changes it only after the recorded target is stale. + */ export function persistResolvedOllamaHost( host: string = getResolvedOllamaHost(), stateRoot: string = resolveSharedLocalAdapterStateRoot(), diff --git a/src/lib/inference/ollama/proxy.test.ts b/src/lib/inference/ollama/proxy.test.ts index 7d92a0cf87d..40093bb0f6e 100644 --- a/src/lib/inference/ollama/proxy.test.ts +++ b/src/lib/inference/ollama/proxy.test.ts @@ -487,7 +487,7 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { expect.arrayContaining([ "run", "--rm", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", "-X", "POST", "Content-Type: application/json", diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index d9c284e5c9c..36b61525f1b 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -130,7 +130,7 @@ describe("Windows Ollama helper", () => { "docker", "run", "--rm", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", "-sf", "--connect-timeout", "2", @@ -183,22 +183,4 @@ describe("Windows Ollama helper", () => { logSpy.mockRestore(); } }); - - it("skips the blocking wait for non-positive delays", () => { - const run = vi.fn(); - const runCapture = vi.fn(); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks( - run, - runCapture, - ); - - try { - windows.sleep(0); - windows.sleep(-1); - expect(atomicsWaitSpy).not.toHaveBeenCalled(); - expect(spawnSyncSpy).not.toHaveBeenCalled(); - } finally { - restore(); - } - }); }); diff --git a/src/lib/inference/onboard-host-docker-internal.test.ts b/src/lib/inference/onboard-host-docker-internal.test.ts index 62cd833ee53..2a4872237b3 100644 --- a/src/lib/inference/onboard-host-docker-internal.test.ts +++ b/src/lib/inference/onboard-host-docker-internal.test.ts @@ -106,7 +106,9 @@ describe("host.docker.internal onboarding inference policy", () => { expect(seenCommands).toHaveLength(1); seenCommands.forEach(({ command, args }) => { expect(command).toBe("docker"); - expect(args).toContain("curlimages/curl:8.10.1"); + expect(args).toContain( + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + ); expect(args).toContain("http://host.docker.internal:11434/v1/chat/completions"); expect(args).not.toContain("--volume"); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index 81a6405bd05..2bbd74f5bdc 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -58,6 +58,7 @@ function deps(overrides: OllamaDepsOverrides = {}): OllamaDeps { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: vi.fn(), persistResolvedOllamaHost: () => () => {}, ...localInference, }, diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index d1e5836d5e8..2a476332194 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -22,7 +22,6 @@ export async function setupOllamaLocalInference( validateLocalProvider, getLocalProviderBaseUrl, applyLocalInferenceRoute, - getOllamaWarmupCommand, run, shouldFrontOllamaWithProxy, ensureOllamaAuthProxy, @@ -166,11 +165,7 @@ export async function setupOllamaLocalInference( let probe: ReturnType; try { log(` Priming Ollama model: ${model}`); - if (localInference.runOllamaWarmup) { - localInference.runOllamaWarmup(model, run); - } else { - run(getOllamaWarmupCommand(model), { ignoreError: true }); - } + localInference.runOllamaWarmup(model, run); probe = localInference.validateOllamaModelWithToolsOverride(model, allowToolsIncompatible); } catch (probeError) { rollbackCleanupRoute(); diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 645914845e3..570468d55b8 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -255,7 +255,7 @@ export type OllamaDeps = CommonDeps & { allowToolsIncompatible: boolean, ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; - runOllamaWarmup?(model: string, runImpl: RunFn): void; + runOllamaWarmup(model: string, runImpl: RunFn): void; loadPendingOllamaModelCleanup?(sandboxName: string): readonly string[]; persistPendingOllamaModelCleanup?(sandboxName: string, models: readonly string[]): void; clearPendingOllamaModelCleanup?(sandboxName: string, releasedModels?: readonly string[]): void; diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 4d115791344..d55cdbd3790 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -229,7 +229,7 @@ describe("detectInferenceProviderHostState", () => { [ "run", "--rm", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", "-sf", "--connect-timeout", "2", diff --git a/src/lib/onboard/setup-inference.test.ts b/src/lib/onboard/setup-inference.test.ts index b9234a97e03..fb806cbbd5b 100644 --- a/src/lib/onboard/setup-inference.test.ts +++ b/src/lib/onboard/setup-inference.test.ts @@ -334,6 +334,7 @@ describe("createProviderReviewDeps", () => { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, persistResolvedOllamaHost: () => () => {}, }, OLLAMA_PROXY_CREDENTIAL_ENV: "NEMOCLAW_OLLAMA_PROXY_TOKEN", diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index f0758ae9b02..61db429830c 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -52,7 +52,7 @@ export interface ServiceOptions { unloadOllamaModels?: () => OllamaUnloadResult | void; /** Whether this scoped stop owns Ollama models that require cleanup. Defaults to true. */ cleanupOllamaModels?: boolean; - /** Injectable retirement of sandbox-scoped Ollama cleanup recovery. */ + /** Clears pending Ollama cleanup recovery after this sandbox's models unload. */ clearPendingOllamaModelCleanup?: (sandboxName: string) => void; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ cloudflareTunnelToken?: string; diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index f00767858f1..c2dd20de36d 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -452,7 +452,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and "--rm", "--add-host", "host.openshell.internal:host-gateway", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", "-s", "-o", "/dev/null", @@ -480,7 +480,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and "--rm", "--add-host", "host.openshell.internal:host-gateway", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", "-sf", "--connect-timeout", "3", diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index 58e6d867954..9670c676ac5 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -124,7 +124,13 @@ describe("Ollama GPU cleanup", () => { expect(calls).toHaveLength(3); calls.forEach(({ command, args }) => { expect(command).toBe("docker"); - expect(args).toEqual(expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"])); + expect(args).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + ]), + ); }); } finally { resetOllamaHostCache(); @@ -151,7 +157,13 @@ describe("Ollama GPU cleanup", () => { "http://host.docker.internal:11434/api/ps", ]); dockerCalls.forEach(({ args }) => { - expect(args).toEqual(expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"])); + expect(args).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + ]), + ); }); }, "host.docker.internal", diff --git a/test/inference/ollama/ollama-pull-timeout.test.ts b/test/inference/ollama/ollama-pull-timeout.test.ts index 0296dd05789..897fbb92319 100644 --- a/test/inference/ollama/ollama-pull-timeout.test.ts +++ b/test/inference/ollama/ollama-pull-timeout.test.ts @@ -102,7 +102,11 @@ pullOllamaModel("qwen3.5:9b") expect(payload.ok).toBe(true); expect(payload.captured.cmd).toBe("docker"); expect(payload.captured.args).toEqual( - expect.arrayContaining(["run", "--rm", "curlimages/curl:8.10.1"]), + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + ]), ); const maxTimeIndex = payload.captured.args.indexOf("--max-time"); expect(maxTimeIndex).toBeGreaterThanOrEqual(0); diff --git a/test/onboarding/onboard-host-local-inference-routing.test.ts b/test/onboarding/onboard-host-local-inference-routing.test.ts index 9d989103284..50cfc409332 100644 --- a/test/onboarding/onboard-host-local-inference-routing.test.ts +++ b/test/onboarding/onboard-host-local-inference-routing.test.ts @@ -499,6 +499,7 @@ describe("onboard host-local inference routing", () => { localInference: { validateOllamaModelWithToolsOverride: legacyOllamaProof, validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, persistResolvedOllamaHost: () => () => {}, }, verifyInferenceRoute: verify, diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index bbe3ba27434..a7ffdfe601d 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -1050,6 +1050,7 @@ describe("re-onboard Ollama GPU release (#9110)", () => { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, persistResolvedOllamaHost: () => () => {}, clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, loadPendingOllamaModelCleanup: options.loadPendingOllamaModelCleanup ?? (() => []), From c3ac1ce819122e11c97397bc9c40db00895965ac Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 16:12:27 -0700 Subject: [PATCH 35/47] test: pair immutable probe image with fast E2E --- test/e2e/support/e2e-clients.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 6c553196f23..9115dc182c1 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -6,6 +6,9 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { + CONTAINER_REACHABILITY_IMAGE, +} from "../../../src/lib/adapters/http/container-curl-probe.ts"; import { assertExitZero, type CommandRunner, @@ -77,6 +80,12 @@ class FakeRunner implements CommandRunner { } describe("E2E fixture clients", () => { + it("pins the live container reachability client by digest", () => { + expect(CONTAINER_REACHABILITY_IMAGE).toBe( + "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + ); + }); + it.each([ "a2345678901234567890", "e2e--sandbox", From 6f3827e91722720a22ddf9dfb20d0c1d79c79313 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 16:35:09 -0700 Subject: [PATCH 36/47] test: expect immutable probe image guidance --- src/lib/inference/local.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 9b288698c3a..79f0b392e8c 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -335,7 +335,7 @@ describe("local inference helpers", () => { expect(result.message).toMatch(/not an Ollama networking failure/); expect(result.message).not.toMatch(/Docker container reachability check failed/); expect(result.message).not.toMatch(/sandbox uses a different network path/); - expect(result.diagnostic).toMatch(/DOCKER_CONFIG=\$\(mktemp -d\) docker pull curlimages\/curl/); + expect(result.diagnostic).toContain(CONTAINER_REACHABILITY_IMAGE); expect(result.diagnostic).toMatch(/credential helper/); expect(result.diagnostic).toMatch(/onboard --resume/); }); @@ -354,7 +354,7 @@ describe("local inference helpers", () => { expect(result.ok).toBe(false); expect(result.message).toMatch(/Docker image-pull failure/); expect(result.message).toMatch(/not a vLLM networking failure/); - expect(result.diagnostic).toMatch(/docker pull curlimages\/curl/); + expect(result.diagnostic).toContain(`docker pull ${CONTAINER_REACHABILITY_IMAGE}`); }); it("keeps the runtime-failure report when the probe image is present locally (#9308)", () => { From 301d64b71cd9ac72119cba3487872e91a8e18f41 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 16:50:45 -0700 Subject: [PATCH 37/47] test: isolate sandbox service tests from Ollama --- src/lib/tunnel/services-sandbox.test.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/lib/tunnel/services-sandbox.test.ts b/src/lib/tunnel/services-sandbox.test.ts index 87fe52cf782..4c9c34ef78c 100644 --- a/src/lib/tunnel/services-sandbox.test.ts +++ b/src/lib/tunnel/services-sandbox.test.ts @@ -21,6 +21,10 @@ function restoreSandboxEnv(saved: Record<(typeof SANDBOX_ENV_NAMES)[number], str } } +function stopAllWithoutOllama(opts: Parameters[0] = {}) { + return stopAll({ ...opts, cleanupOllamaModels: false }); +} + describe("stopAll with sandbox channels", () => { let pidDir: string; let stopSandboxChannels: ReturnType; @@ -46,7 +50,7 @@ describe("stopAll with sandbox channels", () => { it("stops in-sandbox channels when sandboxName is provided", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir, sandboxName: "test-sb" }); + stopAllWithoutOllama({ pidDir, sandboxName: "test-sb" }); expect(stopSandboxChannels).toHaveBeenCalledWith("test-sb", { info: expect.any(Function), @@ -58,7 +62,7 @@ describe("stopAll with sandbox channels", () => { it("warns when no sandbox name is available", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).not.toHaveBeenCalled(); const output = logSpy.mock.calls.map((call) => call[0]).join("\n"); @@ -69,7 +73,7 @@ describe("stopAll with sandbox channels", () => { it("still stops cloudflared when in-sandbox shutdown cannot stop a process", () => { writeFileSync(join(pidDir, "cloudflared.pid"), "999999999"); - stopAll({ pidDir, sandboxName: "test-sb" }); + stopAllWithoutOllama({ pidDir, sandboxName: "test-sb" }); expect(stopSandboxChannels).toHaveBeenCalledTimes(1); expect(existsSync(join(pidDir, "cloudflared.pid"))).toBe(false); @@ -78,7 +82,7 @@ describe("stopAll with sandbox channels", () => { it("reads sandbox name from NEMOCLAW_SANDBOX env when not in opts", () => { process.env.NEMOCLAW_SANDBOX = "env-sandbox"; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("env-sandbox", expect.any(Object)); }); @@ -86,7 +90,7 @@ describe("stopAll with sandbox channels", () => { it("reads sandbox name from NEMOCLAW_SANDBOX_NAME when NEMOCLAW_SANDBOX is unset", () => { process.env.NEMOCLAW_SANDBOX_NAME = "named-sandbox"; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("named-sandbox", expect.any(Object)); }); @@ -95,7 +99,7 @@ describe("stopAll with sandbox channels", () => { process.env.NEMOCLAW_SANDBOX_NAME = "name-sandbox"; process.env.NEMOCLAW_SANDBOX = "other-sandbox"; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("name-sandbox", expect.any(Object)); }); @@ -112,7 +116,7 @@ describe("stopAll with sandbox channels", () => { process.env.NEMOCLAW_SANDBOX = "other-sandbox"; try { - stopAll({ pidDir: effectivePidDir }); + stopAllWithoutOllama({ pidDir: effectivePidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("name-sandbox", expect.any(Object)); expect(existsSync(join(effectivePidDir, "cloudflared.pid"))).toBe(false); @@ -129,7 +133,7 @@ describe("stopAll with sandbox channels", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); process.env.NEMOCLAW_SANDBOX_NAME = invalidName; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).not.toHaveBeenCalled(); expect(logSpy.mock.calls.map((call) => call[0]).join("\n")).toContain("Invalid sandbox name"); @@ -146,7 +150,7 @@ describe("stopAll with sandbox channels", () => { writeFileSync(join(pidDir, "cloudflared.pid"), "999999999"); expect(() => - stopAll({ pidDir, sandboxName: "bad name", releaseGatewayPort: true }), + stopAllWithoutOllama({ pidDir, sandboxName: "bad name", releaseGatewayPort: true }), ).not.toThrow(); expect(stopSandboxChannels).not.toHaveBeenCalled(); @@ -161,7 +165,7 @@ describe("stopAll with sandbox channels", () => { it("does not stop default cloudflared for a malformed sandbox name without an explicit pidDir", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - expect(() => stopAll({ sandboxName: "bad name" })).not.toThrow(); + expect(() => stopAllWithoutOllama({ sandboxName: "bad name" })).not.toThrow(); expect(stopSandboxChannels).not.toHaveBeenCalled(); const output = logSpy.mock.calls.map((call) => call[0]).join("\n"); From 09b199f778c4666b94ee45a70e02eedf035f996b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 17:18:31 -0700 Subject: [PATCH 38/47] fix: use multi-platform Ollama probe image --- .../sandbox/rebuild-local-provider-recreate.test.ts | 1 - src/lib/adapters/http/container-curl-probe.test.ts | 2 +- src/lib/adapters/http/container-curl-probe.ts | 2 +- src/lib/inference/ollama/proxy.test.ts | 2 +- src/lib/inference/ollama/windows.test.ts | 2 +- src/lib/inference/onboard-host-docker-internal.test.ts | 9 ++++++--- src/lib/onboard.ts | 2 -- src/lib/onboard/inference-providers/ollama-local.test.ts | 3 --- src/lib/onboard/inference-providers/types.ts | 1 - src/lib/onboard/provider-host-state.test.ts | 2 +- src/lib/onboard/setup-inference.test.ts | 1 - src/lib/onboard/setup-inference.ts | 2 -- test/e2e/live/ollama-auth-proxy.test.ts | 4 ++-- test/e2e/support/e2e-clients.test.ts | 2 +- test/inference/ollama/ollama-gpu-cleanup.test.ts | 4 ++-- test/inference/ollama/ollama-pull-timeout.test.ts | 2 +- .../onboard-host-local-inference-routing.test.ts | 3 --- 17 files changed, 17 insertions(+), 27 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index d8cd4675cd2..b52ac71629a 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -83,7 +83,6 @@ const localProviderScenarios = [ validateLocalProvider: () => ({ ok: true }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:11435/v1", applyLocalInferenceRoute, - getOllamaWarmupCommand: () => ["true"], run: () => ({ status: 0 }), shouldFrontOllamaWithProxy: () => false, ensureOllamaAuthProxy: vi.fn(), diff --git a/src/lib/adapters/http/container-curl-probe.test.ts b/src/lib/adapters/http/container-curl-probe.test.ts index 5a28e294e1b..baba04a1dd3 100644 --- a/src/lib/adapters/http/container-curl-probe.test.ts +++ b/src/lib/adapters/http/container-curl-probe.test.ts @@ -25,7 +25,7 @@ function successfulSpawn(stdout = "200"): SpawnSyncReturns { describe("container curl probe", () => { it("uses the accepted immutable curl image", () => { expect(CONTAINER_REACHABILITY_IMAGE).toBe( - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", ); }); diff --git a/src/lib/adapters/http/container-curl-probe.ts b/src/lib/adapters/http/container-curl-probe.ts index 483e2154343..54fb020fbee 100644 --- a/src/lib/adapters/http/container-curl-probe.ts +++ b/src/lib/adapters/http/container-curl-probe.ts @@ -12,7 +12,7 @@ import os from "node:os"; import path from "node:path"; export const CONTAINER_REACHABILITY_IMAGE = - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661"; + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b"; const MAX_CONTAINER_CURL_STDOUT_BYTES = 8 * 1024 * 1024 + 256; const HTTP_STATUS_MARKER_PREFIX = "\n__NEMOCLAW_CONTAINER_HTTP_STATUS_"; diff --git a/src/lib/inference/ollama/proxy.test.ts b/src/lib/inference/ollama/proxy.test.ts index 40093bb0f6e..f6bd9d4daeb 100644 --- a/src/lib/inference/ollama/proxy.test.ts +++ b/src/lib/inference/ollama/proxy.test.ts @@ -487,7 +487,7 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { expect.arrayContaining([ "run", "--rm", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-X", "POST", "Content-Type: application/json", diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 36b61525f1b..928500239fb 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -130,7 +130,7 @@ describe("Windows Ollama helper", () => { "docker", "run", "--rm", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", "2", diff --git a/src/lib/inference/onboard-host-docker-internal.test.ts b/src/lib/inference/onboard-host-docker-internal.test.ts index 2a4872237b3..61ce1e96e87 100644 --- a/src/lib/inference/onboard-host-docker-internal.test.ts +++ b/src/lib/inference/onboard-host-docker-internal.test.ts @@ -106,9 +106,12 @@ describe("host.docker.internal onboarding inference policy", () => { expect(seenCommands).toHaveLength(1); seenCommands.forEach(({ command, args }) => { expect(command).toBe("docker"); - expect(args).toContain( - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", - ); + expect(args.slice(0, 3)).toEqual([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + ]); + expect(args).not.toContain("curlimages/curl:8.10.1"); expect(args).toContain("http://host.docker.internal:11434/v1/chat/completions"); expect(args).not.toContain("--volume"); }); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b1eca0570db..dad6121b2ae 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -207,7 +207,6 @@ const { getLocalProviderBaseUrl, getLocalProviderHealthCheck, getLocalProviderValidationBaseUrl, - getOllamaWarmupCommand, validateLocalProvider, } = localInference; const resolveNonInteractiveModel = localInference.resolveNonInteractiveOllamaModel; @@ -2453,7 +2452,6 @@ function getSetupInferenceDeps(): SetupInferenceDeps { getLocalProviderBaseUrl, run, vllmLocalCredentialEnv: VLLM_LOCAL_CREDENTIAL_ENV, - getOllamaWarmupCommand, shouldFrontOllamaWithProxy, ensureOllamaAuthProxy, isProxyHealthy, diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index 2bbd74f5bdc..96b4c161377 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -7,7 +7,6 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { CONTAINER_REACHABILITY_IMAGE, - getOllamaWarmupCommand, loadPersistedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, persistResolvedOllamaHost, @@ -48,7 +47,6 @@ function deps(overrides: OllamaDepsOverrides = {}): OllamaDeps { validateLocalProvider: () => ({ ok: true }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:11434/v1", applyLocalInferenceRoute: async () => false, - getOllamaWarmupCommand: () => ["ollama", "run", "llama3.2:1b"], run: vi.fn(() => ({ status: 0 })), shouldFrontOllamaWithProxy: () => false, ensureOllamaAuthProxy: vi.fn(), @@ -159,7 +157,6 @@ describe("Ollama local provider sandbox-facing model gate", () => { setupOllamaLocalInference( { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, deps({ - getOllamaWarmupCommand, run, localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 570468d55b8..45ca911cef6 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -242,7 +242,6 @@ export type OllamaDeps = CommonDeps & { }; getLocalProviderBaseUrl: (provider: string) => any; applyLocalInferenceRoute: (provider: string, model: string) => Promise; - getOllamaWarmupCommand: (model: string) => any; run: RunFn; shouldFrontOllamaWithProxy: () => boolean; ensureOllamaAuthProxy: () => void; diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index d55cdbd3790..911cdcf39b8 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -229,7 +229,7 @@ describe("detectInferenceProviderHostState", () => { [ "run", "--rm", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", "2", diff --git a/src/lib/onboard/setup-inference.test.ts b/src/lib/onboard/setup-inference.test.ts index fb806cbbd5b..ecffb3ad17e 100644 --- a/src/lib/onboard/setup-inference.test.ts +++ b/src/lib/onboard/setup-inference.test.ts @@ -324,7 +324,6 @@ describe("createProviderReviewDeps", () => { validateLocalProvider: () => ({ ok: true }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:11435/v1", applyLocalInferenceRoute: async () => false, - getOllamaWarmupCommand: () => ["ollama", "run", "qwen3.5:9b"], run: vi.fn() as never, shouldFrontOllamaWithProxy: () => true, ensureOllamaAuthProxy, diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index ff726d078ac..ea429f832c3 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -185,7 +185,6 @@ type ProviderBranchDeps = Pick< > & Pick< OllamaDeps, - | "getOllamaWarmupCommand" | "shouldFrontOllamaWithProxy" | "ensureOllamaAuthProxy" | "isProxyHealthy" @@ -1045,7 +1044,6 @@ export function createSetupInference( runGatewayOpenshell, revalidateSandboxIdentity, ), - getOllamaWarmupCommand: deps.getOllamaWarmupCommand, run: deps.run, shouldFrontOllamaWithProxy: hostLocalRoute ? () => false diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index c2dd20de36d..9bd89e2a6dc 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -452,7 +452,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and "--rm", "--add-host", "host.openshell.internal:host-gateway", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-s", "-o", "/dev/null", @@ -480,7 +480,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and "--rm", "--add-host", "host.openshell.internal:host-gateway", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", "3", diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 9115dc182c1..25f801faf45 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -82,7 +82,7 @@ class FakeRunner implements CommandRunner { describe("E2E fixture clients", () => { it("pins the live container reachability client by digest", () => { expect(CONTAINER_REACHABILITY_IMAGE).toBe( - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", ); }); diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index 9670c676ac5..e684bb4ec97 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -128,7 +128,7 @@ describe("Ollama GPU cleanup", () => { expect.arrayContaining([ "run", "--rm", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", ]), ); }); @@ -161,7 +161,7 @@ describe("Ollama GPU cleanup", () => { expect.arrayContaining([ "run", "--rm", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", ]), ); }); diff --git a/test/inference/ollama/ollama-pull-timeout.test.ts b/test/inference/ollama/ollama-pull-timeout.test.ts index 897fbb92319..389753077af 100644 --- a/test/inference/ollama/ollama-pull-timeout.test.ts +++ b/test/inference/ollama/ollama-pull-timeout.test.ts @@ -105,7 +105,7 @@ pullOllamaModel("qwen3.5:9b") expect.arrayContaining([ "run", "--rm", - "docker.io/curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", ]), ); const maxTimeIndex = payload.captured.args.indexOf("--max-time"); diff --git a/test/onboarding/onboard-host-local-inference-routing.test.ts b/test/onboarding/onboard-host-local-inference-routing.test.ts index 50cfc409332..5e8f1a7ea0b 100644 --- a/test/onboarding/onboard-host-local-inference-routing.test.ts +++ b/test/onboarding/onboard-host-local-inference-routing.test.ts @@ -471,7 +471,6 @@ describe("onboard host-local inference routing", () => { const route = fixture(application, "ollama"); const legacyRun = vi.fn(); const legacyValidate = vi.fn(); - const legacyWarmup = vi.fn(); const legacyOllamaProof = vi.fn(); const verify = vi.fn(() => { route.events.push("gateway-route-verify"); @@ -495,7 +494,6 @@ describe("onboard host-local inference routing", () => { applyLocalInferenceRoute: undefined, run: legacyRun, validateLocalProvider: legacyValidate, - getOllamaWarmupCommand: legacyWarmup, localInference: { validateOllamaModelWithToolsOverride: legacyOllamaProof, validateSandboxFacingOllamaModel: () => ({ ok: true }), @@ -581,7 +579,6 @@ describe("onboard host-local inference routing", () => { ); expect(legacyRun).not.toHaveBeenCalled(); expect(legacyValidate).not.toHaveBeenCalled(); - expect(legacyWarmup).not.toHaveBeenCalled(); expect(legacyOllamaProof).not.toHaveBeenCalled(); expect(route.gatewayRollback).not.toHaveBeenCalled(); }, From afde82aa9215b657825bc5989ca53c9431889858 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 17:23:24 -0700 Subject: [PATCH 39/47] test: verify probe pin through transport --- .../http/container-curl-probe.test.ts | 6 ------ test/e2e/support/e2e-clients.test.ts | 20 ++++++++++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/lib/adapters/http/container-curl-probe.test.ts b/src/lib/adapters/http/container-curl-probe.test.ts index baba04a1dd3..11da9590efb 100644 --- a/src/lib/adapters/http/container-curl-probe.test.ts +++ b/src/lib/adapters/http/container-curl-probe.test.ts @@ -23,12 +23,6 @@ function successfulSpawn(stdout = "200"): SpawnSyncReturns { } describe("container curl probe", () => { - it("uses the accepted immutable curl image", () => { - expect(CONTAINER_REACHABILITY_IMAGE).toBe( - "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", - ); - }); - it("writes the response body and returns the HTTP status without a WSL bind mount (#9116)", () => { const responseBody = '{"choices":[{"message":{"tool_calls":[{}]}}]}'; const spawn = vi.fn( diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 25f801faf45..327701fb38c 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -7,8 +7,9 @@ import path from "node:path"; import { describe, expect, expectTypeOf, it, vi } from "vitest"; import { - CONTAINER_REACHABILITY_IMAGE, -} from "../../../src/lib/adapters/http/container-curl-probe.ts"; + getOllamaApiCommand, + OLLAMA_HOST_DOCKER_INTERNAL, +} from "../../../src/lib/inference/local.ts"; import { assertExitZero, type CommandRunner, @@ -80,10 +81,19 @@ class FakeRunner implements CommandRunner { } describe("E2E fixture clients", () => { - it("pins the live container reachability client by digest", () => { - expect(CONTAINER_REACHABILITY_IMAGE).toBe( - "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + it("uses a digest-pinned image for the live Windows-host transport", () => { + const command = getOllamaApiCommand( + ["-sf", "http://host.docker.internal:11434/api/tags"], + OLLAMA_HOST_DOCKER_INTERNAL, ); + + expect(command.slice(0, 4)).toEqual([ + "docker", + "run", + "--rm", + expect.stringMatching(/^docker\.io\/curlimages\/curl@sha256:[a-f0-9]{64}$/u), + ]); + expect(command).not.toContain("curlimages/curl:8.10.1"); }); it.each([ From 5142e5c3721c8c9dca171e2d5c790036c2ba6bb9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 17:51:46 -0700 Subject: [PATCH 40/47] fix: serialize Ollama route ownership transitions --- src/lib/actions/sandbox/destroy.ts | 21 +++-- .../local-windows-ollama-transport.test.ts | 41 +++++++++ src/lib/inference/ollama/proxy.ts | 6 ++ src/lib/onboard/setup-inference.ts | 84 ++++++++++--------- 4 files changed, 106 insertions(+), 46 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index d4f1258e4ea..414a6c6e3a7 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -22,7 +22,10 @@ import { revokeHttpsPinRuntimeAdapterRoute, } from "../../inference/https-pin-runtime-adapter"; import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup"; -import type { OllamaUnloadResult } from "../../inference/ollama/proxy"; +import { + type OllamaUnloadResult, + withOllamaModelOwnershipTransaction, +} from "../../inference/ollama/proxy"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, normalizeRuntimeProviderIdentity, @@ -1072,13 +1075,15 @@ async function destroySandboxUnlocked( } if (sandbox?.provider?.includes("ollama")) { try { - const remainingSandboxes = registry.listSandboxes().sandboxes; - const { clearPersistedOllamaHostIfUnused } = require("../../inference/local") as { - clearPersistedOllamaHostIfUnused( - providers: readonly (string | null | undefined)[], - ): boolean; - }; - clearPersistedOllamaHostIfUnused(remainingSandboxes.map(({ provider }) => provider)); + await withOllamaModelOwnershipTransaction(() => { + const remainingSandboxes = registry.listSandboxes().sandboxes; + const { clearPersistedOllamaHostIfUnused } = require("../../inference/local") as { + clearPersistedOllamaHostIfUnused( + providers: readonly (string | null | undefined)[], + ): boolean; + }; + clearPersistedOllamaHostIfUnused(remainingSandboxes.map(({ provider }) => provider)); + }); } catch (error) { console.warn( ` ${YW}⚠${R} Failed to retire the final local Ollama route receipt: ${redactDestroyError(error)}`, diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index ab7ca4d242b..e8bd69dc8b5 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -27,6 +27,7 @@ import { validateLocalProvider, validateOllamaModel, } from "./local"; +import { withOllamaModelOwnershipTransaction } from "./ollama/proxy"; function respondsOnlyThroughDockerDesktop(apiPath: string, response: string) { return vi.fn((command: readonly string[]) => { @@ -120,6 +121,46 @@ describe("Windows-host Ollama transport", () => { } }); + it("serializes route publication with final ownership retirement", async () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-transition-")); + const providers: string[] = []; + let staged!: () => void; + let resume!: () => void; + const stagedRoute = new Promise((resolve) => { + staged = resolve; + }); + const resumeOnboarding = new Promise((resolve) => { + resume = resolve; + }); + + try { + const onboarding = withOllamaModelOwnershipTransaction(async () => { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + staged(); + await resumeOnboarding; + providers.push("ollama-local"); + }); + await stagedRoute; + + let retirementEntered = false; + const retirement = withOllamaModelOwnershipTransaction(() => { + retirementEntered = true; + clearPersistedOllamaHostIfUnused(providers, stateRoot); + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(retirementEntered).toBe(false); + + resume(); + await onboarding; + await retirement; + + expect(retirementEntered).toBe(true); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("re-probes a stale persisted route before fresh-process connect discovery", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-connect-")); try { diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 4cd8f01bb50..39f7de74e24 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -134,6 +134,11 @@ function withOllamaModelOwnershipLock(operation: () => T): T { return withMcpLifecycleLockSync(OLLAMA_MODEL_OWNERSHIP_LOCK, operation); } +/** Serialize async host-route publication with final ownership retirement. */ +function withOllamaModelOwnershipTransaction(operation: () => Promise | T): Promise { + return withMcpLifecycleLock(OLLAMA_MODEL_OWNERSHIP_LOCK, operation); +} + function withOllamaProxyLifecycleTransaction(operation: () => Promise | T): Promise { // Async setup steps can call the synchronous helpers below while retaining // this lock through the shared re-entrant lifecycle-lock context. @@ -1731,5 +1736,6 @@ export { startOllamaAuthProxy, unloadOllamaModels, withOllamaModelOwnershipLock, + withOllamaModelOwnershipTransaction, withOllamaProxyLifecycleTransaction, }; diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index ea429f832c3..afed00b5b8b 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -32,6 +32,7 @@ import { startOllamaAuthProxy, type OllamaUnloadResult, withOllamaModelOwnershipLock, + withOllamaModelOwnershipTransaction, } from "../inference/ollama/proxy"; import { assertNoOpenShellGatewayEndpointOverride, @@ -226,6 +227,7 @@ export type SetupInferenceDeps = ProviderBranchDeps & { listSandboxes?: typeof import("../state/registry").listSandboxes; unloadOllamaModels?: (onlyModels: readonly string[]) => OllamaUnloadResult | void; withOllamaModelOwnershipLock?: typeof withOllamaModelOwnershipLock; + withOllamaModelOwnershipTransaction?: typeof withOllamaModelOwnershipTransaction; localInferenceTimeoutSecs: number; vllmLocalCredentialEnv: string; getManagedVllmProviderBinding?: () => { @@ -1018,44 +1020,50 @@ export function createSetupInference( return outcome.result; } } else if (provider === "ollama-local") { - const outcome = await inferenceProviders.setupOllamaLocalInference( - { - model, - provider, - allowToolsIncompatible: options.allowToolsIncompatible === true, - ...(hostLocalRoute ? {} : { preparedProxyToken: options.preparedOllamaProxyToken }), - }, - { - ...commonDeps, - validateLocalProvider: hostLocalRoute - ? () => ({ ok: true as const }) - : deps.validateLocalProvider, - getLocalProviderBaseUrl: hostLocalRoute - ? () => hostLocalRoute.gatewayProviderBaseUrl - : deps.getLocalProviderBaseUrl, - applyLocalInferenceRoute: resolveLocalInferenceRouteApplier( - hostLocalRoute - ? { - ...deps, - exitProcess: commonDeps.exitProcess, - error: commonDeps.error, - } - : deps, - runGatewayOpenshell, - revalidateSandboxIdentity, - ), - run: deps.run, - shouldFrontOllamaWithProxy: hostLocalRoute - ? () => false - : deps.shouldFrontOllamaWithProxy, - ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, - isProxyHealthy: deps.isProxyHealthy, - getOllamaProxyToken: deps.getOllamaProxyToken, - persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, - localInference: deps.localInference, - providerOwnedInferenceProof: hostLocalRoute?.receipt.inference, - OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, - }, + const withOwnershipTransaction = + deps.withOllamaModelOwnershipTransaction ?? withOllamaModelOwnershipTransaction; + const outcome = await withOwnershipTransaction(() => + inferenceProviders.setupOllamaLocalInference( + { + model, + provider, + allowToolsIncompatible: options.allowToolsIncompatible === true, + ...(hostLocalRoute + ? {} + : { preparedProxyToken: options.preparedOllamaProxyToken }), + }, + { + ...commonDeps, + validateLocalProvider: hostLocalRoute + ? () => ({ ok: true as const }) + : deps.validateLocalProvider, + getLocalProviderBaseUrl: hostLocalRoute + ? () => hostLocalRoute.gatewayProviderBaseUrl + : deps.getLocalProviderBaseUrl, + applyLocalInferenceRoute: resolveLocalInferenceRouteApplier( + hostLocalRoute + ? { + ...deps, + exitProcess: commonDeps.exitProcess, + error: commonDeps.error, + } + : deps, + runGatewayOpenshell, + revalidateSandboxIdentity, + ), + run: deps.run, + shouldFrontOllamaWithProxy: hostLocalRoute + ? () => false + : deps.shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, + isProxyHealthy: deps.isProxyHealthy, + getOllamaProxyToken: deps.getOllamaProxyToken, + persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, + localInference: deps.localInference, + providerOwnedInferenceProof: hostLocalRoute?.receipt.inference, + OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, + }, + ), ); if (outcome.done) { if (hostLocalRoute && hostLocalGatewayMutation && hostLocalSelection) { From 60326210c283834e319a8ba2be778cddf32ee1ee Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 17:38:56 -0700 Subject: [PATCH 41/47] test: guard retired Windows sleep path Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 928500239fb..618383b6508 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -27,7 +27,7 @@ function loadWindowsOllamaWithMocks( // Prove the retired subprocess-sleep path stays unused: the module must not // call child_process.spawnSync for its fixed readiness delays. const originalSpawnSync = childProcess.spawnSync; - const spawnSyncSpy = vi.fn(() => ({ status: 0 })); + const spawnSyncSpy = vi.fn((_command: string, _args?: readonly string[]) => ({ status: 0 })); childProcess.spawnSync = spawnSyncSpy; delete require.cache[WINDOWS_DIST_PATH]; @@ -106,7 +106,10 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks( + run, + runCapture, + ); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); @@ -140,6 +143,8 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); + expect(atomicsWaitSpy).not.toHaveBeenCalled(); + expect(spawnSyncSpy.mock.calls.some(([command]) => command === "sleep")).toBe(false); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { From 1cb5f9deb07cf5abafc0b8aaf1c2f90a988f6ef5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 18:16:36 -0700 Subject: [PATCH 42/47] test: allow bounded Windows readiness waits --- src/lib/inference/ollama/windows.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 618383b6508..c353ee8d9ff 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -106,10 +106,7 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks( - run, - runCapture, - ); + const { windows, restore, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); @@ -143,7 +140,6 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); - expect(atomicsWaitSpy).not.toHaveBeenCalled(); expect(spawnSyncSpy.mock.calls.some(([command]) => command === "sleep")).toBe(false); }); From e43ae80d273ab09aadfbe9b122d1bbfe4b8370b2 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 18:21:32 -0700 Subject: [PATCH 43/47] test: focus Windows fallback on behavior Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index c353ee8d9ff..41bbaccb5d1 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -5,7 +5,6 @@ import { createRequire } from "node:module"; import { describe, expect, it, vi } from "vitest"; const require = createRequire(import.meta.url); -const childProcess = require("node:child_process"); const WINDOWS_DIST_PATH = require.resolve("./windows"); const RUNNER_PATH = require.resolve("../../runner"); const LOCAL_INFERENCE_PATH = require.resolve("../local"); @@ -23,12 +22,7 @@ function loadWindowsOllamaWithMocks( const originalRun = runner.run; const originalRunCapture = runner.runCapture; // Stub the blocking wait so this test does not spend time on retry delays. - const atomicsWaitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); - // Prove the retired subprocess-sleep path stays unused: the module must not - // call child_process.spawnSync for its fixed readiness delays. - const originalSpawnSync = childProcess.spawnSync; - const spawnSyncSpy = vi.fn((_command: string, _args?: readonly string[]) => ({ status: 0 })); - childProcess.spawnSync = spawnSyncSpy; + const atomicsWaitStub = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); delete require.cache[WINDOWS_DIST_PATH]; runner.run = run; @@ -36,14 +30,11 @@ function loadWindowsOllamaWithMocks( return { windows: require(WINDOWS_DIST_PATH), - atomicsWaitSpy, - spawnSyncSpy, restore() { delete require.cache[WINDOWS_DIST_PATH]; runner.run = originalRun; runner.runCapture = originalRunCapture; - childProcess.spawnSync = originalSpawnSync; - atomicsWaitSpy.mockRestore(); + atomicsWaitStub.mockRestore(); }, }; } @@ -59,11 +50,10 @@ describe("Windows Ollama helper", () => { const localInference = require(LOCAL_INFERENCE_PATH); localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const { windows, restore, atomicsWaitSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.awaitWindowsOllamaReady()).toBe(false); - expect(atomicsWaitSpy).toHaveBeenCalledTimes(15); expect(runCapture).toHaveBeenCalledTimes(15); expect(localInference.getResolvedOllamaHost()).toBe("127.0.0.1"); } finally { @@ -106,7 +96,7 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); @@ -140,7 +130,6 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); - expect(spawnSyncSpy.mock.calls.some(([command]) => command === "sleep")).toBe(false); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { From ebc31a20cb85e3361cfdb2ba0bae293aac567846 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 18:46:00 -0700 Subject: [PATCH 44/47] test: reject subprocess Windows readiness sleeps --- src/lib/inference/ollama/windows.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 41bbaccb5d1..135fb365592 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -5,6 +5,7 @@ import { createRequire } from "node:module"; import { describe, expect, it, vi } from "vitest"; const require = createRequire(import.meta.url); +const childProcess = require("node:child_process"); const WINDOWS_DIST_PATH = require.resolve("./windows"); const RUNNER_PATH = require.resolve("../../runner"); const LOCAL_INFERENCE_PATH = require.resolve("../local"); @@ -23,6 +24,9 @@ function loadWindowsOllamaWithMocks( const originalRunCapture = runner.runCapture; // Stub the blocking wait so this test does not spend time on retry delays. const atomicsWaitStub = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + const originalSpawnSync = childProcess.spawnSync; + const spawnSyncSpy = vi.fn((_command: string, _args?: readonly string[]) => ({ status: 0 })); + childProcess.spawnSync = spawnSyncSpy; delete require.cache[WINDOWS_DIST_PATH]; runner.run = run; @@ -30,10 +34,12 @@ function loadWindowsOllamaWithMocks( return { windows: require(WINDOWS_DIST_PATH), + spawnSyncSpy, restore() { delete require.cache[WINDOWS_DIST_PATH]; runner.run = originalRun; runner.runCapture = originalRunCapture; + childProcess.spawnSync = originalSpawnSync; atomicsWaitStub.mockRestore(); }, }; @@ -96,7 +102,7 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); @@ -130,6 +136,7 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); + expect(spawnSyncSpy.mock.calls.some(([command]) => command === "sleep")).toBe(false); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { From f7975ea3f119543ab6ecfb1fc5a3dd6598b59f51 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 19:01:56 -0700 Subject: [PATCH 45/47] test: isolate missing advisor model lookup --- .../pull-requests/pr-review-advisor-security-boundaries.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts b/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts index b928fe8ef3e..5d72eca699e 100644 --- a/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts +++ b/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts @@ -20,6 +20,7 @@ describe("PR review advisor security boundaries", () => { const credentialEnv = "PR_REVIEW_ADVISOR_TEST_API_KEY"; vi.stubEnv(credentialEnv, "test-secret"); const configDir = fs.mkdtempSync(path.join(ROOT, ".tmp-pr-advisor-config-")); + vi.spyOn(ModelRegistry.prototype, "find").mockReturnValue(undefined); try { await expect( From 2587eff6802060de0dca933197e925ef5f5c8a1d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 19:14:43 -0700 Subject: [PATCH 46/47] fix: protect local Ollama route ownership Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/destroy.test.ts | 36 ++++++ src/lib/actions/sandbox/destroy.ts | 32 ++++-- src/lib/actions/sandbox/stop.test.ts | 21 ++++ src/lib/actions/sandbox/stop.ts | 17 ++- src/lib/inference/local-adapter-lifecycle.ts | 51 ++++++++- .../local-windows-ollama-transport.test.ts | 56 +++++++++- src/lib/inference/local.ts | 16 ++- .../inference/ollama/model-ownership.test.ts | 103 ++++++++++++++++-- src/lib/inference/ollama/model-ownership.ts | 32 ++++-- src/lib/inference/ollama/proxy.ts | 3 + src/lib/inference/ollama/windows.test.ts | 35 +++--- src/lib/onboard/inference-providers/types.ts | 5 +- src/lib/onboard/setup-inference.ts | 26 +++-- .../onboard-inference-reconciliation.test.ts | 54 +++++++-- 14 files changed, 413 insertions(+), 74 deletions(-) diff --git a/src/lib/actions/sandbox/destroy.test.ts b/src/lib/actions/sandbox/destroy.test.ts index 930da4d702b..5884c3a7809 100644 --- a/src/lib/actions/sandbox/destroy.test.ts +++ b/src/lib/actions/sandbox/destroy.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import type { SandboxEntry } from "../../state/registry"; import { assertUnambiguousDestroyContainerIdentity, cleanupSandboxServices, @@ -74,6 +75,41 @@ describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => { }); }); +describe("cleanupSandboxServices Ollama ownership", () => { + it("keeps a model shared through a compatible endpoint at the same local daemon", () => { + const own = { + name: SANDBOX, + provider: "ollama-local", + model: "llama3", + } as SandboxEntry; + const peer = { + name: "peer", + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + model: "llama3:latest", + } as SandboxEntry; + const unloadOllamaModels = vi.fn(); + + cleanupSandboxServices( + SANDBOX, + { stopHostServices: false }, + { + getSandbox: () => own, + listSandboxes: () => ({ sandboxes: [own, peer], defaultSandbox: null }), + loadPersistedOllamaHost: () => "127.0.0.1", + unloadOllamaModels, + withOllamaModelOwnershipLock: (operation) => operation(), + rmSync: vi.fn(), + runOpenshell: vi.fn(() => ({ status: 0 })), + stopGooglechatWebhookTunnel: vi.fn(() => googlechatPidDir), + googlechatWebhookTunnelPidDir: vi.fn(() => googlechatPidDir), + }, + ); + + expect(unloadOllamaModels).not.toHaveBeenCalled(); + }); +}); + describe("assertUnambiguousDestroyContainerIdentity (#8999)", () => { const dockerSandbox = { openshellDriver: "docker" } as { openshellDriver: string | null }; diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 414a6c6e3a7..a63595ab2d1 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -23,6 +23,8 @@ import { } from "../../inference/https-pin-runtime-adapter"; import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup"; import { + isLocalOllamaRouteOwner, + loadPersistedOllamaHost, type OllamaUnloadResult, withOllamaModelOwnershipTransaction, } from "../../inference/ollama/proxy"; @@ -182,6 +184,7 @@ export type CleanupSandboxServicesDeps = { sandboxName: string, releasedModels?: readonly string[], ) => void; + loadPersistedOllamaHost?: () => "127.0.0.1" | "host.docker.internal" | null; withOllamaModelOwnershipLock?: (operation: () => T) => T; ollamaModelRefsMatch?: (left: string, right: string) => boolean; runOpenshell?: RunOpenshell; @@ -281,6 +284,10 @@ export function cleanupSandboxServices( }; local.clearPendingOllamaModelCleanup(name, releasedModels); }); + const loadPersistedOllamaHost = + deps.loadPersistedOllamaHost ?? + (require("../../inference/local") as typeof import("../../inference/local")) + .loadPersistedOllamaHost; const withOllamaModelOwnershipLock = deps.withOllamaModelOwnershipLock ?? ((operation: () => T): T => { @@ -353,8 +360,9 @@ export function cleanupSandboxServices( ollamaCleanup = withOllamaModelOwnershipLock(() => { const sandbox = getSandbox(validatedSandboxName); const pending = loadPendingOllamaModelCleanup(validatedSandboxName); + const selectedHost = loadPersistedOllamaHost(); const cleanupOllamaModels = Boolean( - sandbox?.provider?.includes("ollama") || pending.length > 0, + (sandbox && isLocalOllamaRouteOwner(sandbox, selectedHost)) || pending.length > 0, ); return stopAll({ sandboxName: validatedSandboxName, @@ -380,15 +388,17 @@ export function cleanupSandboxServices( // branch a single-sandbox destroy would leave models loaded on the GPU. withOllamaModelOwnershipLock(() => { const sb = getSandbox(validatedSandboxName); + const selectedHost = loadPersistedOllamaHost(); const peers = listSandboxes().sandboxes.filter( (candidate) => - candidate.name !== validatedSandboxName && candidate.provider?.includes("ollama"), + candidate.name !== validatedSandboxName && + isLocalOllamaRouteOwner(candidate, selectedHost), ); const pending = loadPendingOllamaModelCleanup(validatedSandboxName); const currentModel = String(sb?.model ?? "").trim(); const candidates = [ ...pending, - ...(sb?.provider?.includes("ollama") && currentModel ? [currentModel] : []), + ...(sb && isLocalOllamaRouteOwner(sb, selectedHost) && currentModel ? [currentModel] : []), ].filter( (model, index, models) => models.findIndex((candidate) => ollamaModelRefsMatch(candidate, model)) === index && @@ -1073,16 +1083,18 @@ async function destroySandboxUnlocked( ` ${YW}⚠${R} Failed to retire portable lifecycle authority for '${sandboxName}': ${redactDestroyError(error)}`, ); } - if (sandbox?.provider?.includes("ollama")) { + const localInference = require("../../inference/local") as { + clearPersistedOllamaHostIfUnused( + routes: readonly { provider?: string | null; endpointUrl?: string | null }[], + ): boolean; + }; + if (sandbox && isLocalOllamaRouteOwner(sandbox)) { try { await withOllamaModelOwnershipTransaction(() => { + const selectedHost = loadPersistedOllamaHost(); + if (!isLocalOllamaRouteOwner(sandbox, selectedHost)) return; const remainingSandboxes = registry.listSandboxes().sandboxes; - const { clearPersistedOllamaHostIfUnused } = require("../../inference/local") as { - clearPersistedOllamaHostIfUnused( - providers: readonly (string | null | undefined)[], - ): boolean; - }; - clearPersistedOllamaHostIfUnused(remainingSandboxes.map(({ provider }) => provider)); + localInference.clearPersistedOllamaHostIfUnused(remainingSandboxes); }); } catch (error) { console.warn( diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 4af84fda5b8..fd332ae4b16 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -765,6 +765,27 @@ describe("stopSandbox Ollama GPU release", () => { ); }); + it("never unloads a model a compatible local endpoint sibling also uses", () => { + const unloadOllamaModels = vi.fn(() => successfulUnload()); + const peer = sandbox({ + endpointUrl: "http://127.0.0.1:11434/v1", + model: "qwen2.5:7b", + name: "peer", + provider: "compatible-endpoint", + }); + const h = harness({ + listSandboxes: registryOf(ollamaSandbox, peer), + loadPersistedOllamaHost: () => "127.0.0.1", + unloadOllamaModels, + }); + h.getSandbox.mockReturnValue(ollamaSandbox); + + const result = stopSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(unloadOllamaModels).not.toHaveBeenCalled(); + }); + it("ignores a stopped sibling registry row and releases the exclusive model (#10074)", () => { const unloadOllamaModels = vi.fn(() => successfulUnload()); const stoppedPeer = sandbox({ diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 356ea66e0f1..381f8b2783c 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -4,7 +4,9 @@ import { CLI_NAME } from "../../cli/branding"; import { decideOllamaModelOwnership, + isLocalOllamaRouteOwner, matchingOllamaModelPeers, + type OllamaHostRoute, } from "../../inference/ollama/model-ownership"; import type { OllamaUnloadResult } from "../../inference/ollama/proxy"; import { @@ -139,16 +141,19 @@ function releaseStoppedSandboxOllamaModel( deps: SandboxStopDeps, log: (message: string) => void, ): OllamaStopReleaseResult { - if (!sandbox.provider?.includes("ollama")) return { ok: true }; + if (!isLocalOllamaRouteOwner(sandbox)) return { ok: true }; try { + const proxy = require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"); const withOwnershipLock = - deps.withOllamaModelOwnershipLock ?? - (require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy")) - .withOllamaModelOwnershipLock; + deps.withOllamaModelOwnershipLock ?? proxy.withOllamaModelOwnershipLock; + const loadPersistedOllamaHost = + deps.loadPersistedOllamaHost ?? proxy.loadPersistedOllamaHost; return withOwnershipLock(() => { + const selectedHost = loadPersistedOllamaHost(); + if (!isLocalOllamaRouteOwner(sandbox, selectedHost)) return { ok: true }; const { sandboxes } = (deps.listSandboxes ?? registry.listSandboxes)(); - const matchingPeers = matchingOllamaModelPeers(sandbox, sandboxes); + const matchingPeers = matchingOllamaModelPeers(sandbox, sandboxes, selectedHost); const discovery = ( deps.discoverActiveOllamaSandboxNames ?? discoverActiveOllamaSandboxNames )(matchingPeers, deps.environment ?? process.env); @@ -166,6 +171,7 @@ function releaseStoppedSandboxOllamaModel( sandbox, sandboxes, discovery.activeSandboxNames, + selectedHost, ); if (ownership.kind === "missing-model") { log(" Ollama model release skipped: the sandbox registry has no model."); @@ -238,6 +244,7 @@ export interface SandboxStopDeps { ) => OllamaActiveOwnershipDiscovery; unloadOllamaModels?: (onlyModels: readonly string[]) => OllamaUnloadResult; decideOllamaModelOwnership?: typeof decideOllamaModelOwnership; + loadPersistedOllamaHost?: () => OllamaHostRoute | null; withOllamaModelOwnershipLock?: typeof import("../../inference/ollama/proxy").withOllamaModelOwnershipLock; withLifecycleLockSync?: typeof withSandboxLifecycleLockSync; log?: (message: string) => void; diff --git a/src/lib/inference/local-adapter-lifecycle.ts b/src/lib/inference/local-adapter-lifecycle.ts index be37413ba43..ae5fda2aca2 100644 --- a/src/lib/inference/local-adapter-lifecycle.ts +++ b/src/lib/inference/local-adapter-lifecycle.ts @@ -8,7 +8,7 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; -import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../core/ports"; +import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT, OLLAMA_PORT } from "../core/ports"; import { waitUntilAsync } from "../core/wait"; import { rejectSymlinksOnPath } from "../state/config-io"; import { nemoclawStateRoot } from "../state/state-root"; @@ -42,6 +42,55 @@ export function resolveSharedLocalAdapterStateRoot(homeDir: string = os.homedir( export const SHARED_LOCAL_ADAPTER_STATE_DIR = resolveSharedLocalAdapterStateRoot(); export const LOCAL_ADAPTER_HEALTH_MAX_RESPONSE_BYTES = 64 * 1024; +export const OLLAMA_LOCALHOST = "127.0.0.1"; +export const OLLAMA_HOST_DOCKER_INTERNAL = "host.docker.internal"; + +export type OllamaHostRoute = + | typeof OLLAMA_LOCALHOST + | typeof OLLAMA_HOST_DOCKER_INTERNAL; + +/** Registry fields that identify a route backed by NemoClaw's host Ollama daemon. */ +export type OllamaRouteHolder = { + readonly provider?: string | null; + readonly endpointUrl?: string | null; +}; + +function isSupportedOllamaRouteHost(host: string): host is OllamaHostRoute { + return host === OLLAMA_LOCALHOST || host === OLLAMA_HOST_DOCKER_INTERNAL; +} + +/** + * Return whether a recorded inference route uses NemoClaw's host Ollama daemon. + * + * Direct and legacy Ollama providers own that daemon by definition. A + * compatible endpoint owns it only when its credential-free HTTP URL names + * the selected fixed host route and Ollama port. Remote compatible endpoints + * are never classified as local owners. + */ +export function isLocalOllamaRouteOwner( + route: OllamaRouteHolder, + selectedHost: OllamaHostRoute | null = null, +): boolean { + if (route.provider === "ollama-local" || route.provider?.startsWith("ollama/")) return true; + if (route.provider !== "compatible-endpoint" || !route.endpointUrl) return false; + + try { + const endpoint = new URL(route.endpointUrl); + const endpointHost = endpoint.hostname.toLowerCase(); + const hostMatches = selectedHost + ? endpointHost === selectedHost + : isSupportedOllamaRouteHost(endpointHost); + return ( + endpoint.protocol === "http:" && + endpoint.username === "" && + endpoint.password === "" && + Number(endpoint.port) === OLLAMA_PORT && + hostMatches + ); + } catch { + return false; + } +} export function ensureLocalAdapterStateDir(stateDir = DEFAULT_LOCAL_ADAPTER_STATE_DIR): void { rejectSymlinksOnPath(stateDir); diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index e8bd69dc8b5..63b8bb3ab98 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -102,7 +102,7 @@ describe("Windows-host Ollama transport", () => { try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); - expect(clearPersistedOllamaHostIfUnused(["nvidia-prod"], stateRoot)).toBe(true); + expect(clearPersistedOllamaHostIfUnused([{ provider: "nvidia-prod" }], stateRoot)).toBe(true); expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); } finally { rmSync(stateRoot, { recursive: true, force: true }); @@ -114,16 +114,62 @@ describe("Windows-host Ollama transport", () => { try { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); - expect(clearPersistedOllamaHostIfUnused(["ollama-local"], stateRoot)).toBe(false); + expect(clearPersistedOllamaHostIfUnused([{ provider: "ollama-local" }], stateRoot)).toBe( + false, + ); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("retains the route for a compatible endpoint at the selected local daemon", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-compatible-retain-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect( + clearPersistedOllamaHostIfUnused( + [ + { + provider: "compatible-endpoint", + endpointUrl: "http://host.docker.internal:11434/v1", + }, + ], + stateRoot, + ), + ).toBe(false); expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); } finally { rmSync(stateRoot, { recursive: true, force: true }); } }); + it("retires the route when only a remote compatible endpoint remains", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-compatible-remote-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect( + clearPersistedOllamaHostIfUnused( + [ + { + provider: "compatible-endpoint", + endpointUrl: "https://ollama.example.com:11434/v1", + }, + ], + stateRoot, + ), + ).toBe(true); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("serializes route publication with final ownership retirement", async () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-transition-")); - const providers: string[] = []; + const routes: Array<{ provider: string }> = []; let staged!: () => void; let resume!: () => void; const stagedRoute = new Promise((resolve) => { @@ -138,14 +184,14 @@ describe("Windows-host Ollama transport", () => { persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); staged(); await resumeOnboarding; - providers.push("ollama-local"); + routes.push({ provider: "ollama-local" }); }); await stagedRoute; let retirementEntered = false; const retirement = withOllamaModelOwnershipTransaction(() => { retirementEntered = true; - clearPersistedOllamaHostIfUnused(providers, stateRoot); + clearPersistedOllamaHostIfUnused(routes, stateRoot); }); await new Promise((resolve) => setTimeout(resolve, 0)); expect(retirementEntered).toBe(false); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 4925fb5c67b..29c2563fb72 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -31,9 +31,13 @@ import { type CaptureResult, run, runCapture, runCaptureEx, shellQuote } from ". import { buildSubprocessEnv } from "../subprocess-env"; import { + isLocalOllamaRouteOwner, + OLLAMA_HOST_DOCKER_INTERNAL, + OLLAMA_LOCALHOST, readLocalAdapterJsonFile, removeLocalAdapterFile, resolveSharedLocalAdapterStateRoot, + type OllamaRouteHolder, writeLocalAdapterJsonFile, } from "./local-adapter-lifecycle"; import { detectNvidiaPlatform } from "./nim"; @@ -134,8 +138,11 @@ export type RunCaptureExFn = (cmd: string[], opts?: { env?: NodeJS.ProcessEnv }) // Hosts that local-provider discovery may try when probing Ollama. The Windows // onboarding path separately checks host.docker.internal from Docker Desktop's // network context because the alias may not resolve from the WSL host. -export const OLLAMA_LOCALHOST = "127.0.0.1"; -export const OLLAMA_HOST_DOCKER_INTERNAL = "host.docker.internal"; +export { + isLocalOllamaRouteOwner, + OLLAMA_HOST_DOCKER_INTERNAL, + OLLAMA_LOCALHOST, +} from "./local-adapter-lifecycle"; /** Build the credential-free Docker Desktop probe for Windows-host Ollama. */ export function getWindowsHostOllamaDockerReachabilityArgs(): string[] { @@ -271,10 +278,11 @@ export function clearPersistedOllamaHost( } export function clearPersistedOllamaHostIfUnused( - providers: readonly (string | null | undefined)[], + routes: readonly OllamaRouteHolder[], stateRoot: string = resolveSharedLocalAdapterStateRoot(), ): boolean { - if (providers.some((provider) => provider?.includes("ollama"))) return false; + const selectedHost = loadPersistedOllamaHost(stateRoot); + if (routes.some((route) => isLocalOllamaRouteOwner(route, selectedHost))) return false; clearPersistedOllamaHost(stateRoot); return true; } diff --git a/src/lib/inference/ollama/model-ownership.test.ts b/src/lib/inference/ollama/model-ownership.test.ts index 60cb341b687..557df044639 100644 --- a/src/lib/inference/ollama/model-ownership.test.ts +++ b/src/lib/inference/ollama/model-ownership.test.ts @@ -12,17 +12,54 @@ import { exclusivelyHeldOllamaModel, loadPendingOllamaModelCleanup, type OllamaModelHolder, + type OllamaModelRoute, persistPendingOllamaModelCleanup, supersededOllamaModel, } from "./model-ownership"; +import { isLocalOllamaRouteOwner } from "./model-ownership"; function holder(overrides: Partial = {}): OllamaModelHolder { return { name: "test-box", provider: "ollama-local", model: "llama3", ...overrides }; } +function route(model: string, overrides: Partial = {}): OllamaModelRoute { + return { provider: "ollama-local", model, ...overrides }; +} + +describe("isLocalOllamaRouteOwner", () => { + it.each([["ollama-local"], ["ollama/qwen3-vl:4b"]])( + "recognizes the direct local provider %s", + (provider) => { + expect(isLocalOllamaRouteOwner({ provider }, "127.0.0.1")).toBe(true); + }, + ); + + it("recognizes a compatible endpoint at the selected local daemon", () => { + expect( + isLocalOllamaRouteOwner( + { + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + }, + "127.0.0.1", + ), + ).toBe(true); + }); + + it.each([ + ["a remote endpoint", "https://ollama.example.com:11434/v1"], + ["the other fixed host route", "http://host.docker.internal:11434/v1"], + ["a different port", "http://127.0.0.1:11435/v1"], + ])("excludes %s", (_label, endpointUrl) => { + expect( + isLocalOllamaRouteOwner({ provider: "compatible-endpoint", endpointUrl }, "127.0.0.1"), + ).toBe(false); + }); +}); + describe("supersededOllamaModel", () => { it("releases the previous model when a re-onboard moves to a different one (#9110)", () => { - expect(supersededOllamaModel(holder(), "qwen2.5:7b", [holder()])).toBe("llama3"); + expect(supersededOllamaModel(holder(), route("qwen2.5:7b"), [holder()])).toBe("llama3"); }); it.each([ @@ -31,41 +68,69 @@ describe("supersededOllamaModel", () => { ["an explicit latest tag on the previous model", "llama3:latest", "llama3"], ])("keeps the model when the next ref is %s (#9110)", (_label, previousModel, nextModel) => { const previous = holder({ model: previousModel }); - expect(supersededOllamaModel(previous, nextModel, [previous])).toBeNull(); + expect(supersededOllamaModel(previous, route(nextModel), [previous])).toBeNull(); }); it.each([["llama3"], ["llama3:latest"]])( "keeps a model an Ollama peer records as %s (#9110)", (peerModel) => { const peer = holder({ model: peerModel, name: "peer" }); - expect(supersededOllamaModel(holder(), "qwen2.5:7b", [holder(), peer])).toBeNull(); + expect(supersededOllamaModel(holder(), route("qwen2.5:7b"), [holder(), peer])).toBeNull(); }, ); it("releases the model when peers hold different ones (#9110)", () => { const peer = holder({ model: "llama3:8b", name: "peer" }); - expect(supersededOllamaModel(holder(), "qwen2.5:7b", [holder(), peer])).toBe("llama3"); + expect(supersededOllamaModel(holder(), route("qwen2.5:7b"), [holder(), peer])).toBe("llama3"); }); it.each([["nvidia-prod"], ["vllm-local"], [undefined]])( "does nothing when the previous provider is %s (#9110)", (provider) => { const previous = holder({ provider }); - expect(supersededOllamaModel(previous, "qwen2.5:7b", [previous])).toBeNull(); + expect(supersededOllamaModel(previous, route("qwen2.5:7b"), [previous])).toBeNull(); }, ); it("does nothing when the previous model is unrecorded (#9110)", () => { const previous = holder({ model: undefined }); - expect(supersededOllamaModel(previous, "qwen2.5:7b", [previous])).toBeNull(); + expect(supersededOllamaModel(previous, route("qwen2.5:7b"), [previous])).toBeNull(); }); it.each([[""], [" "]])("does nothing when the next model is %j (#9110)", (nextModel) => { - expect(supersededOllamaModel(holder(), nextModel, [holder()])).toBeNull(); + expect(supersededOllamaModel(holder(), route(nextModel), [holder()])).toBeNull(); }); it("does nothing when there is no previous entry (#9110)", () => { - expect(supersededOllamaModel(null, "qwen2.5:7b", [])).toBeNull(); + expect(supersededOllamaModel(null, route("qwen2.5:7b"), [])).toBeNull(); + }); + + it("keeps a model selected through a compatible endpoint at the same local daemon", () => { + expect( + supersededOllamaModel( + holder(), + route("llama3:latest", { + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + }), + [holder()], + "127.0.0.1", + ), + ).toBeNull(); + }); + + it("does not mistake a remote compatible endpoint for the local daemon", () => { + expect( + supersededOllamaModel( + holder(), + route("llama3", { + provider: "compatible-endpoint", + endpointUrl: "https://ollama.example.com:11434/v1", + }), + [holder()], + "127.0.0.1", + ), + ).toBe("llama3"); }); }); @@ -121,6 +186,28 @@ describe("decideOllamaModelOwnership", () => { ), ).toEqual({ kind: "exclusive", model: "llama3", stalePeers: [] }); }); + + it("protects a matching compatible endpoint at the same local daemon", () => { + const activePeer = holder({ + name: "compatible-peer", + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + }); + + expect( + decideOllamaModelOwnership( + holder(), + [holder(), activePeer], + new Set(["compatible-peer"]), + "127.0.0.1", + ), + ).toEqual({ + kind: "shared-active", + model: "llama3", + activePeers: ["compatible-peer"], + stalePeers: [], + }); + }); }); describe("exclusivelyHeldOllamaModel", () => { diff --git a/src/lib/inference/ollama/model-ownership.ts b/src/lib/inference/ollama/model-ownership.ts index 22bb1803e11..84a3d69e5f6 100644 --- a/src/lib/inference/ollama/model-ownership.ts +++ b/src/lib/inference/ollama/model-ownership.ts @@ -5,13 +5,18 @@ import path from "node:path"; import type { SandboxEntry } from "../../state/registry"; import { + isLocalOllamaRouteOwner, readLocalAdapterJsonFile, removeLocalAdapterFile, resolveSharedLocalAdapterStateRoot, + type OllamaHostRoute, writeLocalAdapterJsonFile, } from "../local-adapter-lifecycle"; import { ollamaModelRefsMatch } from "./model-discovery"; +export { isLocalOllamaRouteOwner } from "../local-adapter-lifecycle"; +export type { OllamaHostRoute, OllamaRouteHolder } from "../local-adapter-lifecycle"; + const PENDING_CLEANUP_DIRECTORY = "ollama-pending-model-cleanup"; const SAFE_SANDBOX_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; @@ -88,7 +93,9 @@ export function clearPendingOllamaModelCleanup( } /** The registry fields an Ollama GPU-release decision reads. */ -export type OllamaModelHolder = Pick; +export type OllamaModelHolder = Pick; + +export type OllamaModelRoute = Pick; export type OllamaModelOwnershipDecision = | { readonly kind: "missing-model" } @@ -115,13 +122,14 @@ export type OllamaModelOwnershipDecision = export function matchingOllamaModelPeers( sandbox: OllamaModelHolder, peers: readonly T[], + selectedHost: OllamaHostRoute | null = null, ): T[] { const model = sandbox.model?.trim(); if (!model) return []; return peers.filter( (peer) => peer.name !== sandbox.name && - !!peer.provider?.includes("ollama") && + isLocalOllamaRouteOwner(peer, selectedHost) && !!peer.model && ollamaModelRefsMatch(peer.model, model), ); @@ -131,11 +139,12 @@ export function decideOllamaModelOwnership( sandbox: OllamaModelHolder, peers: readonly OllamaModelHolder[], activeSandboxNames: ReadonlySet, + selectedHost: OllamaHostRoute | null = null, ): OllamaModelOwnershipDecision { const model = sandbox.model?.trim(); if (!model) return { kind: "missing-model" }; - const matchingPeers = matchingOllamaModelPeers(sandbox, peers); + const matchingPeers = matchingOllamaModelPeers(sandbox, peers, selectedHost); const activePeers = matchingPeers .filter((peer) => activeSandboxNames.has(peer.name)) .map((peer) => peer.name) @@ -157,11 +166,13 @@ export function decideOllamaModelOwnership( export function exclusivelyHeldOllamaModel( sandbox: OllamaModelHolder, peers: readonly OllamaModelHolder[], + selectedHost: OllamaHostRoute | null = null, ): string | null { const decision = decideOllamaModelOwnership( sandbox, peers, new Set(peers.map((peer) => peer.name)), + selectedHost, ); return decision.kind === "exclusive" ? decision.model : null; } @@ -180,13 +191,16 @@ export function exclusivelyHeldOllamaModel( */ export function supersededOllamaModel( previous: OllamaModelHolder | null, - nextModel: string, + next: OllamaModelRoute, peers: readonly OllamaModelHolder[], + selectedHost: OllamaHostRoute | null = null, ): string | null { - if (!previous?.provider?.includes("ollama")) return null; - const next = nextModel?.trim(); - if (!next) return null; - const held = exclusivelyHeldOllamaModel(previous, peers); + if (!previous || !isLocalOllamaRouteOwner(previous, selectedHost)) return null; + const nextModel = next.model?.trim(); + if (!nextModel) return null; + const held = exclusivelyHeldOllamaModel(previous, peers, selectedHost); if (!held) return null; - return ollamaModelRefsMatch(held, next) ? null : held; + return isLocalOllamaRouteOwner(next, selectedHost) && ollamaModelRefsMatch(held, nextModel) + ? null + : held; } diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 39f7de74e24..411278ba747 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -32,6 +32,7 @@ const { const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./model-discovery"); const { clearPendingOllamaModelCleanup, + isLocalOllamaRouteOwner, loadPendingOllamaModelCleanup, }: typeof import("./model-ownership") = require("./model-ownership"); const { @@ -1721,9 +1722,11 @@ export { ensureOllamaAuthProxy, getOllamaProxyToken, getOllamaPullTimeoutMs, + isLocalOllamaRouteOwner, isProxyHealthy, killStaleProxy, loadPendingOllamaModelCleanup, + loadPersistedOllamaHost, noAuthProxy, ollamaModelRefsMatch, persistAndProbeOllamaProxy, diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 135fb365592..6069431cf2e 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -5,7 +5,6 @@ import { createRequire } from "node:module"; import { describe, expect, it, vi } from "vitest"; const require = createRequire(import.meta.url); -const childProcess = require("node:child_process"); const WINDOWS_DIST_PATH = require.resolve("./windows"); const RUNNER_PATH = require.resolve("../../runner"); const LOCAL_INFERENCE_PATH = require.resolve("../local"); @@ -24,9 +23,6 @@ function loadWindowsOllamaWithMocks( const originalRunCapture = runner.runCapture; // Stub the blocking wait so this test does not spend time on retry delays. const atomicsWaitStub = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); - const originalSpawnSync = childProcess.spawnSync; - const spawnSyncSpy = vi.fn((_command: string, _args?: readonly string[]) => ({ status: 0 })); - childProcess.spawnSync = spawnSyncSpy; delete require.cache[WINDOWS_DIST_PATH]; runner.run = run; @@ -34,12 +30,10 @@ function loadWindowsOllamaWithMocks( return { windows: require(WINDOWS_DIST_PATH), - spawnSyncSpy, restore() { delete require.cache[WINDOWS_DIST_PATH]; runner.run = originalRun; runner.runCapture = originalRunCapture; - childProcess.spawnSync = originalSpawnSync; atomicsWaitStub.mockRestore(); }, }; @@ -48,19 +42,33 @@ function loadWindowsOllamaWithMocks( describe("Windows Ollama helper", () => { it("rejects a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); - const runCapture = vi.fn((command: string | string[]) => - Array.isArray(command) && command.at(-1) === WINDOWS_OLLAMA_TAGS_URL - ? "proxy response" - : "", - ); const localInference = require(LOCAL_INFERENCE_PATH); + const runCapture = vi.fn((command: string | string[]) => { + expect(command).toEqual( + expect.arrayContaining([ + "docker", + "run", + "--rm", + localInference.CONTAINER_REACHABILITY_IMAGE, + WINDOWS_OLLAMA_TAGS_URL, + ]), + ); + expect(command.slice(0, 4)).toEqual([ + "docker", + "run", + "--rm", + localInference.CONTAINER_REACHABILITY_IMAGE, + ]); + expect(command.at(-1)).toBe(WINDOWS_OLLAMA_TAGS_URL); + return "proxy response"; + }); localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.awaitWindowsOllamaReady()).toBe(false); - expect(runCapture).toHaveBeenCalledTimes(15); + expect(runCapture.mock.calls.length).toBeGreaterThan(0); expect(localInference.getResolvedOllamaHost()).toBe("127.0.0.1"); } finally { localInference.resetOllamaHostCache(); @@ -102,7 +110,7 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); @@ -136,7 +144,6 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); - expect(spawnSyncSpy.mock.calls.some(([command]) => command === "sleep")).toBe(false); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 45ca911cef6..40d56416959 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -259,7 +259,10 @@ export type OllamaDeps = CommonDeps & { persistPendingOllamaModelCleanup?(sandboxName: string, models: readonly string[]): void; clearPendingOllamaModelCleanup?(sandboxName: string, releasedModels?: readonly string[]): void; persistResolvedOllamaHost(): () => void; - clearPersistedOllamaHostIfUnused?(providers: readonly (string | null | undefined)[]): boolean; + loadPersistedOllamaHost?(): "127.0.0.1" | "host.docker.internal" | null; + clearPersistedOllamaHostIfUnused?( + routes: readonly { provider?: string | null; endpointUrl?: string | null }[], + ): boolean; }; /** Exact provider-owned proof used instead of legacy host warmup/probes. */ providerOwnedInferenceProof?: { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index afed00b5b8b..d6747dcf46b 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -21,6 +21,7 @@ import { import { getManagedVllmProviderBinding } from "../inference/local"; import { clearPendingOllamaModelCleanup, + isLocalOllamaRouteOwner, loadPendingOllamaModelCleanup, type OllamaModelHolder, persistPendingOllamaModelCleanup, @@ -523,6 +524,7 @@ function releaseSupersededOllamaModel( previous: OllamaModelHolder | null, nextProvider: string, nextModel: string, + nextEndpointUrl: string | null, result: SetupInferenceResult, deps: SetupInferenceDeps, revalidateSandboxIdentity?: (operation: string) => void, @@ -555,20 +557,23 @@ function releaseSupersededOllamaModel( const withOwnershipLock = deps.withOllamaModelOwnershipLock ?? withOllamaModelOwnershipLock; withOwnershipLock(() => { const peers = deps.listSandboxes?.().sandboxes ?? []; - const superseded = supersededOllamaModel(previous, nextModel, peers); + const selectedHost = deps.localInference.loadPersistedOllamaHost?.() ?? null; + const nextRoute = { provider: nextProvider, model: nextModel, endpointUrl: nextEndpointUrl }; + const superseded = supersededOllamaModel(previous, nextRoute, peers, selectedHost); const pending = loadPending(previous.name); const retryablePending = pending.filter((model) => supersededOllamaModel( - { name: previous.name, provider: "ollama-local", model }, - nextModel, + { name: previous.name, provider: "ollama-local", model, endpointUrl: null }, + nextRoute, peers, + selectedHost, ), ); attemptedModels = [...new Set([...(superseded ? [superseded] : []), ...retryablePending])]; const retireRoute = - !!previous.provider?.includes("ollama") && - !nextProvider.includes("ollama") && - !peers.some((peer) => peer.provider?.includes("ollama")); + isLocalOllamaRouteOwner(previous, selectedHost) && + !isLocalOllamaRouteOwner(nextRoute, selectedHost) && + !peers.some((peer) => isLocalOllamaRouteOwner(peer, selectedHost)); if (attemptedModels.length === 0 && !retireRoute) return; try { revalidateSandboxIdentity?.("release the superseded Ollama model"); @@ -599,7 +604,7 @@ function releaseSupersededOllamaModel( `route remains active. ${recoveryAction}. ` + (pendingRecordFailure ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ")} at ${cleanup.endpoint}.` - : `Re-run onboarding, stop, or destroy '${previous.name}' to retry only: ${attemptedModels.join(", ")}.`); + : `Re-run onboarding or destroy '${previous.name}' to retry only: ${attemptedModels.join(", ")}.`); } else { clearPending(previous.name, attemptedModels); } @@ -613,12 +618,12 @@ function releaseSupersededOllamaModel( `route remains active. ` + (pendingRecordFailure ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ")} from the saved local Ollama endpoint.` - : `Re-run onboarding, stop, or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ")}.`); + : `Re-run onboarding or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ")}.`); } } const pendingAfterCleanup = loadPending(previous.name); if (retireRoute && !cleanupWarning && pendingAfterCleanup.length === 0) { - deps.localInference.clearPersistedOllamaHostIfUnused?.(peers.map((peer) => peer.provider)); + deps.localInference.clearPersistedOllamaHostIfUnused?.(peers); } }); } catch (error) { @@ -631,7 +636,7 @@ function releaseSupersededOllamaModel( `inference route remains active. ` + (pendingRecordFailure ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ") || "the superseded model"} from the saved local Ollama endpoint.` - : `Re-run onboarding, stop, or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ") || "none"}.`); + : `Re-run onboarding or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ") || "none"}.`); } if (cleanupWarning) console.warn(cleanupWarning); if (authorityRefusal) throw authorityRefusal; @@ -1229,6 +1234,7 @@ export function createSetupInference( previousSandbox, provider, model, + endpointUrl, result, deps, revalidateSandboxIdentity, diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index a7ffdfe601d..9d25955a331 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -1011,16 +1011,25 @@ console.log(JSON.stringify({ }); describe("re-onboard Ollama GPU release (#9110)", () => { - const priorEntry = { name: "test-box", provider: "ollama-local", model: "llama3" }; + type ReleaseEntry = { + name: string; + provider: string; + model: string; + endpointUrl?: string | null; + }; + const priorEntry: ReleaseEntry = { + name: "test-box", + provider: "ollama-local", + model: "llama3", + }; function releaseHarness(options: { - getSandbox: () => typeof priorEntry | null; - sandboxes: (typeof priorEntry)[] | (() => (typeof priorEntry)[]); + getSandbox: () => ReleaseEntry | null; + sandboxes: ReleaseEntry[] | (() => ReleaseEntry[]); unloadOllamaModels: NonNullable; applyLocalInferenceRoute?: () => Promise; - clearPersistedOllamaHostIfUnused?: ( - providers: readonly (string | null | undefined)[], - ) => boolean; + loadPersistedOllamaHost?: () => "127.0.0.1" | "host.docker.internal" | null; + clearPersistedOllamaHostIfUnused?: SetupInferenceDeps["localInference"]["clearPersistedOllamaHostIfUnused"]; loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; persistPendingOllamaModelCleanup?: (sandboxName: string, models: readonly string[]) => void; clearPendingOllamaModelCleanup?: ( @@ -1052,6 +1061,7 @@ describe("re-onboard Ollama GPU release (#9110)", () => { validateSandboxFacingOllamaModel: () => ({ ok: true }), runOllamaWarmup: () => {}, persistResolvedOllamaHost: () => () => {}, + loadPersistedOllamaHost: options.loadPersistedOllamaHost, clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, loadPendingOllamaModelCleanup: options.loadPendingOllamaModelCleanup ?? (() => []), persistPendingOllamaModelCleanup: options.persistPendingOllamaModelCleanup ?? (() => {}), @@ -1090,7 +1100,9 @@ describe("re-onboard Ollama GPU release (#9110)", () => { ok: true, }); - expect(clearPersistedOllamaHostIfUnused).toHaveBeenCalledWith(["vllm-local"]); + expect(clearPersistedOllamaHostIfUnused).toHaveBeenCalledWith([ + { ...priorEntry, provider: "vllm-local", model: "vllm-model" }, + ]); }); it("keeps the successful route when the superseded model unload fails (#9110)", async () => { @@ -1139,6 +1151,9 @@ describe("re-onboard Ollama GPU release (#9110)", () => { ); expect(warn).toHaveBeenCalledWith(expect.stringContaining("unload-request-failed")); expect(warn).toHaveBeenCalledWith(expect.stringContaining("Allow the model unload request")); + const warning = warn.mock.calls.map(([message]) => String(message)).join("\n"); + expect(warning).toContain("Re-run onboarding or destroy 'test-box'"); + expect(warning).not.toContain("stop, or destroy"); } finally { warn.mockRestore(); } @@ -1262,6 +1277,31 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).not.toHaveBeenCalled(); }); + it("keeps the route and shared model for a compatible local Ollama peer", async () => { + const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); + const clearPersistedOllamaHostIfUnused = vi.fn(() => true); + const peer: ReleaseEntry = { + name: "peer", + provider: "compatible-endpoint", + model: "llama3:latest", + endpointUrl: "http://127.0.0.1:11434/v1", + }; + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }, peer], + unloadOllamaModels, + loadPersistedOllamaHost: () => "127.0.0.1", + clearPersistedOllamaHostIfUnused, + }); + + await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual({ + ok: true, + }); + + expect(unloadOllamaModels).not.toHaveBeenCalled(); + expect(clearPersistedOllamaHostIfUnused).not.toHaveBeenCalled(); + }); + it("reads the prior route and releases the model inside the sandbox mutation lock (#9110)", async () => { const events: string[] = []; const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(() => { From da9fbfc456554e954990e72062775419a989b5bb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 1 Sep 2026 19:30:53 -0700 Subject: [PATCH 47/47] test: verify bounded Windows readiness delays --- src/lib/inference/ollama/windows.test.ts | 25 ++++++++++++++---------- src/lib/inference/ollama/windows.ts | 24 ++++++++++++++++------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 135fb365592..c068ee211a0 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -5,7 +5,6 @@ import { createRequire } from "node:module"; import { describe, expect, it, vi } from "vitest"; const require = createRequire(import.meta.url); -const childProcess = require("node:child_process"); const WINDOWS_DIST_PATH = require.resolve("./windows"); const RUNNER_PATH = require.resolve("../../runner"); const LOCAL_INFERENCE_PATH = require.resolve("../local"); @@ -24,9 +23,6 @@ function loadWindowsOllamaWithMocks( const originalRunCapture = runner.runCapture; // Stub the blocking wait so this test does not spend time on retry delays. const atomicsWaitStub = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); - const originalSpawnSync = childProcess.spawnSync; - const spawnSyncSpy = vi.fn((_command: string, _args?: readonly string[]) => ({ status: 0 })); - childProcess.spawnSync = spawnSyncSpy; delete require.cache[WINDOWS_DIST_PATH]; runner.run = run; @@ -34,12 +30,10 @@ function loadWindowsOllamaWithMocks( return { windows: require(WINDOWS_DIST_PATH), - spawnSyncSpy, restore() { delete require.cache[WINDOWS_DIST_PATH]; runner.run = originalRun; runner.runCapture = originalRunCapture; - childProcess.spawnSync = originalSpawnSync; atomicsWaitStub.mockRestore(); }, }; @@ -59,7 +53,16 @@ describe("Windows Ollama helper", () => { const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { - expect(windows.awaitWindowsOllamaReady()).toBe(false); + expect( + windows.awaitWindowsOllamaReady({ + delay: vi.fn(), + prepareDockerEnvironment: () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + }), + ).toBe(false); expect(runCapture).toHaveBeenCalledTimes(15); expect(localInference.getResolvedOllamaHost()).toBe("127.0.0.1"); } finally { @@ -102,10 +105,11 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const delay = vi.fn(); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath, delay })).toBe(true); } finally { restore(); logSpy.mockRestore(); @@ -136,7 +140,8 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); - expect(spawnSyncSpy.mock.calls.some(([command]) => command === "sleep")).toBe(false); + expect(delay).toHaveBeenCalled(); + expect(delay.mock.calls.every(([seconds]) => seconds > 0 && seconds <= 2)).toBe(true); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index dda0d5d587a..cda316ad6cd 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -120,15 +120,18 @@ function killWindowsOllamaProcesses(): void { ); } -function awaitWindowsOllamaReady(opts: { prepareDockerEnvironment?: () => unknown } = {}): boolean { +function awaitWindowsOllamaReady( + opts: { prepareDockerEnvironment?: () => unknown; delay?: (seconds: number) => void } = {}, +): boolean { console.log(" Waiting for Ollama to respond on host.docker.internal..."); + const delay = opts.delay ?? sleep; const capture = createOllamaApiCapture( runCapture, OLLAMA_HOST_DOCKER_INTERNAL, opts.prepareDockerEnvironment, ); for (let attempt = 0; attempt < 15; attempt++) { - sleep(2); + delay(2); const probe = capture( [ "curl", @@ -153,9 +156,10 @@ function awaitWindowsOllamaReady(opts: { prepareDockerEnvironment?: () => unknow // watcher's auto-restart survive; fall back through the verified installed // path and finally refreshed PATH because stale watcher paths are possible. function launchAndAwaitWindowsOllama( - opts: { watcherPath?: string; installedPath?: string } = {}, + opts: { watcherPath?: string; installedPath?: string; delay?: (seconds: number) => void } = {}, ): boolean { console.log(" Starting Ollama on Windows host via WSL interop..."); + const delay = opts.delay ?? sleep; const watcherPath = typeof opts.watcherPath === "string" ? opts.watcherPath.trim() : ""; const installedPath = typeof opts.installedPath === "string" ? opts.installedPath.trim() : ""; const launchAttempts: Array<{ label: string; script: string }> = []; @@ -188,7 +192,7 @@ function launchAndAwaitWindowsOllama( ignoreError: true, suppressOutput: true, }); - if (result.status === 0 && awaitWindowsOllamaReady()) { + if (result.status === 0 && awaitWindowsOllamaReady({ delay })) { return true; } @@ -201,7 +205,7 @@ function launchAndAwaitWindowsOllama( console.error(` PowerShell launch via ${attempt.label} failed: ${detail}`); if (i < launchAttempts.length - 1) { killWindowsOllamaProcesses(); - sleep(1); + delay(1); } } return false; @@ -211,18 +215,24 @@ function launchAndAwaitWindowsOllama( // installed Ollama. Fresh install fallback passes installedPath to avoid // relying on a newly-mutated Windows PATH from this process. function setupWindowsOllamaWith0000Binding( - opts: { announceStop?: boolean; installedPath?: string } = {}, + opts: { + announceStop?: boolean; + installedPath?: string; + delay?: (seconds: number) => void; + } = {}, ): boolean { + const delay = opts.delay ?? sleep; const watcherPath = captureWindowsOllamaWatcherPath(); persistOllamaHostEnvVar(); if (opts.announceStop) { console.log(" Stopping existing Ollama on Windows host..."); } killWindowsOllamaProcesses(); - sleep(1); + delay(1); return launchAndAwaitWindowsOllama({ watcherPath: watcherPath || undefined, installedPath: opts.installedPath, + delay, }); }