From b21459e1e91987854b4750d3c0c9c1b5ae3be3aa Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 19:39:40 -0700 Subject: [PATCH 01/48] fix: surface Ollama recovery failures Signed-off-by: Prekshi Vyas --- .../sandbox/agent/ollama-restart-recovery.ts | 17 +++++----- .../agent/passthrough-ollama-recovery.test.ts | 32 +++++++++++++++---- .../agent/passthrough-ollama-recovery.ts | 15 +++++++-- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index fd6c78ddd6e..07cd2dc97bb 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -33,7 +33,7 @@ import { type OllamaRuntimeModelStatus, probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; -import { runCaptureEx } from "../../../runner"; +import { redact, redactFull, runCaptureEx } from "../../../runner"; export interface OllamaRestartRecoveryRoute { provider?: string | null; @@ -193,10 +193,9 @@ function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid- } } -function boundedWarmFailureDetail(value: unknown, fallback: string): string { - const detail = String(value ?? "") - .replace(/\s+/g, " ") - .trim(); +export function boundedOllamaRestartRecoveryDetail(value: unknown, fallback: string): string { + const raw = value instanceof Error ? value.message : String(value ?? ""); + const detail = redactFull(redact(raw)).replace(/\s+/g, " ").trim(); return (detail || fallback).slice(0, 300); } @@ -261,7 +260,7 @@ export function maybeWarmOllamaAfterDaemonRestart( timedOut: true, reason: "timeout", endpoint: rawEndpoint, - detail: boundedWarmFailureDetail( + detail: boundedOllamaRestartRecoveryDetail( result.stderr, `warm-up exceeded ${OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS} seconds`, ), @@ -274,7 +273,7 @@ export function maybeWarmOllamaAfterDaemonRestart( timedOut: false, reason: "command-failed", endpoint: rawEndpoint, - detail: boundedWarmFailureDetail( + detail: boundedOllamaRestartRecoveryDetail( result.stderr || result.stdout, `warm-up exited ${String(result.exitCode)}`, ), @@ -305,7 +304,7 @@ export function maybeWarmOllamaAfterDaemonRestart( timedOut: false, reason: response, endpoint: rawEndpoint, - detail: boundedWarmFailureDetail(result.stdout, `Ollama returned ${response}`), + detail: boundedOllamaRestartRecoveryDetail(result.stdout, `Ollama returned ${response}`), }; } return { kind: "warmed", ok: true, timedOut: false }; @@ -316,7 +315,7 @@ export function maybeWarmOllamaAfterDaemonRestart( timedOut: false, reason: "spawn-failed", endpoint: rawEndpoint, - detail: boundedWarmFailureDetail( + detail: boundedOllamaRestartRecoveryDetail( error instanceof Error ? error.message : error, "warm-up process could not start", ), diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 8888031343b..707d4471336 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -136,17 +136,35 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("Ollama was unreachable during the restart check"); }); - it("continues OpenClaw dispatch when Ollama recovery throws", () => { + it("reports bounded recovery detail and guidance while dispatch continues", () => { const { writes, proc } = makeProcMock(); + const exposedToken = "sk-proj-NOT-A-REAL-SECRET-1234567890"; expect(() => - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => { - throw new Error("unexpected"); - }), + runOllamaRestartRecovery( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }, + proc, + () => { + throw new Error( + `synthetic Docker transport failure OPENAI_API_KEY=${exposedToken} ${"x".repeat(400)} END-OF-DETAIL`, + ); + }, + ), ).not.toThrow(); - expect(writes.join("")).toContain( - "Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch", - ); + const stderr = writes.join(""); + expect(stderr).toContain("Ollama restart recovery for 'qwen3.6:35b'"); + expect(stderr).toContain("at the recorded endpoint http://host.openshell.internal:11434/v1"); + expect(stderr).toContain("synthetic Docker transport failure"); + expect(stderr).toContain("OPENAI_API_KEY="); + expect(stderr).toContain("OpenClaw dispatch will continue"); + expect(stderr).toContain("Restore Ollama access to that endpoint"); + expect(stderr).toContain("then rerun this command"); + expect(stderr).not.toContain(exposedToken); + expect(stderr).not.toContain("END-OF-DETAIL"); }); }); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 4687c5ca201..2299bf2c4fd 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { + boundedOllamaRestartRecoveryDetail, maybeWarmOllamaAfterDaemonRestart, OLLAMA_LOCAL_PROVIDER, type OllamaRestartRecoveryFailureReason, @@ -19,6 +20,11 @@ export interface OllamaRestartRecoveryProcess { stderr: { write(s: string): unknown }; } +function recordedEndpointLabel(route: OllamaRestartRecoveryRoute): string { + const endpoint = boundedOllamaRestartRecoveryDetail(route.endpointUrl, ""); + return endpoint ? `at the recorded endpoint ${endpoint}` : "at the saved local Ollama endpoint"; +} + function describeWarmFailure(reason: OllamaRestartRecoveryFailureReason): string { switch (reason) { case "timeout": @@ -100,9 +106,14 @@ export function runOllamaRestartRecovery( proc.stderr.write(" Checking whether the Ollama model is loaded...\n"); try { reportRecovery(route, recoverOllama(route), proc); - } catch { + } catch (error) { + const model = String(route.model ?? "").trim() || "the registered model"; + const endpoint = recordedEndpointLabel(route); + const detail = boundedOllamaRestartRecoveryDetail(error, "unknown recovery error"); proc.stderr.write( - " Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch.\n", + ` Ollama restart recovery for '${model}' ${endpoint} failed unexpectedly: ${detail}. ` + + `OpenClaw dispatch will continue. Restore Ollama access to that endpoint, confirm it ` + + `serves '${model}', then rerun this command.\n`, ); } } From 433c1b2677f09071ec01a6272f0fb7c8b2bcac08 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 19:58:39 -0700 Subject: [PATCH 02/48] test: keep Windows readiness coverage behavioral Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 50 +++++++++++------------- src/lib/inference/ollama/windows.ts | 24 ++++-------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 554a3750e0d..b78dd851f9d 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -40,27 +40,13 @@ function loadWindowsOllamaWithMocks( } describe("Windows Ollama helper", () => { - it("rejects a nonempty invalid Docker readiness response (#10100)", () => { + it("continues probing after a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); const localInference = require(LOCAL_INFERENCE_PATH); - const runCapture = vi.fn((command: string | string[]) => { - expect(command).toEqual( - expect.arrayContaining([ - "docker", - "run", - "--rm", - localInference.CONTAINER_REACHABILITY_IMAGE, - WINDOWS_OLLAMA_TAGS_URL, - ]), - ); - expect(command.slice(0, 4)).toEqual([ - "docker", - "run", - "--rm", - localInference.CONTAINER_REACHABILITY_IMAGE, - ]); - expect(command.at(-1)).toBe(WINDOWS_OLLAMA_TAGS_URL); - return "proxy response"; + let probeAttempts = 0; + const runCapture = vi.fn((_command: string | string[]) => { + probeAttempts += 1; + return probeAttempts === 1 ? "proxy response" : JSON.stringify({ models: [] }); }); localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); @@ -69,16 +55,29 @@ describe("Windows Ollama helper", () => { try { expect( windows.awaitWindowsOllamaReady({ - delay: vi.fn(), prepareDockerEnvironment: () => ({ env: {}, isolatedCredentialConfig: false, cleanup: () => ({ ok: true }), }), }), - ).toBe(false); - expect(runCapture.mock.calls.length).toBeGreaterThan(0); - expect(localInference.getResolvedOllamaHost()).toBe("127.0.0.1"); + ).toBe(true); + expect(runCapture.mock.calls.length).toBeGreaterThan(1); + expect( + runCapture.mock.calls.every( + ([command]) => + Array.isArray(command) && + command + .slice(0, 4) + .every( + (argument, index) => + argument === + ["docker", "run", "--rm", localInference.CONTAINER_REACHABILITY_IMAGE][index], + ) && + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL, + ), + ).toBe(true); + expect(localInference.getResolvedOllamaHost()).toBe("host.docker.internal"); } finally { localInference.resetOllamaHostCache(); restore(); @@ -119,11 +118,10 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const delay = vi.fn(); const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath, delay })).toBe(true); + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); } finally { restore(); logSpy.mockRestore(); @@ -154,8 +152,6 @@ describe("Windows Ollama helper", () => { ], expect.objectContaining({ ignoreError: true }), ); - expect(delay).toHaveBeenCalled(); - expect(delay.mock.calls.every(([seconds]) => seconds > 0 && seconds <= 2)).toBe(true); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index cda316ad6cd..dda0d5d587a 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -120,18 +120,15 @@ function killWindowsOllamaProcesses(): void { ); } -function awaitWindowsOllamaReady( - opts: { prepareDockerEnvironment?: () => unknown; delay?: (seconds: number) => void } = {}, -): boolean { +function awaitWindowsOllamaReady(opts: { prepareDockerEnvironment?: () => unknown } = {}): boolean { console.log(" Waiting for Ollama to respond on host.docker.internal..."); - const delay = opts.delay ?? sleep; const capture = createOllamaApiCapture( runCapture, OLLAMA_HOST_DOCKER_INTERNAL, opts.prepareDockerEnvironment, ); for (let attempt = 0; attempt < 15; attempt++) { - delay(2); + sleep(2); const probe = capture( [ "curl", @@ -156,10 +153,9 @@ function awaitWindowsOllamaReady( // watcher's auto-restart survive; fall back through the verified installed // path and finally refreshed PATH because stale watcher paths are possible. function launchAndAwaitWindowsOllama( - opts: { watcherPath?: string; installedPath?: string; delay?: (seconds: number) => void } = {}, + opts: { watcherPath?: string; installedPath?: string } = {}, ): boolean { console.log(" Starting Ollama on Windows host via WSL interop..."); - const delay = opts.delay ?? sleep; const watcherPath = typeof opts.watcherPath === "string" ? opts.watcherPath.trim() : ""; const installedPath = typeof opts.installedPath === "string" ? opts.installedPath.trim() : ""; const launchAttempts: Array<{ label: string; script: string }> = []; @@ -192,7 +188,7 @@ function launchAndAwaitWindowsOllama( ignoreError: true, suppressOutput: true, }); - if (result.status === 0 && awaitWindowsOllamaReady({ delay })) { + if (result.status === 0 && awaitWindowsOllamaReady()) { return true; } @@ -205,7 +201,7 @@ function launchAndAwaitWindowsOllama( console.error(` PowerShell launch via ${attempt.label} failed: ${detail}`); if (i < launchAttempts.length - 1) { killWindowsOllamaProcesses(); - delay(1); + sleep(1); } } return false; @@ -215,24 +211,18 @@ function launchAndAwaitWindowsOllama( // installed Ollama. Fresh install fallback passes installedPath to avoid // relying on a newly-mutated Windows PATH from this process. function setupWindowsOllamaWith0000Binding( - opts: { - announceStop?: boolean; - installedPath?: string; - delay?: (seconds: number) => void; - } = {}, + opts: { announceStop?: boolean; installedPath?: string } = {}, ): boolean { - const delay = opts.delay ?? sleep; const watcherPath = captureWindowsOllamaWatcherPath(); persistOllamaHostEnvVar(); if (opts.announceStop) { console.log(" Stopping existing Ollama on Windows host..."); } killWindowsOllamaProcesses(); - delay(1); + sleep(1); return launchAndAwaitWindowsOllama({ watcherPath: watcherPath || undefined, installedPath: opts.installedPath, - delay, }); } From cff68c14a6af6a456a8936df17fb7daf5880575a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 21:18:48 -0700 Subject: [PATCH 03/48] fix: harden Ollama recovery diagnostics Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 6 ---- .../sandbox/agent/ollama-restart-recovery.ts | 30 ++++++++----------- .../agent/passthrough-ollama-recovery.test.ts | 18 ++++++----- .../agent/passthrough-ollama-recovery.ts | 24 ++++++++++----- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 4e8bdf699a6..f58d0526dea 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; -import { prepareOllamaApiExecution } from "../../../inference/local"; import { maybeWarmOllamaAfterDaemonRestart, type OllamaRestartRecoveryDeps, @@ -76,11 +75,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureImpl, runCaptureExImpl, prepareDockerEnvironment, - prepareOllamaApiExecution: (command, host, options) => - prepareOllamaApiExecution(command, host, { - ...options, - prepareDockerEnvironment, - }), }, ), ).toEqual({ kind: "warmed", ok: true, timedOut: false }); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 07cd2dc97bb..841bf3657da 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -19,12 +19,12 @@ import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, createOllamaApiCapture, + createOllamaApiCaptureEx, getOllamaApiCommand, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, - prepareOllamaApiExecution, probeOllamaEndpointInventory, type RunCaptureFn, type RunCaptureExFn, @@ -33,7 +33,7 @@ import { type OllamaRuntimeModelStatus, probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; -import { redact, redactFull, runCaptureEx } from "../../../runner"; +import { redact, redactFull } from "../../../runner"; export interface OllamaRestartRecoveryRoute { provider?: string | null; @@ -52,7 +52,6 @@ export interface OllamaRestartRecoveryDeps { getOllamaHost?: () => string; runCaptureImpl?: RunCaptureFn; prepareDockerEnvironment?: Parameters[2]; - prepareOllamaApiExecution?: typeof prepareOllamaApiExecution; } export type OllamaRestartRecoveryFailureReason = @@ -195,7 +194,10 @@ function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid- export function boundedOllamaRestartRecoveryDetail(value: unknown, fallback: string): string { const raw = value instanceof Error ? value.message : String(value ?? ""); - const detail = redactFull(redact(raw)).replace(/\s+/g, " ").trim(); + const detail = redactFull(redact(raw)) + .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ") + .replace(/\s+/gu, " ") + .trim(); return (detail || fallback).slice(0, 300); } @@ -225,6 +227,11 @@ export function maybeWarmOllamaAfterDaemonRestart( rawHost, deps.prepareDockerEnvironment, ); + const rawCaptureEx = createOllamaApiCaptureEx( + deps.runCaptureExImpl, + rawHost, + deps.prepareDockerEnvironment, + ); let status: OllamaRuntimeModelStatus; try { status = probe(model, () => rawHost, rawCapture); @@ -238,21 +245,8 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "skipped", reason: "already-loaded" }; } - const captureEx = deps.runCaptureExImpl ?? runCaptureEx; try { - const execution = (deps.prepareOllamaApiExecution ?? prepareOllamaApiExecution)( - buildWarmCommand(model, rawHost), - rawHost, - { operation: `Ollama restart warm-up for '${model}'` }, - ); - let result; - try { - result = captureEx(execution.command, { - ...(execution.env === undefined ? {} : { env: execution.env }), - }); - } finally { - execution.cleanup(); - } + const result = rawCaptureEx(buildWarmCommand(model, rawHost)); if (result.timedOut) { return { kind: "warmed", diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 707d4471336..e1fe415fc50 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -128,7 +128,7 @@ describe("runOllamaRestartRecovery", () => { const stderr = writes.join(""); expect(stderr).toContain( - "Ollama at http://host.docker.internal:11434 reports 'gemma4:26b' as unavailable", + "Ollama at http://host.docker.internal:11434/ reports 'gemma4:26b' as unavailable", ); expect(stderr).toContain("reported models: llama3.2:1b"); expect(stderr).toContain("continuing to OpenClaw dispatch"); @@ -136,7 +136,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("Ollama was unreachable during the restart check"); }); - it("reports bounded recovery detail and guidance while dispatch continues", () => { + it("continues dispatch with redacted, bounded detail when recovery throws", () => { const { writes, proc } = makeProcMock(); const exposedToken = "sk-proj-NOT-A-REAL-SECRET-1234567890"; @@ -144,27 +144,31 @@ describe("runOllamaRestartRecovery", () => { runOllamaRestartRecovery( { provider: "ollama-local", - model: "qwen3.6:35b", - endpointUrl: "http://host.openshell.internal:11434/v1", + model: "qwen3.6:35b\u001b[2J\u0007", + endpointUrl: "http://host.openshell.internal:11434/v1\u001b]52;c;clipboard\u0007", }, proc, () => { throw new Error( - `synthetic Docker transport failure OPENAI_API_KEY=${exposedToken} ${"x".repeat(400)} END-OF-DETAIL`, + `synthetic\u001b[2J Docker transport failure\u0007 OPENAI_API_KEY=${exposedToken} ${"x".repeat(400)} END-OF-DETAIL`, ); }, ), ).not.toThrow(); const stderr = writes.join(""); - expect(stderr).toContain("Ollama restart recovery for 'qwen3.6:35b'"); + expect(stderr).toContain("Ollama restart recovery for 'qwen3.6:35b"); expect(stderr).toContain("at the recorded endpoint http://host.openshell.internal:11434/v1"); - expect(stderr).toContain("synthetic Docker transport failure"); + expect(stderr).toContain("synthetic"); + expect(stderr).toContain("Docker transport failure"); expect(stderr).toContain("OPENAI_API_KEY="); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("Restore Ollama access to that endpoint"); expect(stderr).toContain("then rerun this command"); expect(stderr).not.toContain(exposedToken); expect(stderr).not.toContain("END-OF-DETAIL"); + expect(stderr).not.toContain("\u001b"); + expect(stderr).not.toContain("\u0007"); + expect(stderr.replace(/\n/gu, "")).not.toMatch(/[\u0000-\u001f\u007f-\u009f]/u); }); }); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 2299bf2c4fd..9cc1ad72695 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -45,25 +45,35 @@ function reportRecovery( result: OllamaRestartRecoveryResult, proc: OllamaRestartRecoveryProcess, ): void { - const model = String(route.model ?? "").trim() || "the registered model"; + const model = boundedOllamaRestartRecoveryDetail(route.model, "the registered model"); if (result.kind === "warmed") { if (result.ok) { proc.stderr.write(` Ollama model '${model}' is loaded and ready.\n`); return; } + const endpoint = boundedOllamaRestartRecoveryDetail( + result.endpoint, + "the saved local Ollama endpoint", + ); + const detail = boundedOllamaRestartRecoveryDetail(result.detail, "unknown warm-up error"); proc.stderr.write( - ` Ollama warm-up for '${model}' at ${result.endpoint} ${describeWarmFailure(result.reason)} ` + - `(${result.detail}). OpenClaw dispatch will continue. To retry the warm-up, restore ` + - `Ollama access to ${result.endpoint} and confirm that it serves '${model}', then rerun ` + + ` Ollama warm-up for '${model}' at ${endpoint} ${describeWarmFailure(result.reason)} ` + + `(${detail}). OpenClaw dispatch will continue. To retry the warm-up, restore ` + + `Ollama access to ${endpoint} and confirm that it serves '${model}', then rerun ` + `this command.\n`, ); return; } if (result.reason === "model-absent") { + const endpoint = boundedOllamaRestartRecoveryDetail( + result.endpoint, + "the saved local Ollama endpoint", + ); + const inventoryLabel = boundedOllamaRestartRecoveryDetail(result.inventoryLabel, "none"); proc.stderr.write( - ` Ollama at ${result.endpoint} reports '${model}' as unavailable ` + - `(reported models: ${result.inventoryLabel}); continuing to OpenClaw dispatch.\n`, + ` Ollama at ${endpoint} reports '${model}' as unavailable ` + + `(reported models: ${inventoryLabel}); continuing to OpenClaw dispatch.\n`, ); proc.stderr.write( ` Either the daemon answering that endpoint changed, or the model was removed from ` + @@ -107,7 +117,7 @@ export function runOllamaRestartRecovery( try { reportRecovery(route, recoverOllama(route), proc); } catch (error) { - const model = String(route.model ?? "").trim() || "the registered model"; + const model = boundedOllamaRestartRecoveryDetail(route.model, "the registered model"); const endpoint = recordedEndpointLabel(route); const detail = boundedOllamaRestartRecoveryDetail(error, "unknown recovery error"); proc.stderr.write( From d119c196b5a74bae566c80da1c78278bba6f1ab1 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 21:56:11 -0700 Subject: [PATCH 04/48] fix: preserve Ollama recovery display semantics Signed-off-by: Prekshi Vyas --- .../agent/passthrough-ollama-recovery.test.ts | 2 +- .../agent/passthrough-ollama-recovery.ts | 18 +-- src/lib/inference/ollama/windows.test.ts | 138 ++++++++---------- 3 files changed, 70 insertions(+), 88 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index e1fe415fc50..74270d0a9e6 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -128,7 +128,7 @@ describe("runOllamaRestartRecovery", () => { const stderr = writes.join(""); expect(stderr).toContain( - "Ollama at http://host.docker.internal:11434/ reports 'gemma4:26b' as unavailable", + "Ollama at http://host.docker.internal:11434 reports 'gemma4:26b' as unavailable", ); expect(stderr).toContain("reported models: llama3.2:1b"); expect(stderr).toContain("continuing to OpenClaw dispatch"); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 9cc1ad72695..a179a9c6fee 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -20,8 +20,14 @@ export interface OllamaRestartRecoveryProcess { stderr: { write(s: string): unknown }; } +function boundedRecoveryEndpoint(value: unknown, fallback: string): string { + const original = String(value ?? "").trim(); + const endpoint = boundedOllamaRestartRecoveryDetail(value, fallback); + return endpoint.endsWith("/") && !original.endsWith("/") ? endpoint.slice(0, -1) : endpoint; +} + function recordedEndpointLabel(route: OllamaRestartRecoveryRoute): string { - const endpoint = boundedOllamaRestartRecoveryDetail(route.endpointUrl, ""); + const endpoint = boundedRecoveryEndpoint(route.endpointUrl, ""); return endpoint ? `at the recorded endpoint ${endpoint}` : "at the saved local Ollama endpoint"; } @@ -51,10 +57,7 @@ function reportRecovery( proc.stderr.write(` Ollama model '${model}' is loaded and ready.\n`); return; } - const endpoint = boundedOllamaRestartRecoveryDetail( - result.endpoint, - "the saved local Ollama endpoint", - ); + const endpoint = boundedRecoveryEndpoint(result.endpoint, "the saved local Ollama endpoint"); const detail = boundedOllamaRestartRecoveryDetail(result.detail, "unknown warm-up error"); proc.stderr.write( ` Ollama warm-up for '${model}' at ${endpoint} ${describeWarmFailure(result.reason)} ` + @@ -66,10 +69,7 @@ function reportRecovery( } if (result.reason === "model-absent") { - const endpoint = boundedOllamaRestartRecoveryDetail( - result.endpoint, - "the saved local Ollama endpoint", - ); + const endpoint = boundedRecoveryEndpoint(result.endpoint, "the saved local Ollama endpoint"); const inventoryLabel = boundedOllamaRestartRecoveryDetail(result.inventoryLabel, "none"); proc.stderr.write( ` Ollama at ${endpoint} reports '${model}' as unavailable ` + diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index b78dd851f9d..1d1f810daa2 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -14,6 +14,12 @@ function commandText(command: string | string[]): string { return Array.isArray(command) ? command.join(" ") : String(command); } +function isDockerTagsRequest(command: string | string[]): boolean { + return ( + Array.isArray(command) && command[0] === "docker" && command.includes(WINDOWS_OLLAMA_TAGS_URL) + ); +} + function loadWindowsOllamaWithMocks( run: ReturnType, runCapture: ReturnType, @@ -43,11 +49,23 @@ describe("Windows Ollama helper", () => { it("continues probing after a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); const localInference = require(LOCAL_INFERENCE_PATH); - let probeAttempts = 0; - const runCapture = vi.fn((_command: string | string[]) => { - probeAttempts += 1; - return probeAttempts === 1 ? "proxy response" : JSON.stringify({ models: [] }); - }); + let invalidResponseServed = false; + let validResponseServed = false; + const serveInvalidResponse = () => { + invalidResponseServed = true; + return "proxy response"; + }; + const serveValidResponse = () => { + validResponseServed = true; + return JSON.stringify({ models: [] }); + }; + const runCapture = vi.fn((command: string | string[]) => + isDockerTagsRequest(command) + ? invalidResponseServed + ? serveValidResponse() + : serveInvalidResponse() + : "", + ); localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); @@ -62,21 +80,8 @@ describe("Windows Ollama helper", () => { }), }), ).toBe(true); - expect(runCapture.mock.calls.length).toBeGreaterThan(1); - expect( - runCapture.mock.calls.every( - ([command]) => - Array.isArray(command) && - command - .slice(0, 4) - .every( - (argument, index) => - argument === - ["docker", "run", "--rm", localInference.CONTAINER_REACHABILITY_IMAGE][index], - ) && - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL, - ), - ).toBe(true); + expect(invalidResponseServed).toBe(true); + expect(validResponseServed).toBe(true); expect(localInference.getResolvedOllamaHost()).toBe("host.docker.internal"); } finally { localInference.resetOllamaHostCache(); @@ -88,34 +93,36 @@ describe("Windows Ollama helper", () => { it("falls back from a stale watcher path and checks readiness from Docker Desktop (#8127)", () => { const watcherPath = "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama app.exe"; const installedPath = "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe"; - const launchScripts: string[] = []; - const stopCommands: string[] = []; + let watcherLaunchAttempted = false; + let installedLaunchAttempted = false; + let dockerReadinessObserved = false; const run = vi.fn((command: string[]) => { - const script = command[2] || ""; - launchScripts.push(script); - if (script.includes(watcherPath)) { - return { status: 1, stderr: "stale watcher path" }; - } - return { status: 0, stderr: "" }; + const launch = commandText(command); + const isWatcherLaunch = launch.includes(watcherPath); + const isInstalledLaunch = launch.includes(installedPath); + watcherLaunchAttempted ||= isWatcherLaunch; + installedLaunchAttempted ||= isInstalledLaunch; + return isWatcherLaunch + ? { status: 1, stderr: "stale watcher path" } + : isInstalledLaunch + ? { status: 0, stderr: "" } + : { status: 1, stderr: "unexpected launch target" }; }); const runCapture = vi.fn((command: string | string[]) => { const cmd = commandText(command); - if (cmd.includes("Get-Process 'ollama app'") && cmd.includes("ExpandProperty Path")) { - return watcherPath; - } - if (cmd.includes("Stop-Process")) { - stopCommands.push(cmd); - return ""; - } - if (Array.isArray(command) && command.at(-1) === WINDOWS_OLLAMA_TAGS_URL) { - return command[0] === "docker" && - launchScripts.some((script) => script.includes(installedPath)) + const capturesWatcherPath = + cmd.includes("Get-Process 'ollama app'") && cmd.includes("ExpandProperty Path"); + const probesDockerReadiness = isDockerTagsRequest(command); + dockerReadinessObserved ||= probesDockerReadiness; + return capturesWatcherPath + ? watcherPath + : probesDockerReadiness && installedLaunchAttempted ? JSON.stringify({ models: [] }) : ""; - } - return ""; }); + const localInference = require(LOCAL_INFERENCE_PATH); + localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); @@ -123,47 +130,28 @@ describe("Windows Ollama helper", () => { try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); } finally { + localInference.resetOllamaHostCache(); restore(); logSpy.mockRestore(); errorSpy.mockRestore(); } - expect(run).toHaveBeenCalledTimes(2); - expect(launchScripts[0]).toContain(watcherPath); - expect(launchScripts[1]).toContain(installedPath); - expect(launchScripts[1]).toContain("-ArgumentList 'serve'"); - expect( - launchScripts.some((script) => script.includes("Start-Process -FilePath ollama.exe")), - ).toBe(false); - expect(stopCommands[0]).toContain("Get-Process 'ollama app'"); - expect(stopCommands[1]).toContain("Get-Process ollama"); - expect(runCapture).toHaveBeenCalledWith( - [ - "docker", - "run", - "--rm", - "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", - "-sf", - "--connect-timeout", - "2", - "--max-time", - "5", - "http://host.docker.internal:11434/api/tags", - ], - expect.objectContaining({ ignoreError: true }), - ); + expect(watcherLaunchAttempted).toBe(true); + expect(installedLaunchAttempted).toBe(true); + expect(dockerReadinessObserved).toBe(true); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { const run = vi.fn(); const cleanup = vi.fn(() => ({ ok: true as const })); - const runCapture = vi.fn((command: string | string[], options?: { env?: NodeJS.ProcessEnv }) => - Array.isArray(command) && - command[0] === "docker" && - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL && - options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" - ? JSON.stringify({ models: [] }) - : "", + let credentialFreeDockerRequestObserved = false; + const runCapture = vi.fn( + (command: string | string[], options?: { env?: NodeJS.ProcessEnv }) => { + credentialFreeDockerRequestObserved = + isDockerTagsRequest(command) && + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker"; + return credentialFreeDockerRequestObserved ? JSON.stringify({ models: [] }) : ""; + }, ); const localInference = require(LOCAL_INFERENCE_PATH); localInference.resetOllamaHostCache(); @@ -181,13 +169,7 @@ describe("Windows Ollama helper", () => { }), ).toBe(true); expect(localInference.getResolvedOllamaHost()).toBe("host.docker.internal"); - expect(runCapture).toHaveBeenCalledWith( - expect.arrayContaining(["docker", "run", "--rm", WINDOWS_OLLAMA_TAGS_URL]), - expect.objectContaining({ - ignoreError: true, - env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, - }), - ); + expect(credentialFreeDockerRequestObserved).toBe(true); expect(cleanup).toHaveBeenCalledOnce(); } finally { localInference.resetOllamaHostCache(); From d2fe9519dcede03b4d8c773d5734c4f5512d3bcc Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 22:27:32 -0700 Subject: [PATCH 05/48] fix: make Ollama recovery actionable Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 6 +- .../sandbox/agent/ollama-restart-recovery.ts | 7 ++- .../agent/passthrough-ollama-recovery.test.ts | 59 ++++++++++++++++++- .../agent/passthrough-ollama-recovery.ts | 8 ++- 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index f58d0526dea..fd271ca2998 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -193,7 +193,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { { provider: "ollama-local", model: "qwen3.6:35b" }, { runCaptureImpl: () => "", runCaptureExImpl }, ), - ).toEqual({ kind: "skipped", reason: "unreachable" }); + ).toEqual({ + kind: "skipped", + reason: "unreachable", + endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, + }); expect(runCaptureExImpl).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 841bf3657da..8cb6cac2e7e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -62,7 +62,8 @@ export type OllamaRestartRecoveryFailureReason = | "spawn-failed"; export type OllamaRestartRecoveryResult = - | { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" | "unreachable" } + | { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" } + | { kind: "skipped"; reason: "unreachable"; endpoint: string } | { kind: "skipped"; reason: "model-absent"; endpoint: string; inventoryLabel: string } | { kind: "warmed"; ok: true; timedOut: false } | { @@ -236,10 +237,10 @@ export function maybeWarmOllamaAfterDaemonRestart( try { status = probe(model, () => rawHost, rawCapture); } catch { - return { kind: "skipped", reason: "unreachable" }; + return { kind: "skipped", reason: "unreachable", endpoint: rawEndpoint }; } if (!status.probed) { - return { kind: "skipped", reason: "unreachable" }; + return { kind: "skipped", reason: "unreachable", endpoint: rawEndpoint }; } if (status.loaded) { return { kind: "skipped", reason: "already-loaded" }; diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 74270d0a9e6..0ec052c596a 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -94,7 +94,6 @@ describe("runOllamaRestartRecovery", () => { it.each([ ["already-loaded", "Ollama model 'qwen3.6:35b' is already loaded"], - ["unreachable", "Ollama was unreachable during the model check"], ["missing-model", "No Ollama model is recorded for this sandbox"], ["not-ollama", "Checking whether the Ollama model is loaded"], ] as const)("handles the %s skip reason", (reason, message) => { @@ -108,6 +107,23 @@ describe("runOllamaRestartRecovery", () => { expect(writes.join("")).toContain(message); }); + it("reports the endpoint, model, and recovery action when Ollama is unreachable", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "skipped", + reason: "unreachable", + endpoint: "http://host.docker.internal:11434", + })); + + const stderr = writes.join(""); + expect(stderr).toContain("http://host.docker.internal:11434"); + expect(stderr).toContain("qwen3.6:35b"); + expect(stderr).toContain("Restore Ollama access"); + expect(stderr).toContain("confirm that it serves"); + expect(stderr).toContain("then rerun this command"); + }); + it("names the endpoint and its reported models when the model is absent (#9455)", () => { const { writes, proc } = makeProcMock(); @@ -136,7 +152,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("Ollama was unreachable during the restart check"); }); - it("continues dispatch with redacted, bounded detail when recovery throws", () => { + it("redacts and bounds recovery exceptions", () => { const { writes, proc } = makeProcMock(); const exposedToken = "sk-proj-NOT-A-REAL-SECRET-1234567890"; @@ -253,6 +269,45 @@ describe("agent passthrough Ollama recovery ordering", () => { expect(events).toEqual(["recovery", "dispatch"]); }); + it("dispatches after reporting an Ollama recovery exception", async () => { + const events: string[] = []; + const diagnostics: string[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }; + const deps = makePassthroughDeps(route, events); + deps.process = { + ...deps.process!, + stderr: { + write: (value: string) => { + diagnostics.push(value); + value.includes("failed unexpectedly") && events.push("diagnostic"); + return true; + }, + }, + }; + const runRecovery = ( + registeredRoute: Parameters[0], + proc: Parameters[1], + ) => + runOllamaRestartRecovery(registeredRoute, proc, () => { + throw new Error("synthetic recovery failure"); + }); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping"] }, + { ...deps, runOllamaRestartRecovery: runRecovery }, + ), + ).rejects.toThrow("__exit:0"); + + expect(events).toEqual(["diagnostic", "dispatch"]); + expect(diagnostics.join("")).toContain("failed unexpectedly"); + }); + it("does not run Ollama recovery for a non-Ollama route", async () => { const events: string[] = []; const deps = makePassthroughDeps( diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index a179a9c6fee..29ed67cff6a 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -88,11 +88,15 @@ function reportRecovery( case "already-loaded": proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); break; - case "unreachable": + case "unreachable": { + const endpoint = boundedRecoveryEndpoint(result.endpoint, "the saved local Ollama endpoint"); proc.stderr.write( - " Ollama was unreachable during the model check; continuing to OpenClaw dispatch.\n", + ` Ollama at ${endpoint} was unreachable while checking '${model}'; continuing to ` + + `OpenClaw dispatch. Restore Ollama access to ${endpoint}, confirm that it serves ` + + `'${model}', then rerun this command.\n`, ); break; + } case "missing-model": proc.stderr.write( " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", From e8a9d98e53b4120c095c1acabeb31a754bfaeff1 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 22:53:51 -0700 Subject: [PATCH 06/48] test: validate Ollama readiness URL semantics Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 1d1f810daa2..82631c96b9b 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -8,16 +8,31 @@ const require = createRequire(import.meta.url); const WINDOWS_DIST_PATH = require.resolve("./windows"); const RUNNER_PATH = require.resolve("../../runner"); const LOCAL_INFERENCE_PATH = require.resolve("../local"); -const WINDOWS_OLLAMA_TAGS_URL = "http://host.docker.internal:11434/api/tags"; function commandText(command: string | string[]): string { return Array.isArray(command) ? command.join(" ") : String(command); } +function isWindowsOllamaTagsUrl(value: string): boolean { + try { + const url = new URL(value); + return ( + url.protocol === "http:" && + url.username === "" && + url.password === "" && + url.hostname === "host.docker.internal" && + url.port === "11434" && + url.pathname === "/api/tags" && + url.search === "" && + url.hash === "" + ); + } catch { + return false; + } +} + function isDockerTagsRequest(command: string | string[]): boolean { - return ( - Array.isArray(command) && command[0] === "docker" && command.includes(WINDOWS_OLLAMA_TAGS_URL) - ); + return Array.isArray(command) && command[0] === "docker" && command.some(isWindowsOllamaTagsUrl); } function loadWindowsOllamaWithMocks( From 31460b731ca6243909ffc7bd09ce1f0d87f32a9a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 1 Sep 2026 23:33:26 -0700 Subject: [PATCH 07/48] fix: constrain Ollama recovery safety Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 46 ++++++++++++++ .../sandbox/agent/ollama-restart-recovery.ts | 47 +++++++++++++-- .../agent/passthrough-ollama-recovery.test.ts | 60 ++++++++++++++----- .../agent/passthrough-ollama-recovery.ts | 14 ++++- src/lib/actions/sandbox/agent/passthrough.ts | 4 +- src/lib/inference/local.ts | 5 +- 6 files changed, 152 insertions(+), 24 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index fd271ca2998..111d7ad3e81 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -224,6 +224,52 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); + it("limits warm-up to the command timeout budget remaining after the probe", () => { + const runCaptureExImpl = vi.fn( + (_command: string[], _options?: { env?: NodeJS.ProcessEnv; timeout?: number }) => + successfulWarmResult(), + ); + const now = vi.fn().mockReturnValueOnce(1_000).mockReturnValueOnce(6_000); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl, + timeoutSeconds: 30, + now, + }, + ), + ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + + const warmCommand = runCaptureExImpl.mock.calls[0][0]; + expect(warmCommand[warmCommand.indexOf("--max-time") + 1]).toBe("25"); + expect(runCaptureExImpl.mock.calls[0][1]?.timeout).toBe(25_000); + }); + + it("skips warm-up when the probe consumes the command timeout budget", () => { + const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const now = vi.fn().mockReturnValueOnce(1_000).mockReturnValueOnce(31_000); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl, + timeoutSeconds: 30, + now, + }, + ), + ).toEqual({ + kind: "skipped", + reason: "deadline-exhausted", + endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, + }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { expect( maybeWarmOllamaAfterDaemonRestart( diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 8cb6cac2e7e..376360c77be 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -41,7 +41,11 @@ export interface OllamaRestartRecoveryRoute { endpointUrl?: string | null; } -export interface OllamaRestartRecoveryDeps { +export interface OllamaRestartRecoveryOptions { + timeoutSeconds?: number; +} + +export interface OllamaRestartRecoveryDeps extends OllamaRestartRecoveryOptions { probeRuntimeModelStatus?: ( model: string, getOllamaHost: () => string, @@ -52,6 +56,7 @@ export interface OllamaRestartRecoveryDeps { getOllamaHost?: () => string; runCaptureImpl?: RunCaptureFn; prepareDockerEnvironment?: Parameters[2]; + now?: () => number; } export type OllamaRestartRecoveryFailureReason = @@ -64,6 +69,7 @@ export type OllamaRestartRecoveryFailureReason = export type OllamaRestartRecoveryResult = | { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" } | { kind: "skipped"; reason: "unreachable"; endpoint: string } + | { kind: "skipped"; reason: "deadline-exhausted"; endpoint: string } | { kind: "skipped"; reason: "model-absent"; endpoint: string; inventoryLabel: string } | { kind: "warmed"; ok: true; timedOut: false } | { @@ -145,7 +151,7 @@ function resolveRawOllamaHost( return getAllowedFallbackHost(getOllamaHost); } -function buildWarmCommand(model: string, hostname: string): string[] { +function buildWarmCommand(model: string, hostname: string, maxTimeSeconds: number): string[] { const body = JSON.stringify({ model, prompt: "Hello, reply in less than 5 words", @@ -160,7 +166,7 @@ function buildWarmCommand(model: string, hostname: string): string[] { "--connect-timeout", "3", "--max-time", - String(OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS), + String(maxTimeSeconds), "-H", "Content-Type: application/json", "-d", @@ -171,6 +177,24 @@ function buildWarmCommand(model: string, hostname: string): string[] { ); } +function recoveryDeadlineMilliseconds( + timeoutSeconds: number | undefined, + now: () => number, +): number | null { + if (timeoutSeconds === undefined) return null; + const boundedSeconds = + Number.isFinite(timeoutSeconds) && timeoutSeconds > 0 + ? Math.min(timeoutSeconds, OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS) + : 0; + return now() + boundedSeconds * 1000; +} + +function remainingRecoveryMilliseconds(deadline: number | null, now: () => number): number { + return deadline === null + ? OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS * 1000 + : Math.max(0, Math.floor(deadline - now())); +} + function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" { try { const parsed = JSON.parse(stdout) as { @@ -196,7 +220,7 @@ function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid- export function boundedOllamaRestartRecoveryDetail(value: unknown, fallback: string): string { const raw = value instanceof Error ? value.message : String(value ?? ""); const detail = redactFull(redact(raw)) - .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ") + .replace(/[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/gu, " ") .replace(/\s+/gu, " ") .trim(); return (detail || fallback).slice(0, 300); @@ -219,6 +243,9 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "skipped", reason: "missing-model" }; } + const now = deps.now ?? Date.now; + const recoveryDeadline = recoveryDeadlineMilliseconds(deps.timeoutSeconds, now); + const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); const rawEndpoint = `http://${rawHost}:${OLLAMA_PORT}`; @@ -246,8 +273,16 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "skipped", reason: "already-loaded" }; } + const warmupTimeoutMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); + if (warmupTimeoutMilliseconds === 0) { + return { kind: "skipped", reason: "deadline-exhausted", endpoint: rawEndpoint }; + } + const warmupTimeoutSeconds = warmupTimeoutMilliseconds / 1000; + try { - const result = rawCaptureEx(buildWarmCommand(model, rawHost)); + const result = rawCaptureEx(buildWarmCommand(model, rawHost, warmupTimeoutSeconds), { + timeout: warmupTimeoutMilliseconds, + }); if (result.timedOut) { return { kind: "warmed", @@ -257,7 +292,7 @@ export function maybeWarmOllamaAfterDaemonRestart( endpoint: rawEndpoint, detail: boundedOllamaRestartRecoveryDetail( result.stderr, - `warm-up exceeded ${OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS} seconds`, + `warm-up exceeded ${String(warmupTimeoutSeconds)} seconds`, ), }; } diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 0ec052c596a..f43f8b45f73 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -29,16 +29,16 @@ describe("runOllamaRestartRecovery", () => { endpointUrl, }; - runOllamaRestartRecovery(route, proc, recoverOllama); + runOllamaRestartRecovery(route, proc, {}, recoverOllama); - expect(recoverOllama).toHaveBeenCalledWith(route); + expect(recoverOllama).toHaveBeenCalledWith(route, {}); expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is already loaded"); }); it("reports a successful warm-up", () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "warmed", ok: true, timedOut: false, @@ -50,7 +50,7 @@ describe("runOllamaRestartRecovery", () => { it("reports a timeout before continuing to OpenClaw", () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "warmed", ok: false, timedOut: true, @@ -76,7 +76,7 @@ describe("runOllamaRestartRecovery", () => { ] as const)("reports a %s warm-up failure", (reason, message) => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "warmed", ok: false, timedOut: false, @@ -99,7 +99,7 @@ describe("runOllamaRestartRecovery", () => { ] as const)("handles the %s skip reason", (reason, message) => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "skipped", reason, })); @@ -110,7 +110,7 @@ describe("runOllamaRestartRecovery", () => { it("reports the endpoint, model, and recovery action when Ollama is unreachable", () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "skipped", reason: "unreachable", endpoint: "http://host.docker.internal:11434", @@ -124,6 +124,28 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("then rerun this command"); }); + it("reports a warm-up skipped after the command timeout budget is exhausted", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery( + { provider: "ollama-local", model: "qwen3.6:35b" }, + proc, + { timeoutSeconds: 1 }, + () => ({ + kind: "skipped", + reason: "deadline-exhausted", + endpoint: "http://host.docker.internal:11434", + }), + ); + + const stderr = writes.join(""); + expect(stderr).toContain("warm-up for 'qwen3.6:35b'"); + expect(stderr).toContain("http://host.docker.internal:11434"); + expect(stderr).toContain("was skipped"); + expect(stderr).toContain("timeout left no recovery budget"); + expect(stderr).toContain("continuing to OpenClaw dispatch"); + }); + it("names the endpoint and its reported models when the model is absent (#9455)", () => { const { writes, proc } = makeProcMock(); @@ -134,6 +156,7 @@ describe("runOllamaRestartRecovery", () => { endpointUrl: "http://host.openshell.internal:11434/v1", }, proc, + {}, () => ({ kind: "skipped", reason: "model-absent", @@ -155,15 +178,18 @@ describe("runOllamaRestartRecovery", () => { it("redacts and bounds recovery exceptions", () => { const { writes, proc } = makeProcMock(); const exposedToken = "sk-proj-NOT-A-REAL-SECRET-1234567890"; + const directionalControls = + "\u061c\u200e\u200f\u2028\u2029\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069"; expect(() => runOllamaRestartRecovery( { provider: "ollama-local", - model: "qwen3.6:35b\u001b[2J\u0007", - endpointUrl: "http://host.openshell.internal:11434/v1\u001b]52;c;clipboard\u0007", + model: `qwen3.6:35b${directionalControls}\u001b[2J\u0007`, + endpointUrl: `http://host.openshell.internal:11434/v1${directionalControls}\u001b]52;c;clipboard\u0007`, }, proc, + {}, () => { throw new Error( `synthetic\u001b[2J Docker transport failure\u0007 OPENAI_API_KEY=${exposedToken} ${"x".repeat(400)} END-OF-DETAIL`, @@ -184,7 +210,9 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("END-OF-DETAIL"); expect(stderr).not.toContain("\u001b"); expect(stderr).not.toContain("\u0007"); - expect(stderr.replace(/\n/gu, "")).not.toMatch(/[\u0000-\u001f\u007f-\u009f]/u); + expect(stderr.replace(/\n/gu, "")).not.toMatch( + /[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/u, + ); }); }); @@ -241,11 +269,11 @@ describe("agent passthrough Ollama recovery ordering", () => { ), ).rejects.toThrow("__exit:0"); - expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process); + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process, {}); expect(events).toEqual(["recovery", "dispatch"]); }); - it("checks a WSL direct route before non-JSON dispatch", async () => { + it("passes a short command timeout budget before non-JSON dispatch", async () => { const events: string[] = []; const route = { provider: "ollama-local", @@ -260,12 +288,14 @@ describe("agent passthrough Ollama recovery ordering", () => { await expect( runAgentPassthrough( "alpha", - { extraArgs: ["--agent", "main", "-m", "ping"] }, + { extraArgs: ["--agent", "main", "--timeout", "30", "-m", "ping"] }, { ...deps, runOllamaRestartRecovery: runRecovery }, ), ).rejects.toThrow("__exit:0"); - expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process); + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process, { + timeoutSeconds: 30, + }); expect(events).toEqual(["recovery", "dispatch"]); }); @@ -292,7 +322,7 @@ describe("agent passthrough Ollama recovery ordering", () => { registeredRoute: Parameters[0], proc: Parameters[1], ) => - runOllamaRestartRecovery(registeredRoute, proc, () => { + runOllamaRestartRecovery(registeredRoute, proc, {}, () => { throw new Error("synthetic recovery failure"); }); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 29ed67cff6a..3bfcc16f701 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -6,6 +6,7 @@ import { maybeWarmOllamaAfterDaemonRestart, OLLAMA_LOCAL_PROVIDER, type OllamaRestartRecoveryFailureReason, + type OllamaRestartRecoveryOptions, type OllamaRestartRecoveryResult, type OllamaRestartRecoveryRoute, } from "./ollama-restart-recovery"; @@ -14,6 +15,7 @@ export { OLLAMA_LOCAL_PROVIDER }; export type OllamaRestartRecoveryFn = ( route: OllamaRestartRecoveryRoute, + options?: OllamaRestartRecoveryOptions, ) => OllamaRestartRecoveryResult; export interface OllamaRestartRecoveryProcess { @@ -83,6 +85,15 @@ function reportRecovery( return; } + if (result.reason === "deadline-exhausted") { + const endpoint = boundedRecoveryEndpoint(result.endpoint, "the saved local Ollama endpoint"); + proc.stderr.write( + ` Ollama warm-up for '${model}' at ${endpoint} was skipped because the agent command ` + + `timeout left no recovery budget; continuing to OpenClaw dispatch.\n`, + ); + return; + } + const reason = result.reason; switch (reason) { case "already-loaded": @@ -115,11 +126,12 @@ function reportRecovery( export function runOllamaRestartRecovery( route: OllamaRestartRecoveryRoute, proc: OllamaRestartRecoveryProcess, + options: OllamaRestartRecoveryOptions = {}, recoverOllama: OllamaRestartRecoveryFn = maybeWarmOllamaAfterDaemonRestart, ): void { proc.stderr.write(" Checking whether the Ollama model is loaded...\n"); try { - reportRecovery(route, recoverOllama(route), proc); + reportRecovery(route, recoverOllama(route, options), proc); } catch (error) { const model = boundedOllamaRestartRecoveryDetail(route.model, "the registered model"); const endpoint = recordedEndpointLabel(route); diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index 318ccea1a34..9a122b980a9 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -132,6 +132,7 @@ import { isTimedOutAgentDispatch, OPENCLAW_AGENT_BOOLEAN_FLAGS, OPENCLAW_AGENT_VALUE_FLAGS, + requestedAgentTimeoutSeconds, runAgentDispatch, SILENT_AGENT_DISPATCH_EXIT_CODE, TIMED_OUT_AGENT_TURN_EXIT_CODE, @@ -553,7 +554,8 @@ export async function runAgentPassthrough( if (isOpenClawPassthroughCommand(command)) { if (lookup.kind === "agent" && lookup.provider === OLLAMA_LOCAL_PROVIDER) { const recoverOllama = deps.runOllamaRestartRecovery ?? runOllamaRestartRecovery; - recoverOllama(lookup, proc); + const timeoutSeconds = requestedAgentTimeoutSeconds(command); + recoverOllama(lookup, proc, timeoutSeconds === null ? {} : { timeoutSeconds }); } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); } diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 29c2563fb72..d0d0f72b883 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -133,7 +133,10 @@ export { MIN_OLLAMA_VERSION, } from "./ollama-version"; -export type RunCaptureExFn = (cmd: string[], opts?: { env?: NodeJS.ProcessEnv }) => CaptureResult; +export type RunCaptureExFn = ( + cmd: string[], + opts?: { env?: NodeJS.ProcessEnv; timeout?: number }, +) => CaptureResult; // Hosts that local-provider discovery may try when probing Ollama. The Windows // onboarding path separately checks host.docker.internal from Docker Desktop's From 6de489f0ab2447c9b7b5809643d5a6e853b5247d Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 00:22:48 -0700 Subject: [PATCH 08/48] fix: share Ollama recovery deadline Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 12 +-- .../sandbox/agent/ollama-restart-recovery.ts | 26 +++---- .../agent/passthrough-dispatch.test.ts | 13 +++- .../sandbox/agent/passthrough-dispatch.ts | 65 +++++++++++----- .../agent/passthrough-ollama-recovery.test.ts | 75 ++++++++++++------- src/lib/actions/sandbox/agent/passthrough.ts | 48 +++++++++++- 6 files changed, 164 insertions(+), 75 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 111d7ad3e81..ab546d46c49 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -77,7 +77,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { prepareDockerEnvironment, }, ), - ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + ).toEqual({ kind: "warmed", ok: true }); expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, @@ -217,7 +217,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toEqual({ kind: "warmed", ok: false, - timedOut: true, reason: "timeout", endpoint: "http://127.0.0.1:11434", detail: "warm-up exceeded 300 seconds", @@ -241,7 +240,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { now, }, ), - ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + ).toEqual({ kind: "warmed", ok: true }); const warmCommand = runCaptureExImpl.mock.calls[0][0]; expect(warmCommand[warmCommand.indexOf("--max-time") + 1]).toBe("25"); @@ -287,7 +286,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toMatchObject({ kind: "warmed", ok: false, - timedOut: false, reason: "ollama-error", endpoint: "http://127.0.0.1:11434", detail: expect.stringContaining("model not found"), @@ -340,7 +338,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toMatchObject({ kind: "warmed", ok: false, - timedOut: false, reason: "ollama-error", endpoint: "http://127.0.0.1:11434", detail: expect.stringContaining("runner stopped unexpectedly"), @@ -360,7 +357,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + ).toEqual({ kind: "warmed", ok: true }); }); it.each([ @@ -380,7 +377,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toMatchObject({ kind: "warmed", ok: false, - timedOut: false, reason: "invalid-response", endpoint: "http://127.0.0.1:11434", }); @@ -398,7 +394,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toEqual({ kind: "warmed", ok: false, - timedOut: false, reason: "command-failed", endpoint: "http://127.0.0.1:11434", detail: "warm-up exited 7", @@ -418,7 +413,6 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toEqual({ kind: "warmed", ok: false, - timedOut: false, reason: "spawn-failed", endpoint: "http://127.0.0.1:11434", detail: "spawn failed", diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 376360c77be..571699f0aad 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -71,11 +71,10 @@ export type OllamaRestartRecoveryResult = | { kind: "skipped"; reason: "unreachable"; endpoint: string } | { kind: "skipped"; reason: "deadline-exhausted"; endpoint: string } | { kind: "skipped"; reason: "model-absent"; endpoint: string; inventoryLabel: string } - | { kind: "warmed"; ok: true; timedOut: false } + | { kind: "warmed"; ok: true } | { kind: "warmed"; ok: false; - timedOut: boolean; reason: OllamaRestartRecoveryFailureReason; endpoint: string; detail: string; @@ -182,11 +181,7 @@ function recoveryDeadlineMilliseconds( now: () => number, ): number | null { if (timeoutSeconds === undefined) return null; - const boundedSeconds = - Number.isFinite(timeoutSeconds) && timeoutSeconds > 0 - ? Math.min(timeoutSeconds, OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS) - : 0; - return now() + boundedSeconds * 1000; + return now() + Math.min(timeoutSeconds, OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS) * 1000; } function remainingRecoveryMilliseconds(deadline: number | null, now: () => number): number { @@ -243,12 +238,17 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "skipped", reason: "missing-model" }; } - const now = deps.now ?? Date.now; - const recoveryDeadline = recoveryDeadlineMilliseconds(deps.timeoutSeconds, now); - const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); const rawEndpoint = `http://${rawHost}:${OLLAMA_PORT}`; + if ( + deps.timeoutSeconds !== undefined && + (!Number.isFinite(deps.timeoutSeconds) || deps.timeoutSeconds <= 0) + ) { + return { kind: "skipped", reason: "deadline-exhausted", endpoint: rawEndpoint }; + } + const now = deps.now ?? Date.now; + const recoveryDeadline = recoveryDeadlineMilliseconds(deps.timeoutSeconds, now); const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; const rawCapture = createOllamaApiCapture( deps.runCaptureImpl, @@ -287,7 +287,6 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "warmed", ok: false, - timedOut: true, reason: "timeout", endpoint: rawEndpoint, detail: boundedOllamaRestartRecoveryDetail( @@ -300,7 +299,6 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "warmed", ok: false, - timedOut: false, reason: "command-failed", endpoint: rawEndpoint, detail: boundedOllamaRestartRecoveryDetail( @@ -331,18 +329,16 @@ export function maybeWarmOllamaAfterDaemonRestart( return { kind: "warmed", ok: false, - timedOut: false, reason: response, endpoint: rawEndpoint, detail: boundedOllamaRestartRecoveryDetail(result.stdout, `Ollama returned ${response}`), }; } - return { kind: "warmed", ok: true, timedOut: false }; + return { kind: "warmed", ok: true }; } catch (error) { return { kind: "warmed", ok: false, - timedOut: false, reason: "spawn-failed", endpoint: rawEndpoint, detail: boundedOllamaRestartRecoveryDetail( diff --git a/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts b/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts index f43acf7dfdd..a28273e0359 100644 --- a/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts @@ -12,6 +12,7 @@ import { agentDispatchStdio, isSilentAgentDispatch, isTimedOutAgentDispatch, + replaceRequestedAgentTimeoutSeconds, requestedAgentTimeoutSeconds, runAgentDispatch, SILENT_AGENT_DISPATCH_EXIT_CODE, @@ -174,11 +175,19 @@ describe("requestedAgentTimeoutSeconds", () => { }); it("reads a separated --timeout value (#8723)", () => { - expect(requestedAgentTimeoutSeconds(agent("--agent", "main", "--timeout", "30"))).toBe(30); + const command = agent("--agent", "main", "--timeout", "30"); + expect(requestedAgentTimeoutSeconds(command)).toBe(30); + expect(replaceRequestedAgentTimeoutSeconds(command, 20)).toEqual( + agent("--agent", "main", "--timeout", "20"), + ); }); it("reads an equals-form --timeout value (#8723)", () => { - expect(requestedAgentTimeoutSeconds(agent("--timeout=45", "-m", "hi"))).toBe(45); + const command = agent("--timeout=45", "-m", "hi"); + expect(requestedAgentTimeoutSeconds(command)).toBe(45); + expect(replaceRequestedAgentTimeoutSeconds(command, 20)).toEqual( + agent("--timeout=20", "-m", "hi"), + ); }); it("reads a timeout after documented boolean and equals-form options (#8723)", () => { diff --git a/src/lib/actions/sandbox/agent/passthrough-dispatch.ts b/src/lib/actions/sandbox/agent/passthrough-dispatch.ts index 123fea76904..0fdff8a3bb0 100644 --- a/src/lib/actions/sandbox/agent/passthrough-dispatch.ts +++ b/src/lib/actions/sandbox/agent/passthrough-dispatch.ts @@ -320,23 +320,25 @@ export const OPENCLAW_AGENT_BOOLEAN_FLAGS = new Set(["--deliver"]); */ export const AGENT_DISPATCH_DEADLINE_BUFFER_SECONDS = 30; -/** - * The `--timeout` an `openclaw agent` argv requests, or null when the argv - * requests none. - * - * Mirrors the documented flag grammar only far enough to read one value. - * Anything unrecognized, malformed, or past a `--` terminator returns null so - * the host keeps the wait unbounded rather than shortening a turn without - * evidence. `--timeout 0` disables the deadline upstream and returns null here - * for the same reason. - */ -export function requestedAgentTimeoutSeconds(argv: readonly string[]): number | null { +type RequestedAgentTimeout = { + seconds: number; + argumentIndex: number; + inline: boolean; +}; + +function findRequestedAgentTimeout(argv: readonly string[]): RequestedAgentTimeout | null { if (argv[0] !== "openclaw" || argv[1] !== "agent") return null; for (let index = 2; index < argv.length; index += 1) { const arg = argv[index] as string; if (arg === "--") return null; - if (arg === "--timeout") return parseDeadlineSeconds(argv[index + 1]); - if (arg.startsWith("--timeout=")) return parseDeadlineSeconds(arg.slice("--timeout=".length)); + if (arg === "--timeout") { + const seconds = parseDeadlineSeconds(argv[index + 1]); + return seconds === null ? null : { seconds, argumentIndex: index, inline: false }; + } + if (arg.startsWith("--timeout=")) { + const seconds = parseDeadlineSeconds(arg.slice("--timeout=".length)); + return seconds === null ? null : { seconds, argumentIndex: index, inline: true }; + } if (OPENCLAW_AGENT_VALUE_FLAGS.has(arg)) { index += 1; continue; @@ -349,11 +351,7 @@ export function requestedAgentTimeoutSeconds(argv: readonly string[]): number | ) { continue; } - if ( - arg === "--json" || - arg.startsWith("--json=") || - OPENCLAW_AGENT_BOOLEAN_FLAGS.has(arg) - ) { + if (arg === "--json" || arg.startsWith("--json=") || OPENCLAW_AGENT_BOOLEAN_FLAGS.has(arg)) { continue; } return null; @@ -361,6 +359,37 @@ export function requestedAgentTimeoutSeconds(argv: readonly string[]): number | return null; } +/** + * The `--timeout` an `openclaw agent` argv requests, or null when the argv + * requests none. + * + * Mirrors the documented flag grammar only far enough to read one value. + * Anything unrecognized, malformed, or past a `--` terminator returns null so + * the host keeps the wait unbounded rather than shortening a turn without + * evidence. `--timeout 0` disables the deadline upstream and returns null here + * for the same reason. + */ +export function requestedAgentTimeoutSeconds(argv: readonly string[]): number | null { + return findRequestedAgentTimeout(argv)?.seconds ?? null; +} + +/** Replace a valid agent timeout while preserving its separated or equals form. */ +export function replaceRequestedAgentTimeoutSeconds( + argv: readonly string[], + timeoutSeconds: number, +): readonly string[] { + const requested = findRequestedAgentTimeout(argv); + if (requested === null) return argv; + const replacement = String(Math.max(1, Math.floor(timeoutSeconds))); + const command = [...argv]; + if (requested.inline) { + command[requested.argumentIndex] = `--timeout=${replacement}`; + } else { + command[requested.argumentIndex + 1] = replacement; + } + return command; +} + function parseDeadlineSeconds(raw: string | undefined): number | null { if (raw === undefined || !/^\d+$/.test(raw)) return null; const seconds = Number(raw); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index f43f8b45f73..a4768621427 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { type AgentPassthroughDeps, runAgentPassthrough } from "./passthrough"; +import { requestedAgentTimeoutSeconds } from "./passthrough-dispatch"; import { runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; function makeProcMock() { @@ -41,7 +42,6 @@ describe("runOllamaRestartRecovery", () => { runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "warmed", ok: true, - timedOut: false, })); expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is loaded and ready"); @@ -53,7 +53,6 @@ describe("runOllamaRestartRecovery", () => { runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "warmed", ok: false, - timedOut: true, reason: "timeout", endpoint: "http://host.docker.internal:11434", detail: "curl timed out after 300 seconds", @@ -79,7 +78,6 @@ describe("runOllamaRestartRecovery", () => { runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ kind: "warmed", ok: false, - timedOut: false, reason, endpoint: "http://host.docker.internal:11434", detail: "bounded failure detail", @@ -273,31 +271,52 @@ describe("agent passthrough Ollama recovery ordering", () => { expect(events).toEqual(["recovery", "dispatch"]); }); - it("passes a short command timeout budget before non-JSON dispatch", async () => { - const events: string[] = []; - const route = { - provider: "ollama-local", - model: "qwen3.6:35b", - endpointUrl: "http://host.openshell.internal:11434/v1", - }; - const deps = makePassthroughDeps(route, events); - const runRecovery = vi.fn(() => { - events.push("recovery"); - }); - - await expect( - runAgentPassthrough( - "alpha", - { extraArgs: ["--agent", "main", "--timeout", "30", "-m", "ping"] }, - { ...deps, runOllamaRestartRecovery: runRecovery }, - ), - ).rejects.toThrow("__exit:0"); - - expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process, { - timeoutSeconds: 30, - }); - expect(events).toEqual(["recovery", "dispatch"]); - }); + it.each([ + ["partial recovery", 5_000, 25], + ["an exhausted recovery budget", 30_000, 1], + ])( + "reduces a 30-second timeout after %s", + async (_name, elapsedMilliseconds, expectedTimeout) => { + const events: string[] = []; + const dispatchedTimeouts: number[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }; + const deps = makePassthroughDeps(route, events); + const now = vi + .fn() + .mockReturnValueOnce(0) + .mockReturnValueOnce(0) + .mockReturnValueOnce(elapsedMilliseconds); + const runRecovery = vi.fn(() => { + events.push("recovery"); + }); + const execNonJson = vi.fn((( + _sandboxName: string, + dispatchedCommand: readonly string[], + ): never => { + dispatchedTimeouts.push(requestedAgentTimeoutSeconds(dispatchedCommand) ?? -1); + events.push("dispatch"); + throw new Error("__exit:0"); + }) as NonNullable); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "--timeout", "30", "-m", "ping"] }, + { ...deps, execNonJson, now, runOllamaRestartRecovery: runRecovery }, + ), + ).rejects.toThrow("__exit:0"); + + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process, { + timeoutSeconds: 29, + }); + expect(events).toEqual(["recovery", "dispatch"]); + expect(dispatchedTimeouts).toEqual([expectedTimeout]); + }, + ); it("dispatches after reporting an Ollama recovery exception", async () => { const events: string[] = []; diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index 9a122b980a9..c44d0c48c9e 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -111,6 +111,7 @@ // - Drop Ollama pre-dispatch recovery when supported daemon restarts preserve // loaded runners or NemoClaw manages and warms the daemon lifecycle. +import { performance } from "node:perf_hooks"; import { type AgentDefinition, isTerminalAgent, listAgents, loadAgent } from "../../../agent/defs"; import { CLI_NAME } from "../../../cli/branding"; import { isStdinTty } from "../../../core/stdin"; @@ -132,6 +133,7 @@ import { isTimedOutAgentDispatch, OPENCLAW_AGENT_BOOLEAN_FLAGS, OPENCLAW_AGENT_VALUE_FLAGS, + replaceRequestedAgentTimeoutSeconds, requestedAgentTimeoutSeconds, runAgentDispatch, SILENT_AGENT_DISPATCH_EXIT_CODE, @@ -242,6 +244,7 @@ export interface AgentPassthroughDeps { execNonJson?: typeof runAgentNonJsonPassthrough; runOllamaRestartRecovery?: typeof runOllamaRestartRecovery; getRecentShieldsAutoRestore?: (sandboxName: string) => ShieldsAutoRestoreReadResult; + now?: () => number; process?: { exit(code: number): never; stdout?: { write(s: string): unknown }; @@ -249,6 +252,41 @@ export interface AgentPassthroughDeps { }; } +const MINIMUM_AGENT_DISPATCH_BUDGET_SECONDS = 1; + +function startAgentCommandDeadline(command: readonly string[], now: () => number): number | null { + const timeoutSeconds = requestedAgentTimeoutSeconds(command); + const timeoutMilliseconds = timeoutSeconds === null ? null : timeoutSeconds * 1000; + return timeoutMilliseconds === null || !Number.isSafeInteger(timeoutMilliseconds) + ? null + : now() + timeoutMilliseconds; +} + +function remainingAgentCommandSeconds(deadline: number | null, now: () => number): number | null { + return deadline === null ? null : Math.max(0, (deadline - now()) / 1000); +} + +function recoveryBudgetSeconds(deadline: number | null, now: () => number): number | null { + const remaining = remainingAgentCommandSeconds(deadline, now); + // Keep one second for the actual turn so best-effort recovery cannot consume + // the entire user-requested deadline before OpenClaw starts. + return remaining === null ? null : Math.max(0, remaining - MINIMUM_AGENT_DISPATCH_BUDGET_SECONDS); +} + +function commandWithRemainingDeadline( + command: readonly string[], + deadline: number | null, + now: () => number, +): readonly string[] { + const remaining = remainingAgentCommandSeconds(deadline, now); + return remaining === null + ? command + : replaceRequestedAgentTimeoutSeconds( + command, + Math.max(MINIMUM_AGENT_DISPATCH_BUDGET_SECONDS, Math.ceil(remaining)), + ); +} + type RegistryReadResult = | { kind: "missing" } | { @@ -551,17 +589,21 @@ export async function runAgentPassthrough( if (isOpenClawPassthroughCommand(command) && !hasTargetSelector(extraArgs)) { rejectNoTargetSelector(proc); } + let dispatchCommand: readonly string[] = command; if (isOpenClawPassthroughCommand(command)) { + const now = deps.now ?? (() => performance.now()); + const commandDeadline = startAgentCommandDeadline(command, now); if (lookup.kind === "agent" && lookup.provider === OLLAMA_LOCAL_PROVIDER) { const recoverOllama = deps.runOllamaRestartRecovery ?? runOllamaRestartRecovery; - const timeoutSeconds = requestedAgentTimeoutSeconds(command); + const timeoutSeconds = recoveryBudgetSeconds(commandDeadline, now); recoverOllama(lookup, proc, timeoutSeconds === null ? {} : { timeoutSeconds }); } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); + dispatchCommand = commandWithRemainingDeadline(command, commandDeadline, now); } if (isOpenClawPassthroughCommand(command) && requestsOpenClawJsonOutput(extraArgs)) { const execJson = deps.execJson ?? runAgentJsonPassthrough; - await execJson(sandboxName, command, { + await execJson(sandboxName, dispatchCommand, { exit: proc.exit.bind(proc), stdout: proc.stdout ?? process.stdout, stderr: proc.stderr, @@ -570,7 +612,7 @@ export async function runAgentPassthrough( } if (isOpenClawPassthroughCommand(command)) { const execNonJson = deps.execNonJson ?? runAgentNonJsonPassthrough; - await execNonJson(sandboxName, command, proc); + await execNonJson(sandboxName, dispatchCommand, proc); return; } const exec = deps.exec ?? execSandbox; From d065f6744bc7e786cb54b19f38d324d22eab2fcf Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 00:46:38 -0700 Subject: [PATCH 09/48] fix(agent): bound Ollama recovery probes Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 57 ++++++++++++++++--- .../sandbox/agent/ollama-restart-recovery.ts | 47 +++++++++++---- src/lib/inference/local.ts | 17 ++++-- .../inference/ollama-runtime-context.test.ts | 18 +++++- src/lib/inference/ollama-runtime-context.ts | 17 ++++-- .../sandbox-facing-ollama-model.test.ts | 13 ++++- 6 files changed, 140 insertions(+), 29 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index ab546d46c49..d5fd2f4c523 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -224,20 +224,24 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("limits warm-up to the command timeout budget remaining after the probe", () => { + let nowMs = 1_000; const runCaptureExImpl = vi.fn( (_command: string[], _options?: { env?: NodeJS.ProcessEnv; timeout?: number }) => successfulWarmResult(), ); - const now = vi.fn().mockReturnValueOnce(1_000).mockReturnValueOnce(6_000); + const probeRuntimeModelStatus = vi.fn(() => { + nowMs = 6_000; + return unloadedStatus; + }); expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { - probeRuntimeModelStatus: () => unloadedStatus, + probeRuntimeModelStatus, runCaptureExImpl, timeoutSeconds: 30, - now, + now: () => nowMs, }, ), ).toEqual({ kind: "warmed", ok: true }); @@ -245,20 +249,30 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { const warmCommand = runCaptureExImpl.mock.calls[0][0]; expect(warmCommand[warmCommand.indexOf("--max-time") + 1]).toBe("25"); expect(runCaptureExImpl.mock.calls[0][1]?.timeout).toBe(25_000); + expect(probeRuntimeModelStatus).toHaveBeenCalledWith( + "qwen3.6:35b", + expect.any(Function), + expect.any(Function), + 5_000, + ); }); it("skips warm-up when the probe consumes the command timeout budget", () => { + let nowMs = 1_000; const runCaptureExImpl = vi.fn(() => successfulWarmResult()); - const now = vi.fn().mockReturnValueOnce(1_000).mockReturnValueOnce(31_000); + const probeRuntimeModelStatus = vi.fn(() => { + nowMs = 31_000; + return unloadedStatus; + }); expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { - probeRuntimeModelStatus: () => unloadedStatus, + probeRuntimeModelStatus, runCaptureExImpl, timeoutSeconds: 30, - now, + now: () => nowMs, }, ), ).toEqual({ @@ -269,6 +283,31 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runCaptureExImpl).not.toHaveBeenCalled(); }); + it("bounds the daemon probe and skips warm-up when a short timeout is consumed", () => { + let nowMs = 1_000; + const probeRuntimeModelStatus = vi.fn(() => { + nowMs = 3_000; + return unloadedStatus; + }); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, timeoutSeconds: 2, now: () => nowMs }, + ), + ).toEqual({ + kind: "skipped", + reason: "deadline-exhausted", + endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, + }); + expect(probeRuntimeModelStatus).toHaveBeenCalledWith( + "qwen3.6:35b", + expect.any(Function), + expect.any(Function), + 2_000, + ); + }); + it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { expect( maybeWarmOllamaAfterDaemonRestart( @@ -318,7 +357,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, inventoryLabel: "llama3.2:1b", }); - expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", expect.any(Function)); + expect(probeModelInventory).toHaveBeenCalledWith( + "host.docker.internal", + expect.any(Function), + 5_000, + ); }); it("keeps the warm failure when the daemon does hold the model (#9455)", () => { diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 571699f0aad..2b6941b11f0 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -50,8 +50,13 @@ export interface OllamaRestartRecoveryDeps extends OllamaRestartRecoveryOptions model: string, getOllamaHost: () => string, runCaptureImpl?: RunCaptureFn, + timeoutMilliseconds?: number, ) => OllamaRuntimeModelStatus; - probeModelInventory?: (host: string, runCaptureImpl?: RunCaptureFn) => string[] | null; + probeModelInventory?: ( + host: string, + runCaptureImpl?: RunCaptureFn, + timeoutMilliseconds?: number, + ) => string[] | null; runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; runCaptureImpl?: RunCaptureFn; @@ -82,6 +87,7 @@ export type OllamaRestartRecoveryResult = export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; +const OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS = 5_000; const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ OLLAMA_LOCALHOST, @@ -250,6 +256,10 @@ export function maybeWarmOllamaAfterDaemonRestart( const now = deps.now ?? Date.now; const recoveryDeadline = recoveryDeadlineMilliseconds(deps.timeoutSeconds, now); const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; + const probeBudgetMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); + if (probeBudgetMilliseconds === 0) { + return { kind: "skipped", reason: "deadline-exhausted", endpoint: rawEndpoint }; + } const rawCapture = createOllamaApiCapture( deps.runCaptureImpl, rawHost, @@ -262,7 +272,12 @@ export function maybeWarmOllamaAfterDaemonRestart( ); let status: OllamaRuntimeModelStatus; try { - status = probe(model, () => rawHost, rawCapture); + status = probe( + model, + () => rawHost, + rawCapture, + Math.min(OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, probeBudgetMilliseconds), + ); } catch { return { kind: "skipped", reason: "unreachable", endpoint: rawEndpoint }; } @@ -314,15 +329,25 @@ export function maybeWarmOllamaAfterDaemonRestart( // valid (#9455). Ask the same daemon for its inventory to tell them apart; // an unreadable inventory keeps the original warm-failure reason. if (response === "ollama-error") { - const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory; - const inventory = probeInventory(rawHost, rawCapture); - if (inventory && !ollamaInventoryContainsModel(inventory, model)) { - return { - kind: "skipped", - reason: "model-absent", - endpoint: `http://${rawHost}:${OLLAMA_PORT}`, - inventoryLabel: describeModelInventory(inventory), - }; + const inventoryBudgetMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); + if (inventoryBudgetMilliseconds > 0) { + const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory; + const inventory = probeInventory( + rawHost, + rawCapture, + Math.min( + OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, + inventoryBudgetMilliseconds, + ), + ); + if (inventory && !ollamaInventoryContainsModel(inventory, model)) { + return { + kind: "skipped", + reason: "model-absent", + endpoint: `http://${rawHost}:${OLLAMA_PORT}`, + inventoryLabel: describeModelInventory(inventory), + }; + } } } if (response !== "ok") { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 9c2ecfdc369..cf6fcb62264 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -122,7 +122,7 @@ export const QWEN3_6_OLLAMA_MODEL = assertRegistryTag("qwen3.6:35b"); export type RunCaptureFn = ( cmd: readonly string[], - opts?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv }, + opts?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv; timeout?: number }, ) => string; type PrepareDockerEnvironmentFn = () => PreparedDockerBuildEnvironment; @@ -1379,21 +1379,30 @@ export function getLocalProviderContainerReachabilityCheck( export function probeOllamaEndpointInventory( host: string, runCaptureImpl?: RunCaptureFn, + timeoutMilliseconds = 5_000, ): string[] | null { const capture = createOllamaApiCapture(runCaptureImpl, host); + const normalizedTimeoutMilliseconds = + Number.isFinite(timeoutMilliseconds) && timeoutMilliseconds > 0 + ? Math.floor(timeoutMilliseconds) + : 5_000; + const boundedTimeoutMilliseconds = Math.max( + 1, + Math.min(5_000, normalizedTimeoutMilliseconds), + ); const body = capture( [ "curl", ...buildValidatedCurlCommandArgs([ "-sf", "--connect-timeout", - "3", + String(Math.min(3, boundedTimeoutMilliseconds / 1000)), "--max-time", - "5", + String(boundedTimeoutMilliseconds / 1000), `http://${host}:${OLLAMA_PORT}/api/tags`, ]), ], - { ignoreError: true }, + { ignoreError: true, timeout: boundedTimeoutMilliseconds }, ); return parseOllamaModelInventory(body); } diff --git a/src/lib/inference/ollama-runtime-context.test.ts b/src/lib/inference/ollama-runtime-context.test.ts index 37005554dcf..ff96d319d1b 100644 --- a/src/lib/inference/ollama-runtime-context.test.ts +++ b/src/lib/inference/ollama-runtime-context.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT } from "../core/ports"; import { @@ -66,6 +66,22 @@ describe("Ollama runtime context helpers", () => { ).toBeNull(); }); + it("bounds the daemon status probe process and curl request", () => { + const capture = vi.fn( + ( + _command: readonly string[], + _options?: { ignoreError?: boolean; timeout?: number }, + ) => JSON.stringify({ models: [] }), + ); + + probeOllamaRuntimeModelStatus("qwen3.6:35b", getOllamaHost, capture, 1_200); + + expect(capture.mock.calls[0][0]).toEqual( + expect.arrayContaining(["--connect-timeout", "1.2", "--max-time", "1.2"]), + ); + expect(capture.mock.calls[0][1]).toMatchObject({ ignoreError: true, timeout: 1_200 }); + }); + it.each(["bogus", "1.5", 0, -1])( "warns and ignores malformed Ollama /api/ps context length %#", (value) => { diff --git a/src/lib/inference/ollama-runtime-context.ts b/src/lib/inference/ollama-runtime-context.ts index 39c22515e58..f232a4d9f89 100644 --- a/src/lib/inference/ollama-runtime-context.ts +++ b/src/lib/inference/ollama-runtime-context.ts @@ -16,7 +16,7 @@ import { runCapture } from "../runner"; export type OllamaRuntimeRunCaptureFn = ( cmd: readonly string[], - opts?: { ignoreError?: boolean }, + opts?: { ignoreError?: boolean; timeout?: number }, ) => string; export interface OllamaRuntimeModelStatus { @@ -140,22 +140,31 @@ export function probeOllamaRuntimeModelStatus( model: string, getOllamaHost: () => string, runCaptureImpl?: OllamaRuntimeRunCaptureFn, + timeoutMilliseconds = 5_000, ): OllamaRuntimeModelStatus { const capture = runCaptureImpl ?? runCapture; const host = getOllamaHost(); + const normalizedTimeoutMilliseconds = + Number.isFinite(timeoutMilliseconds) && timeoutMilliseconds > 0 + ? Math.floor(timeoutMilliseconds) + : 5_000; + const boundedTimeoutMilliseconds = Math.max( + 1, + Math.min(5_000, normalizedTimeoutMilliseconds), + ); const output = capture( [ "curl", ...buildValidatedCurlCommandArgs([ "-sf", "--connect-timeout", - "3", + String(Math.min(3, boundedTimeoutMilliseconds / 1000)), "--max-time", - "5", + String(boundedTimeoutMilliseconds / 1000), `http://${host}:${OLLAMA_PORT}/api/ps`, ]), ], - { ignoreError: true }, + { ignoreError: true, timeout: boundedTimeoutMilliseconds }, ); if (!output) return { probed: false, loaded: false, cpuOnly: false }; diff --git a/src/lib/inference/sandbox-facing-ollama-model.test.ts b/src/lib/inference/sandbox-facing-ollama-model.test.ts index f7c4899a908..11d089dc71c 100644 --- a/src/lib/inference/sandbox-facing-ollama-model.test.ts +++ b/src/lib/inference/sandbox-facing-ollama-model.test.ts @@ -108,13 +108,22 @@ describe("sandbox-facing Ollama model validation", () => { describe("Ollama model inventory", () => { it("queries the given daemon for its inventory", () => { - const capture = vi.fn((_command: readonly string[]) => tagsBody("llama3.2:1b")); + const capture = vi.fn( + ( + _command: readonly string[], + _options?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv; timeout?: number }, + ) => tagsBody("llama3.2:1b"), + ); - const inventory = probeOllamaEndpointInventory("host.docker.internal", capture); + const inventory = probeOllamaEndpointInventory("host.docker.internal", capture, 1_200); expect(commandUrl(capture.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ); + expect(capture.mock.calls[0][0]).toEqual( + expect.arrayContaining(["--connect-timeout", "1.2", "--max-time", "1.2"]), + ); + expect(capture.mock.calls[0][1]).toMatchObject({ ignoreError: true, timeout: 1_200 }); expect(inventory).toEqual(["llama3.2:1b"]); expect(ollamaInventoryContainsModel(inventory ?? [], "gemma4:26b")).toBe(false); }); From d071aaf3356e23253bef20cf5e6d53b6a58365dd Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 01:12:16 -0700 Subject: [PATCH 10/48] fix(agent): finish Ollama recovery follow-up Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 16 +++++++++++++++ .../agent/passthrough-ollama-recovery.test.ts | 10 ++++++++-- .../agent/passthrough-ollama-recovery.ts | 10 +++++----- src/lib/inference/local.ts | 2 ++ src/lib/inference/ollama-runtime-context.ts | 2 +- src/lib/inference/ollama/windows.ts | 20 +++++-------------- 6 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index d5fd2f4c523..2e3b5395e7e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -201,6 +201,22 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runCaptureExImpl).not.toHaveBeenCalled(); }); + it("skips the warm-up when the daemon status response is malformed", () => { + const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { runCaptureImpl: () => "not-json", runCaptureExImpl }, + ), + ).toEqual({ + kind: "skipped", + reason: "unreachable", + endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, + }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + it("reports a bounded warm-up timeout", () => { expect( maybeWarmOllamaAfterDaemonRestart( diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index a4768621427..f2c0337ca82 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -65,6 +65,8 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("at http://host.docker.internal:11434"); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); + expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).not.toContain("rerun this command"); }); it.each([ @@ -88,6 +90,8 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("http://host.docker.internal:11434"); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); + expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).not.toContain("rerun this command"); }); it.each([ @@ -119,7 +123,8 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("qwen3.6:35b"); expect(stderr).toContain("Restore Ollama access"); expect(stderr).toContain("confirm that it serves"); - expect(stderr).toContain("then rerun this command"); + expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).not.toContain("rerun this command"); }); it("reports a warm-up skipped after the command timeout budget is exhausted", () => { @@ -203,7 +208,8 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("OPENAI_API_KEY="); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("Restore Ollama access to that endpoint"); - expect(stderr).toContain("then rerun this command"); + expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).not.toContain("rerun this command"); expect(stderr).not.toContain(exposedToken); expect(stderr).not.toContain("END-OF-DETAIL"); expect(stderr).not.toContain("\u001b"); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 3bfcc16f701..50be11c00a9 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -63,9 +63,9 @@ function reportRecovery( const detail = boundedOllamaRestartRecoveryDetail(result.detail, "unknown warm-up error"); proc.stderr.write( ` Ollama warm-up for '${model}' at ${endpoint} ${describeWarmFailure(result.reason)} ` + - `(${detail}). OpenClaw dispatch will continue. To retry the warm-up, restore ` + - `Ollama access to ${endpoint} and confirm that it serves '${model}', then rerun ` + - `this command.\n`, + `(${detail}). OpenClaw dispatch will continue. Restore Ollama access to ${endpoint} ` + + `and confirm that it serves '${model}'. NemoClaw will retry the warm-up before the ` + + `next agent command.\n`, ); return; } @@ -104,7 +104,7 @@ function reportRecovery( proc.stderr.write( ` Ollama at ${endpoint} was unreachable while checking '${model}'; continuing to ` + `OpenClaw dispatch. Restore Ollama access to ${endpoint}, confirm that it serves ` + - `'${model}', then rerun this command.\n`, + `'${model}'. NemoClaw will retry the warm-up before the next agent command.\n`, ); break; } @@ -139,7 +139,7 @@ export function runOllamaRestartRecovery( proc.stderr.write( ` Ollama restart recovery for '${model}' ${endpoint} failed unexpectedly: ${detail}. ` + `OpenClaw dispatch will continue. Restore Ollama access to that endpoint, confirm it ` + - `serves '${model}', then rerun this command.\n`, + `serves '${model}'. NemoClaw will retry the warm-up before the next agent command.\n`, ); } } diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index cf6fcb62264..a0150dd8e5b 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -30,6 +30,8 @@ import { containerCanReachHostLoopback, isWsl, type WslDetectionOptions } from " import { type CaptureResult, run, runCapture, runCaptureEx, shellQuote } from "../runner"; import { buildSubprocessEnv } from "../subprocess-env"; +export { sleepSeconds }; + import { isLocalOllamaRouteOwner, OLLAMA_HOST_DOCKER_INTERNAL, diff --git a/src/lib/inference/ollama-runtime-context.ts b/src/lib/inference/ollama-runtime-context.ts index f232a4d9f89..bbdd25c69a5 100644 --- a/src/lib/inference/ollama-runtime-context.ts +++ b/src/lib/inference/ollama-runtime-context.ts @@ -202,7 +202,7 @@ export function probeOllamaRuntimeModelStatus( ...(hasSizeVram ? { sizeVram: rawSizeVram } : {}), }; } catch { - return { probed: true, loaded: false, cpuOnly: false }; + return { probed: false, loaded: false, cpuOnly: false }; } } diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index dda0d5d587a..acbc4340055 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -13,20 +13,10 @@ const { isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, setResolvedOllamaHost, + sleepSeconds, } = require("../local"); const { OLLAMA_PORT } = require("../../core/ports"); -// Avoid starting a subprocess for each fixed readiness delay. -// The supported Windows-host Ollama path runs through WSL PowerShell interop. -// Native Windows activation remains gated by #8178. -const sleepBuffer = new SharedArrayBuffer(4); -const sleepArray = new Int32Array(sleepBuffer); - -function sleep(seconds: number): void { - if (seconds <= 0) return; - Atomics.wait(sleepArray, 0, 0, seconds * 1000); -} - function psSingleQuote(value: string): string { return `'${String(value).replace(/'/g, "''")}'`; } @@ -128,7 +118,7 @@ function awaitWindowsOllamaReady(opts: { prepareDockerEnvironment?: () => unknow opts.prepareDockerEnvironment, ); for (let attempt = 0; attempt < 15; attempt++) { - sleep(2); + sleepSeconds(2); const probe = capture( [ "curl", @@ -201,7 +191,7 @@ function launchAndAwaitWindowsOllama( console.error(` PowerShell launch via ${attempt.label} failed: ${detail}`); if (i < launchAttempts.length - 1) { killWindowsOllamaProcesses(); - sleep(1); + sleepSeconds(1); } } return false; @@ -219,7 +209,7 @@ function setupWindowsOllamaWith0000Binding( console.log(" Stopping existing Ollama on Windows host..."); } killWindowsOllamaProcesses(); - sleep(1); + sleepSeconds(1); return launchAndAwaitWindowsOllama({ watcherPath: watcherPath || undefined, installedPath: opts.installedPath, @@ -245,7 +235,7 @@ module.exports = { installOllamaOnWindowsHost, awaitWindowsOllamaReady, setupWindowsOllamaWith0000Binding, - sleep, + sleep: sleepSeconds, switchToWindowsOllamaHost, printWindowsOllamaTimeoutDiagnostics, }; From 136ca069a350f88065c4532a403fb6733278b1c2 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 01:24:12 -0700 Subject: [PATCH 11/48] fix: reject invalid Ollama status shapes Signed-off-by: Prekshi Vyas --- .../inference/ollama-runtime-context.test.ts | 17 +++++++++++++---- src/lib/inference/ollama-runtime-context.ts | 12 ++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/lib/inference/ollama-runtime-context.test.ts b/src/lib/inference/ollama-runtime-context.test.ts index ff96d319d1b..7aa131c51ec 100644 --- a/src/lib/inference/ollama-runtime-context.test.ts +++ b/src/lib/inference/ollama-runtime-context.test.ts @@ -68,10 +68,8 @@ describe("Ollama runtime context helpers", () => { it("bounds the daemon status probe process and curl request", () => { const capture = vi.fn( - ( - _command: readonly string[], - _options?: { ignoreError?: boolean; timeout?: number }, - ) => JSON.stringify({ models: [] }), + (_command: readonly string[], _options?: { ignoreError?: boolean; timeout?: number }) => + JSON.stringify({ models: [] }), ); probeOllamaRuntimeModelStatus("qwen3.6:35b", getOllamaHost, capture, 1_200); @@ -82,6 +80,17 @@ describe("Ollama runtime context helpers", () => { expect(capture.mock.calls[0][1]).toMatchObject({ ignoreError: true, timeout: 1_200 }); }); + it.each(["{}", "null", '{"models":{}}'])( + "treats an invalid daemon status shape as a failed probe: %s", + (response) => { + expect(probeOllamaRuntimeModelStatus("qwen3.6:35b", getOllamaHost, () => response)).toEqual({ + probed: false, + loaded: false, + cpuOnly: false, + }); + }, + ); + it.each(["bogus", "1.5", 0, -1])( "warns and ignores malformed Ollama /api/ps context length %#", (value) => { diff --git a/src/lib/inference/ollama-runtime-context.ts b/src/lib/inference/ollama-runtime-context.ts index bbdd25c69a5..eadc5f8f2b8 100644 --- a/src/lib/inference/ollama-runtime-context.ts +++ b/src/lib/inference/ollama-runtime-context.ts @@ -148,10 +148,7 @@ export function probeOllamaRuntimeModelStatus( Number.isFinite(timeoutMilliseconds) && timeoutMilliseconds > 0 ? Math.floor(timeoutMilliseconds) : 5_000; - const boundedTimeoutMilliseconds = Math.max( - 1, - Math.min(5_000, normalizedTimeoutMilliseconds), - ); + const boundedTimeoutMilliseconds = Math.max(1, Math.min(5_000, normalizedTimeoutMilliseconds)); const output = capture( [ "curl", @@ -169,8 +166,11 @@ export function probeOllamaRuntimeModelStatus( if (!output) return { probed: false, loaded: false, cpuOnly: false }; try { - const parsed = JSON.parse(String(output || "")); - const models = Array.isArray(parsed?.models) ? parsed.models : []; + const parsed = JSON.parse(String(output || "")) as { models?: unknown } | null; + if (!parsed || !Array.isArray(parsed.models)) { + return { probed: false, loaded: false, cpuOnly: false }; + } + const models = parsed.models; const target = normalizeOllamaModelName(model); const loaded = models.find((entry: { name?: unknown; model?: unknown }) => { return ( From 6767b5f6255c0b724be4d21065c625c9455d8130 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 01:39:30 -0700 Subject: [PATCH 12/48] fix: preserve Ollama recovery diagnostics Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 24 +++++++++++++++++++ .../sandbox/agent/ollama-restart-recovery.ts | 21 +++++++++------- src/lib/inference/local.ts | 10 ++++---- .../sandbox-facing-ollama-model.test.ts | 14 +++++++++++ 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 2e3b5395e7e..57c28903e8a 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -347,6 +347,30 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); + it("keeps the warm-up error when the inventory probe throws", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + probeModelInventory: () => { + throw new Error("inventory unavailable"); + }, + runCaptureExImpl: () => ({ + stdout: JSON.stringify({ error: "runner stopped unexpectedly" }), + exitCode: 0, + timedOut: false, + }), + }, + ), + ).toMatchObject({ + kind: "warmed", + ok: false, + reason: "ollama-error", + detail: expect.stringContaining("runner stopped unexpectedly"), + }); + }); + it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", () => { const probeModelInventory = vi.fn(() => ["llama3.2:1b"]); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 2b6941b11f0..205f3572991 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -332,14 +332,19 @@ export function maybeWarmOllamaAfterDaemonRestart( const inventoryBudgetMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); if (inventoryBudgetMilliseconds > 0) { const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory; - const inventory = probeInventory( - rawHost, - rawCapture, - Math.min( - OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, - inventoryBudgetMilliseconds, - ), - ); + let inventory: string[] | null = null; + try { + inventory = probeInventory( + rawHost, + rawCapture, + Math.min( + OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, + inventoryBudgetMilliseconds, + ), + ); + } catch { + // Inventory only refines the original warm-up error. + } if (inventory && !ollamaInventoryContainsModel(inventory, model)) { return { kind: "skipped", diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index a0150dd8e5b..152ce6abc7b 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -641,7 +641,10 @@ export function ollamaInventoryContainsModel(inventory: string[], model: string) } function sanitizeModelNameForDisplay(value: string): string { - const sanitized = value.replace(/[\u0000-\u001f\u007f-\u009f]/g, ""); + const sanitized = value.replace( + /[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/gu, + "", + ); return sanitized.length > 120 ? `${sanitized.slice(0, 117)}...` : sanitized; } @@ -1388,10 +1391,7 @@ export function probeOllamaEndpointInventory( Number.isFinite(timeoutMilliseconds) && timeoutMilliseconds > 0 ? Math.floor(timeoutMilliseconds) : 5_000; - const boundedTimeoutMilliseconds = Math.max( - 1, - Math.min(5_000, normalizedTimeoutMilliseconds), - ); + const boundedTimeoutMilliseconds = Math.max(1, Math.min(5_000, normalizedTimeoutMilliseconds)); const body = capture( [ "curl", diff --git a/src/lib/inference/sandbox-facing-ollama-model.test.ts b/src/lib/inference/sandbox-facing-ollama-model.test.ts index 11d089dc71c..59d9a156cbf 100644 --- a/src/lib/inference/sandbox-facing-ollama-model.test.ts +++ b/src/lib/inference/sandbox-facing-ollama-model.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT } from "../core/ports"; import { + describeModelInventory, getLocalProviderContainerReachabilityCheck, ollamaInventoryContainsModel, probeOllamaEndpointInventory, @@ -140,4 +141,17 @@ describe("Ollama model inventory", () => { it("keeps a valid empty inventory authoritative", () => { expect(probeOllamaEndpointInventory("127.0.0.1", () => tagsBody())).toEqual([]); }); + + it("removes directional controls from inventory labels and validation messages", () => { + const controls = + "\u061c\u200e\u200f\u2028\u2029\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069"; + const model = `qwen3.6${controls}:35b`; + + expect(describeModelInventory([model])).toBe("qwen3.6:35b"); + + const result = validateSandboxFacingOllamaModel("llama3.2:1b", () => tagsBody(model)); + expect(result.ok).toBe(false); + expect(result.message).toContain("reported models: qwen3.6:35b"); + expect(result.message).not.toMatch(/[\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/u); + }); }); From eace473239ed4c8d2659f2e9e4df860ceade516a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 02:45:58 -0700 Subject: [PATCH 13/48] fix(agent): make Ollama recovery interruptible Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 282 ++++++++++++++---- .../sandbox/agent/ollama-restart-recovery.ts | 252 +++++++++++++--- .../agent/passthrough-ollama-recovery.test.ts | 125 +++++--- .../agent/passthrough-ollama-recovery.ts | 15 +- src/lib/actions/sandbox/agent/passthrough.ts | 13 +- .../sandbox/doctor-system-checks.test.ts | 55 ++++ .../actions/sandbox/doctor-system-checks.ts | 25 +- src/lib/inference/health.ts | 49 ++- src/lib/inference/local.ts | 3 +- src/lib/inference/ollama-runtime-context.ts | 8 + src/lib/runner.ts | 1 + 11 files changed, 671 insertions(+), 157 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 57c28903e8a..c28206ee7bb 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { EventEmitter } from "node:events"; +import type { StdioOptions } from "node:child_process"; + import { describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import type { SandboxExecSignalSource } from "../exec"; +import type { AgentDispatchChild } from "./passthrough-dispatch"; import { maybeWarmOllamaAfterDaemonRestart, + runOllamaRecoveryCapture, type OllamaRestartRecoveryDeps, } from "./ollama-restart-recovery"; @@ -32,20 +38,20 @@ function getCommandBody(command: readonly string[]): Record { } describe("maybeWarmOllamaAfterDaemonRestart", () => { - it("skips routes that are not local Ollama", () => { - expect( + it("skips routes that are not local Ollama", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart({ provider: "vllm-local", model: "meta/llama" }), - ).toEqual({ kind: "skipped", reason: "not-ollama" }); + ).resolves.toEqual({ kind: "skipped", reason: "not-ollama" }); }); - it("skips a local Ollama route without a registered model", () => { - expect(maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local" })).toEqual({ + it("skips a local Ollama route without a registered model", async () => { + await expect(maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local" })).resolves.toEqual({ kind: "skipped", reason: "missing-model", }); }); - it("uses the persisted direct bridge route for both the default probe and warm-up", () => { + it("uses the persisted direct bridge route for both the default probe and warm-up", async () => { const cleanup = vi.fn(() => ({ ok: true as const })); const prepareDockerEnvironment = () => ({ env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, @@ -64,7 +70,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { : { stdout: "", exitCode: 1, timedOut: false }, ); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", @@ -77,7 +83,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { prepareDockerEnvironment, }, ), - ).toEqual({ kind: "warmed", ok: true }); + ).resolves.toEqual({ kind: "warmed", ok: true }); expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, @@ -99,11 +105,166 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(cleanup).toHaveBeenCalledTimes(2); }); - it("maps an auth-proxy route back to host loopback", () => { + it("runs the production status probe and warm-up through the async capture boundary", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce({ + stdout: JSON.stringify({ models: [] }), + stderr: "", + exitCode: 0, + timedOut: false, + }) + .mockResolvedValueOnce({ ...successfulWarmResult(), stderr: "" }); + + await expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { runRecoveryCaptureImpl }, + ), + ).resolves.toEqual({ kind: "warmed", ok: true }); + expect(runRecoveryCaptureImpl).toHaveBeenCalledTimes(2); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0]?.[0] ?? [])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? [])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, + ); + }); + + it("stops recovery after an async status probe is cancelled", async () => { + const runRecoveryCaptureImpl = vi.fn().mockResolvedValue({ + stdout: "", + stderr: "", + exitCode: null, + timedOut: false, + signal: "SIGTERM", + }); + + await expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { runRecoveryCaptureImpl }, + ), + ).resolves.toEqual({ kind: "cancelled", signal: "SIGTERM" }); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); + }); + + it("prepares and cleans one Docker environment for each recovery request", async () => { + const cleanups: Array> = []; + const prepareDockerEnvironment = vi.fn(() => { + const cleanup = vi.fn(() => ({ ok: true as const })); + cleanups.push(cleanup); + return { + env: { DOCKER_CONFIG: `/tmp/credential-free-docker-${cleanups.length}` }, + isolatedCredentialConfig: true, + cleanup, + }; + }); + const commands: string[][] = []; + const runCaptureImpl = vi.fn((command: readonly string[]) => { + commands.push([...command]); + return getCommandUrl(command).endsWith("/api/ps") + ? JSON.stringify({ models: [] }) + : JSON.stringify({ models: [{ name: "llama3.2:1b" }] }); + }); + const runCaptureExImpl = vi.fn((command: string[]) => { + commands.push([...command]); + return { + stdout: JSON.stringify({ error: "model not found" }), + exitCode: 0, + timedOut: false, + }; + }); + + await expect( + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, + }, + { runCaptureImpl, runCaptureExImpl, prepareDockerEnvironment }, + ), + ).resolves.toMatchObject({ kind: "skipped", reason: "model-absent" }); + + expect(commands.map(getCommandUrl)).toEqual([ + `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, + `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, + `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, + ]); + expect(commands.every((command) => command[0] === "docker")).toBe(true); + expect(prepareDockerEnvironment).toHaveBeenCalledTimes(3); + expect(cleanups).toHaveLength(3); + expect(cleanups[0]).toHaveBeenCalledOnce(); + expect(cleanups[1]).toHaveBeenCalledOnce(); + expect(cleanups[2]).toHaveBeenCalledOnce(); + }); + + it("forwards SIGTERM to an active recovery child and releases its Docker environment", async () => { + const childEvents = new EventEmitter(); + const signalEvents = new EventEmitter(); + const stderr = new EventEmitter(); + const stdout = new EventEmitter(); + const cleanup = vi.fn(() => ({ ok: true as const })); + const child: AgentDispatchChild = { + exitCode: null, + signalCode: null, + kill: vi.fn((signal) => { + child.signalCode = signal; + queueMicrotask(() => childEvents.emit("close", null, signal)); + return true; + }), + once: ((event: string, listener: (...args: unknown[]) => void) => + childEvents.once(event, listener)) as AgentDispatchChild["once"], + stderr, + stdout, + }; + const signalSource: SandboxExecSignalSource = { + add: (signal, listener) => signalEvents.on(signal, listener), + remove: (signal, listener) => signalEvents.off(signal, listener), + }; + const spawnRecoveryChild = vi.fn( + (_binary: string, _args: readonly string[], _stdio: StdioOptions, _env: NodeJS.ProcessEnv) => + child, + ); + + const pending = runOllamaRecoveryCapture( + ["curl", `http://host.docker.internal:${OLLAMA_PORT}/api/ps`], + { + host: "host.docker.internal", + timeoutMilliseconds: 300_000, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + signalSource, + spawnRecoveryChild, + }, + ); + signalEvents.emit("SIGTERM"); + + await expect(pending).resolves.toMatchObject({ + exitCode: null, + signal: "SIGTERM", + timedOut: false, + }); + expect(child.kill).toHaveBeenCalledOnce(); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + expect(spawnRecoveryChild.mock.calls[0]?.[0]).toBe("docker"); + expect(spawnRecoveryChild.mock.calls[0]?.[3]?.DOCKER_CONFIG).toBe( + "/tmp/credential-free-docker", + ); + expect(cleanup).toHaveBeenCalledOnce(); + expect(signalEvents.listenerCount("SIGTERM")).toBe(0); + expect(signalEvents.listenerCount("SIGINT")).toBe(0); + }); + + it("maps an auth-proxy route back to host loopback", async () => { const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); - maybeWarmOllamaAfterDaemonRestart( + await maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b", @@ -122,11 +283,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("curl"); }); - it("falls back to an allowlisted host instead of probing an arbitrary registry URL", () => { + it("falls back to an allowlisted host instead of probing an arbitrary registry URL", async () => { const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); - maybeWarmOllamaAfterDaemonRestart( + await maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b", @@ -143,11 +304,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); }); - it("does not map an unrecognized proxy-port host to host loopback (#6039)", () => { + it("does not map an unrecognized proxy-port host to host loopback (#6039)", async () => { const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); - maybeWarmOllamaAfterDaemonRestart( + await maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b", @@ -168,7 +329,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ); }); - it("skips the warm-up when the selected model is already loaded", () => { + it("skips the warm-up when the selected model is already loaded", async () => { const probeRuntimeModelStatus = vi.fn(() => ({ probed: true, loaded: true, @@ -176,24 +337,24 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { })); const runCaptureExImpl = vi.fn(() => successfulWarmResult()); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { probeRuntimeModelStatus, runCaptureExImpl }, ), - ).toEqual({ kind: "skipped", reason: "already-loaded" }); + ).resolves.toEqual({ kind: "skipped", reason: "already-loaded" }); expect(runCaptureExImpl).not.toHaveBeenCalled(); }); - it("skips the warm-up when the daemon probe is unreachable", () => { + it("skips the warm-up when the daemon probe is unreachable", async () => { const runCaptureExImpl = vi.fn(() => successfulWarmResult()); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { runCaptureImpl: () => "", runCaptureExImpl }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "skipped", reason: "unreachable", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, @@ -201,15 +362,15 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runCaptureExImpl).not.toHaveBeenCalled(); }); - it("skips the warm-up when the daemon status response is malformed", () => { + it("skips the warm-up when the daemon status response is malformed", async () => { const runCaptureExImpl = vi.fn(() => successfulWarmResult()); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { runCaptureImpl: () => "not-json", runCaptureExImpl }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "skipped", reason: "unreachable", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, @@ -217,8 +378,8 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runCaptureExImpl).not.toHaveBeenCalled(); }); - it("reports a bounded warm-up timeout", () => { - expect( + it("reports a bounded warm-up timeout", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -230,7 +391,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "warmed", ok: false, reason: "timeout", @@ -239,7 +400,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("limits warm-up to the command timeout budget remaining after the probe", () => { + it("limits warm-up to the command timeout budget remaining after the probe", async () => { let nowMs = 1_000; const runCaptureExImpl = vi.fn( (_command: string[], _options?: { env?: NodeJS.ProcessEnv; timeout?: number }) => @@ -250,7 +411,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { return unloadedStatus; }); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -260,7 +421,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { now: () => nowMs, }, ), - ).toEqual({ kind: "warmed", ok: true }); + ).resolves.toEqual({ kind: "warmed", ok: true }); const warmCommand = runCaptureExImpl.mock.calls[0][0]; expect(warmCommand[warmCommand.indexOf("--max-time") + 1]).toBe("25"); @@ -273,7 +434,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ); }); - it("skips warm-up when the probe consumes the command timeout budget", () => { + it("skips warm-up when the probe consumes the command timeout budget", async () => { let nowMs = 1_000; const runCaptureExImpl = vi.fn(() => successfulWarmResult()); const probeRuntimeModelStatus = vi.fn(() => { @@ -281,7 +442,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { return unloadedStatus; }); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -291,7 +452,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { now: () => nowMs, }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "skipped", reason: "deadline-exhausted", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, @@ -299,19 +460,19 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runCaptureExImpl).not.toHaveBeenCalled(); }); - it("bounds the daemon probe and skips warm-up when a short timeout is consumed", () => { + it("bounds the daemon probe and skips warm-up when a short timeout is consumed", async () => { let nowMs = 1_000; const probeRuntimeModelStatus = vi.fn(() => { nowMs = 3_000; return unloadedStatus; }); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { probeRuntimeModelStatus, timeoutSeconds: 2, now: () => nowMs }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "skipped", reason: "deadline-exhausted", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, @@ -324,8 +485,8 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ); }); - it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { - expect( + it("does not treat an exit-zero Ollama error body as a successful warm-up", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "missing:latest" }, { @@ -338,7 +499,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toMatchObject({ + ).resolves.toMatchObject({ kind: "warmed", ok: false, reason: "ollama-error", @@ -347,8 +508,8 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("keeps the warm-up error when the inventory probe throws", () => { - expect( + it("keeps the warm-up error when the inventory probe throws", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -363,7 +524,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toMatchObject({ + ).resolves.toMatchObject({ kind: "warmed", ok: false, reason: "ollama-error", @@ -371,10 +532,10 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", () => { + it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", async () => { const probeModelInventory = vi.fn(() => ["llama3.2:1b"]); - expect( + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", @@ -391,7 +552,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "skipped", reason: "model-absent", endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, @@ -399,13 +560,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); expect(probeModelInventory).toHaveBeenCalledWith( "host.docker.internal", - expect.any(Function), + undefined, 5_000, + undefined, ); }); - it("keeps the warm failure when the daemon does hold the model (#9455)", () => { - expect( + it("keeps the warm failure when the daemon does hold the model (#9455)", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -418,7 +580,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toMatchObject({ + ).resolves.toMatchObject({ kind: "warmed", ok: false, reason: "ollama-error", @@ -427,8 +589,8 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("accepts a completed thinking-only response from a thinking model", () => { - expect( + it("accepts a completed thinking-only response from a thinking model", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -440,7 +602,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: true }); + ).resolves.toEqual({ kind: "warmed", ok: true }); }); it.each([ @@ -448,8 +610,8 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ["malformed JSON", "not-json"], ["missing done marker", JSON.stringify({ response: "Hello!" })], ["empty response", JSON.stringify({ response: "", done: true })], - ])("rejects an invalid warm response: %s", (_name, stdout) => { - expect( + ])("rejects an invalid warm response: %s", async (_name, stdout) => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -457,7 +619,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), }, ), - ).toMatchObject({ + ).resolves.toMatchObject({ kind: "warmed", ok: false, reason: "invalid-response", @@ -465,8 +627,8 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("reports a non-zero warm command exit", () => { - expect( + it("reports a non-zero warm command exit", async () => { + await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { @@ -474,7 +636,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), }, ), - ).toEqual({ + ).resolves.toEqual({ kind: "warmed", ok: false, reason: "command-failed", @@ -483,7 +645,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("reports a warm process spawn failure without throwing", () => { + it("reports a warm process spawn failure without throwing", async () => { const deps: OllamaRestartRecoveryDeps = { probeRuntimeModelStatus: () => unloadedStatus, runCaptureExImpl: () => { @@ -491,9 +653,9 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }, }; - expect( + await expect( maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local", model: "qwen3.6:35b" }, deps), - ).toEqual({ + ).resolves.toEqual({ kind: "warmed", ok: false, reason: "spawn-failed", diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 205f3572991..637dace2eeb 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -14,26 +14,36 @@ // supported Ollama versions persist runners across restart, or when NemoClaw // manages daemon lifecycle and can warm the model at restart time instead. +import { spawn, type StdioOptions } from "node:child_process"; + import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, createOllamaApiCapture, createOllamaApiCaptureEx, - getOllamaApiCommand, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, + parseOllamaModelInventory, + prepareOllamaApiExecution, probeOllamaEndpointInventory, type RunCaptureFn, type RunCaptureExFn, } from "../../../inference/local"; import { type OllamaRuntimeModelStatus, + parseOllamaRuntimeModelStatus, probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; -import { redact, redactFull } from "../../../runner"; +import { buildSubprocessEnv, redact, redactFull } from "../../../runner"; +import type { SandboxExecSignalSource } from "../exec"; +import { + type AgentDispatchChild, + type AgentDispatchSpawner, + runAgentDispatch, +} from "./passthrough-dispatch"; export interface OllamaRestartRecoveryRoute { provider?: string | null; @@ -56,11 +66,15 @@ export interface OllamaRestartRecoveryDeps extends OllamaRestartRecoveryOptions host: string, runCaptureImpl?: RunCaptureFn, timeoutMilliseconds?: number, + prepareDockerEnvironment?: Parameters[2], ) => string[] | null; runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; runCaptureImpl?: RunCaptureFn; prepareDockerEnvironment?: Parameters[2]; + runRecoveryCaptureImpl?: OllamaRecoveryCaptureFn; + signalSource?: SandboxExecSignalSource; + spawnRecoveryChild?: OllamaRecoverySpawner; now?: () => number; } @@ -77,6 +91,7 @@ export type OllamaRestartRecoveryResult = | { kind: "skipped"; reason: "deadline-exhausted"; endpoint: string } | { kind: "skipped"; reason: "model-absent"; endpoint: string; inventoryLabel: string } | { kind: "warmed"; ok: true } + | { kind: "cancelled"; signal: NodeJS.Signals } | { kind: "warmed"; ok: false; @@ -88,6 +103,7 @@ export type OllamaRestartRecoveryResult = export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; const OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS = 5_000; +const OLLAMA_RESTART_RECOVERY_MAX_BUFFER_BYTES = 1024 * 1024; const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ OLLAMA_LOCALHOST, @@ -156,6 +172,111 @@ function resolveRawOllamaHost( return getAllowedFallbackHost(getOllamaHost); } +export type OllamaRecoveryCaptureResult = { + stdout: string; + stderr: string; + exitCode: number | null; + timedOut: boolean; + signal?: NodeJS.Signals | null; + error?: Error; +}; + +export type OllamaRecoveryCaptureFn = ( + command: readonly string[], + options: { + host: string; + timeoutMilliseconds: number; + prepareDockerEnvironment?: Parameters[2]; + signalSource?: SandboxExecSignalSource; + spawnRecoveryChild?: OllamaRecoverySpawner; + }, +) => Promise; + +export type OllamaRecoverySpawner = ( + binary: string, + args: readonly string[], + stdio: StdioOptions, + env: NodeJS.ProcessEnv, +) => AgentDispatchChild; + +const defaultOllamaRecoverySpawner: OllamaRecoverySpawner = (binary, args, stdio, env) => + spawn(binary, [...args], { stdio, env }) as unknown as AgentDispatchChild; + +/** Capture one bounded recovery command through the shared signal-aware child supervisor. */ +export async function runOllamaRecoveryCapture( + command: readonly string[], + options: Parameters[1], +): Promise { + const execution = prepareOllamaApiExecution(command, options.host, { + env: buildSubprocessEnv(), + prepareDockerEnvironment: options.prepareDockerEnvironment, + operation: "Ollama restart recovery", + }); + const [binary, ...args] = execution.command; + if (!binary) { + execution.cleanup(); + return { + stdout: "", + stderr: "", + exitCode: null, + timedOut: false, + error: new Error("Ollama recovery command is empty"), + }; + } + + let timedOut = false; + let timeout: ReturnType | undefined; + const spawnRecoveryChild = options.spawnRecoveryChild ?? defaultOllamaRecoverySpawner; + const spawnChild: AgentDispatchSpawner = (runBinary, runArgs, stdio) => { + const child = spawnRecoveryChild(runBinary, runArgs, stdio, execution.env ?? {}); + timeout = setTimeout(() => { + timedOut = true; + if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM"); + }, options.timeoutMilliseconds); + timeout.unref?.(); + return child; + }; + + try { + const result = await runAgentDispatch( + binary, + args, + { maxBufferBytes: OLLAMA_RESTART_RECOVERY_MAX_BUFFER_BYTES, stdinIsTty: true }, + { signalSource: options.signalSource, spawnChild }, + ); + return { + stdout: result.stdout.trim(), + stderr: result.stderr.trim(), + exitCode: result.status, + timedOut: timedOut || result.status === 28, + signal: result.signal, + ...(result.error ? { error: result.error } : {}), + }; + } finally { + if (timeout) clearTimeout(timeout); + execution.cleanup(); + } +} + +function buildOllamaProbeCommand( + hostname: string, + path: "/api/ps" | "/api/tags", + timeoutMilliseconds: number, +): string[] { + const maxTimeSeconds = timeoutMilliseconds / 1000; + return [ + "curl", + ...buildValidatedCurlCommandArgs([ + "-sf", + "--connect-timeout", + String(Math.min(3, maxTimeSeconds)), + "--max-time", + String(maxTimeSeconds), + `http://${hostname}:${OLLAMA_PORT}${path}`, + ]), + ]; +} + function buildWarmCommand(model: string, hostname: string, maxTimeSeconds: number): string[] { const body = JSON.stringify({ model, @@ -165,8 +286,9 @@ function buildWarmCommand(model: string, hostname: string, maxTimeSeconds: numbe keep_alive: "15m", options: { num_predict: 16 }, }); - return getOllamaApiCommand( - buildValidatedCurlCommandArgs([ + return [ + "curl", + ...buildValidatedCurlCommandArgs([ "-sS", "--connect-timeout", "3", @@ -178,8 +300,7 @@ function buildWarmCommand(model: string, hostname: string, maxTimeSeconds: numbe body, `http://${hostname}:${OLLAMA_PORT}/api/generate`, ]), - hostname, - ); + ]; } function recoveryDeadlineMilliseconds( @@ -231,10 +352,10 @@ export function boundedOllamaRestartRecoveryDetail(value: unknown, fallback: str * Warm a registered local Ollama model only when `/api/ps` proves that the * daemon is reachable and the selected model is no longer loaded. */ -export function maybeWarmOllamaAfterDaemonRestart( +export async function maybeWarmOllamaAfterDaemonRestart( route: OllamaRestartRecoveryRoute, deps: OllamaRestartRecoveryDeps = {}, -): OllamaRestartRecoveryResult { +): Promise { if (normalizeRouteValue(route.provider) !== OLLAMA_LOCAL_PROVIDER) { return { kind: "skipped", reason: "not-ollama" }; } @@ -260,24 +381,39 @@ export function maybeWarmOllamaAfterDaemonRestart( if (probeBudgetMilliseconds === 0) { return { kind: "skipped", reason: "deadline-exhausted", endpoint: rawEndpoint }; } - const rawCapture = createOllamaApiCapture( - deps.runCaptureImpl, - rawHost, - deps.prepareDockerEnvironment, - ); - const rawCaptureEx = createOllamaApiCaptureEx( - deps.runCaptureExImpl, - rawHost, - deps.prepareDockerEnvironment, - ); let status: OllamaRuntimeModelStatus; try { - status = probe( - model, - () => rawHost, - rawCapture, - Math.min(OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, probeBudgetMilliseconds), + const statusTimeoutMilliseconds = Math.min( + OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, + probeBudgetMilliseconds, ); + if (deps.probeRuntimeModelStatus || deps.runCaptureImpl) { + const rawCapture = createOllamaApiCapture( + deps.runCaptureImpl, + rawHost, + deps.prepareDockerEnvironment, + ); + status = probe(model, () => rawHost, rawCapture, statusTimeoutMilliseconds); + } else { + const capture = deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture; + const result = await capture( + buildOllamaProbeCommand(rawHost, "/api/ps", statusTimeoutMilliseconds), + { + host: rawHost, + timeoutMilliseconds: statusTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }, + ); + if (result.signal && !result.timedOut) { + return { kind: "cancelled", signal: result.signal }; + } + status = + result.exitCode === 0 && !result.error + ? parseOllamaRuntimeModelStatus(model, result.stdout) + : { probed: false, loaded: false, cpuOnly: false }; + } } catch { return { kind: "skipped", reason: "unreachable", endpoint: rawEndpoint }; } @@ -295,9 +431,24 @@ export function maybeWarmOllamaAfterDaemonRestart( const warmupTimeoutSeconds = warmupTimeoutMilliseconds / 1000; try { - const result = rawCaptureEx(buildWarmCommand(model, rawHost, warmupTimeoutSeconds), { - timeout: warmupTimeoutMilliseconds, - }); + const command = buildWarmCommand(model, rawHost, warmupTimeoutSeconds); + const result = deps.runCaptureExImpl + ? createOllamaApiCaptureEx( + deps.runCaptureExImpl, + rawHost, + deps.prepareDockerEnvironment, + )(command, { timeout: warmupTimeoutMilliseconds }) + : await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)(command, { + host: rawHost, + timeoutMilliseconds: warmupTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }); + const asyncResult = result as Partial; + if (asyncResult.signal && !result.timedOut) { + return { kind: "cancelled", signal: asyncResult.signal }; + } if (result.timedOut) { return { kind: "warmed", @@ -310,6 +461,18 @@ export function maybeWarmOllamaAfterDaemonRestart( ), }; } + if (asyncResult.error) { + return { + kind: "warmed", + ok: false, + reason: "spawn-failed", + endpoint: rawEndpoint, + detail: boundedOllamaRestartRecoveryDetail( + asyncResult.error, + "warm-up process could not start", + ), + }; + } if (result.exitCode !== 0) { return { kind: "warmed", @@ -331,17 +494,38 @@ export function maybeWarmOllamaAfterDaemonRestart( if (response === "ollama-error") { const inventoryBudgetMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); if (inventoryBudgetMilliseconds > 0) { - const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory; let inventory: string[] | null = null; try { - inventory = probeInventory( - rawHost, - rawCapture, - Math.min( - OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, - inventoryBudgetMilliseconds, - ), + const inventoryTimeoutMilliseconds = Math.min( + OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, + inventoryBudgetMilliseconds, ); + if (deps.probeModelInventory || deps.runCaptureImpl) { + inventory = (deps.probeModelInventory ?? probeOllamaEndpointInventory)( + rawHost, + deps.runCaptureImpl, + inventoryTimeoutMilliseconds, + deps.prepareDockerEnvironment, + ); + } else { + const inventoryResult = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)( + buildOllamaProbeCommand(rawHost, "/api/tags", inventoryTimeoutMilliseconds), + { + host: rawHost, + timeoutMilliseconds: inventoryTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }, + ); + if (inventoryResult.signal && !inventoryResult.timedOut) { + return { kind: "cancelled", signal: inventoryResult.signal }; + } + inventory = + inventoryResult.exitCode === 0 && !inventoryResult.error + ? parseOllamaModelInventory(inventoryResult.stdout) + : null; + } } catch { // Inventory only refines the original warm-up error. } diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index f2c0337ca82..0443377ded5 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -18,7 +18,7 @@ describe("runOllamaRestartRecovery", () => { it.each([ ["auth proxy", "http://host.openshell.internal:11435/v1"], ["WSL direct bridge", "http://host.openshell.internal:11434/v1"], - ])("forwards the persisted %s route to recovery", (_name, endpointUrl) => { + ])("forwards the persisted %s route to recovery", async (_name, endpointUrl) => { const recoverOllama = vi.fn(() => ({ kind: "skipped" as const, reason: "already-loaded" as const, @@ -30,33 +30,40 @@ describe("runOllamaRestartRecovery", () => { endpointUrl, }; - runOllamaRestartRecovery(route, proc, {}, recoverOllama); + await runOllamaRestartRecovery(route, proc, {}, recoverOllama); expect(recoverOllama).toHaveBeenCalledWith(route, {}); expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is already loaded"); }); - it("reports a successful warm-up", () => { + it("reports a successful warm-up", async () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ - kind: "warmed", - ok: true, - })); + await runOllamaRestartRecovery( + { provider: "ollama-local", model: "qwen3.6:35b" }, + proc, + {}, + () => ({ kind: "warmed", ok: true }), + ); expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is loaded and ready"); }); - it("reports a timeout before continuing to OpenClaw", () => { + it("reports a timeout before continuing to OpenClaw", async () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ - kind: "warmed", - ok: false, - reason: "timeout", - endpoint: "http://host.docker.internal:11434", - detail: "curl timed out after 300 seconds", - })); + await runOllamaRestartRecovery( + { provider: "ollama-local", model: "qwen3.6:35b" }, + proc, + {}, + () => ({ + kind: "warmed", + ok: false, + reason: "timeout", + endpoint: "http://host.docker.internal:11434", + detail: "curl timed out after 300 seconds", + }), + ); const stderr = writes.join(""); expect(stderr).toContain("Checking whether the Ollama model is loaded"); @@ -74,16 +81,21 @@ describe("runOllamaRestartRecovery", () => { ["ollama-error", "Ollama returned an error"], ["invalid-response", "Ollama returned an invalid response"], ["spawn-failed", "the warm-up process could not start"], - ] as const)("reports a %s warm-up failure", (reason, message) => { + ] as const)("reports a %s warm-up failure", async (reason, message) => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ - kind: "warmed", - ok: false, - reason, - endpoint: "http://host.docker.internal:11434", - detail: "bounded failure detail", - })); + await runOllamaRestartRecovery( + { provider: "ollama-local", model: "qwen3.6:35b" }, + proc, + {}, + () => ({ + kind: "warmed", + ok: false, + reason, + endpoint: "http://host.docker.internal:11434", + detail: "bounded failure detail", + }), + ); const stderr = writes.join(""); expect(stderr).toContain(message); @@ -98,25 +110,32 @@ describe("runOllamaRestartRecovery", () => { ["already-loaded", "Ollama model 'qwen3.6:35b' is already loaded"], ["missing-model", "No Ollama model is recorded for this sandbox"], ["not-ollama", "Checking whether the Ollama model is loaded"], - ] as const)("handles the %s skip reason", (reason, message) => { + ] as const)("reports the diagnostic for the %s recovery result", async (reason, message) => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ - kind: "skipped", - reason, - })); + await runOllamaRestartRecovery( + { provider: "ollama-local", model: "qwen3.6:35b" }, + proc, + {}, + () => ({ kind: "skipped", reason }), + ); expect(writes.join("")).toContain(message); }); - it("reports the endpoint, model, and recovery action when Ollama is unreachable", () => { + it("reports the endpoint, model, and recovery action when Ollama is unreachable", async () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, {}, () => ({ - kind: "skipped", - reason: "unreachable", - endpoint: "http://host.docker.internal:11434", - })); + await runOllamaRestartRecovery( + { provider: "ollama-local", model: "qwen3.6:35b" }, + proc, + {}, + () => ({ + kind: "skipped", + reason: "unreachable", + endpoint: "http://host.docker.internal:11434", + }), + ); const stderr = writes.join(""); expect(stderr).toContain("http://host.docker.internal:11434"); @@ -127,10 +146,10 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("rerun this command"); }); - it("reports a warm-up skipped after the command timeout budget is exhausted", () => { + it("reports a warm-up skipped after the command timeout budget is exhausted", async () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery( + await runOllamaRestartRecovery( { provider: "ollama-local", model: "qwen3.6:35b" }, proc, { timeoutSeconds: 1 }, @@ -149,10 +168,10 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("continuing to OpenClaw dispatch"); }); - it("names the endpoint and its reported models when the model is absent (#9455)", () => { + it("names the endpoint and its reported models when the model is absent (#9455)", async () => { const { writes, proc } = makeProcMock(); - runOllamaRestartRecovery( + await runOllamaRestartRecovery( { provider: "ollama-local", model: "gemma4:26b", @@ -178,13 +197,13 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("Ollama was unreachable during the restart check"); }); - it("redacts and bounds recovery exceptions", () => { + it("redacts and bounds recovery exceptions", async () => { const { writes, proc } = makeProcMock(); const exposedToken = "sk-proj-NOT-A-REAL-SECRET-1234567890"; const directionalControls = "\u061c\u200e\u200f\u2028\u2029\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069"; - expect(() => + await expect( runOllamaRestartRecovery( { provider: "ollama-local", @@ -199,7 +218,7 @@ describe("runOllamaRestartRecovery", () => { ); }, ), - ).not.toThrow(); + ).resolves.toBeNull(); const stderr = writes.join(""); expect(stderr).toContain("Ollama restart recovery for 'qwen3.6:35b"); expect(stderr).toContain("at the recorded endpoint http://host.openshell.internal:11434/v1"); @@ -363,6 +382,30 @@ describe("agent passthrough Ollama recovery ordering", () => { expect(diagnostics.join("")).toContain("failed unexpectedly"); }); + it("does not dispatch after Ollama recovery receives SIGTERM", async () => { + const events: string[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }; + const deps = makePassthroughDeps(route, events); + const runRecovery = vi.fn(async () => { + events.push("recovery-cancelled"); + return "SIGTERM" as const; + }); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping"] }, + { ...deps, runOllamaRestartRecovery: runRecovery }, + ), + ).rejects.toThrow("__exit:143"); + + expect(events).toEqual(["recovery-cancelled"]); + }); + it("does not run Ollama recovery for a non-Ollama route", async () => { const events: string[] = []; const deps = makePassthroughDeps( diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 50be11c00a9..71cdc3f1764 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -16,7 +16,7 @@ export { OLLAMA_LOCAL_PROVIDER }; export type OllamaRestartRecoveryFn = ( route: OllamaRestartRecoveryRoute, options?: OllamaRestartRecoveryOptions, -) => OllamaRestartRecoveryResult; +) => OllamaRestartRecoveryResult | Promise; export interface OllamaRestartRecoveryProcess { stderr: { write(s: string): unknown }; @@ -50,7 +50,7 @@ function describeWarmFailure(reason: OllamaRestartRecoveryFailureReason): string function reportRecovery( route: OllamaRestartRecoveryRoute, - result: OllamaRestartRecoveryResult, + result: Exclude, proc: OllamaRestartRecoveryProcess, ): void { const model = boundedOllamaRestartRecoveryDetail(route.model, "the registered model"); @@ -122,16 +122,18 @@ function reportRecovery( } } -/** Run best-effort Ollama recovery without blocking the canonical agent error path. */ -export function runOllamaRestartRecovery( +/** Continue after best-effort recovery failures, but preserve an operator cancellation. */ +export async function runOllamaRestartRecovery( route: OllamaRestartRecoveryRoute, proc: OllamaRestartRecoveryProcess, options: OllamaRestartRecoveryOptions = {}, recoverOllama: OllamaRestartRecoveryFn = maybeWarmOllamaAfterDaemonRestart, -): void { +): Promise { proc.stderr.write(" Checking whether the Ollama model is loaded...\n"); try { - reportRecovery(route, recoverOllama(route, options), proc); + const result = await recoverOllama(route, options); + if (result.kind === "cancelled") return result.signal; + reportRecovery(route, result, proc); } catch (error) { const model = boundedOllamaRestartRecoveryDetail(route.model, "the registered model"); const endpoint = recordedEndpointLabel(route); @@ -142,4 +144,5 @@ export function runOllamaRestartRecovery( `serves '${model}'. NemoClaw will retry the warm-up before the next agent command.\n`, ); } + return null; } diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index c44d0c48c9e..bc32df5873a 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -242,7 +242,9 @@ export interface AgentPassthroughDeps { exec?: typeof execSandbox; execJson?: typeof runAgentJsonPassthrough; execNonJson?: typeof runAgentNonJsonPassthrough; - runOllamaRestartRecovery?: typeof runOllamaRestartRecovery; + runOllamaRestartRecovery?: ( + ...args: Parameters + ) => ReturnType | NodeJS.Signals | null | void; getRecentShieldsAutoRestore?: (sandboxName: string) => ShieldsAutoRestoreReadResult; now?: () => number; process?: { @@ -596,7 +598,14 @@ export async function runAgentPassthrough( if (lookup.kind === "agent" && lookup.provider === OLLAMA_LOCAL_PROVIDER) { const recoverOllama = deps.runOllamaRestartRecovery ?? runOllamaRestartRecovery; const timeoutSeconds = recoveryBudgetSeconds(commandDeadline, now); - recoverOllama(lookup, proc, timeoutSeconds === null ? {} : { timeoutSeconds }); + const recoverySignal = await recoverOllama( + lookup, + proc, + timeoutSeconds === null ? {} : { timeoutSeconds }, + ); + if (recoverySignal) { + return proc.exit(computeExitCode({ status: null, signal: recoverySignal }).code); + } } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); dispatchCommand = commandWithRemainingDeadline(command, commandDeadline, now); diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts index 704d73adf82..03ce15541e4 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.test.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -31,4 +31,59 @@ describe("doctor system checks", () => { hint: "expected host port 19080 for this sandbox gateway", }); }); + + it("probes a loopback Ollama route with direct curl", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => + JSON.stringify({ models: [{ name: "qwen3.6:35b" }] }), + ); + const prepareDockerEnvironment = vi.fn(); + const { ollamaDoctorCheck } = requireDist(modulePath); + + expect( + ollamaDoctorCheck("ollama-local", { + getOllamaHost: () => "127.0.0.1", + runCaptureImpl, + prepareDockerEnvironment, + }), + ).toMatchObject({ + status: "ok", + detail: "reachable at http://127.0.0.1:11434/api/tags (1 model(s))", + }); + expect(runCaptureImpl.mock.calls[0]?.[0]?.[0]).toBe("curl"); + expect(runCaptureImpl.mock.calls[0]?.[0]).toContain("http://127.0.0.1:11434/api/tags"); + expect(prepareDockerEnvironment).not.toHaveBeenCalled(); + }); + + it("probes a persisted Windows Ollama route through credential-free Docker", () => { + const cleanup = vi.fn(() => ({ ok: true as const })); + const prepareDockerEnvironment = vi.fn(() => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })); + const runCaptureImpl = vi.fn( + (command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + command[0] === "docker" && options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const { ollamaDoctorCheck } = requireDist(modulePath); + + expect( + ollamaDoctorCheck("ollama-local", { + getOllamaHost: () => "host.docker.internal", + runCaptureImpl, + prepareDockerEnvironment, + }), + ).toMatchObject({ + status: "ok", + detail: "reachable at http://host.docker.internal:11434/api/tags (0 model(s))", + }); + expect(runCaptureImpl.mock.calls[0]?.[0]?.[0]).toBe("docker"); + expect(runCaptureImpl.mock.calls[0]?.[0]).toContain( + "http://host.docker.internal:11434/api/tags", + ); + expect(prepareDockerEnvironment).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); + }); }); diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index 3ad56d794d7..47dc59037ef 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -2,11 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import path from "node:path"; -import { buildValidatedCurlCommandArgs } from "../../adapters/http/curl-args"; import { stripAnsi } from "../../adapters/openshell/client"; import { CLI_NAME } from "../../cli/branding"; +import { GATEWAY_PORT } from "../../core/ports"; import { gatewayStartGuidance } from "../../gateway-start-guidance"; -import { GATEWAY_PORT, OLLAMA_PORT } from "../../core/ports"; +import { + type OllamaHostInventoryProbeOptions, + probeOllamaHostInventory, +} from "../../inference/health"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, resolveCurrentRuntimeProviderBundle, @@ -169,15 +172,15 @@ export function cloudflaredDoctorCheck(sandboxName: string): DoctorCheck { } } -export function ollamaDoctorCheck(currentProvider: string): DoctorCheck { - const endpoint = `http://127.0.0.1:${OLLAMA_PORT}/api/tags`; - const result = captureHostCommand( - "curl", - buildValidatedCurlCommandArgs(["-sS", "--connect-timeout", "2", "--max-time", "4", endpoint]), - 6000, - ); +export type OllamaDoctorCheckDeps = OllamaHostInventoryProbeOptions; + +export function ollamaDoctorCheck( + currentProvider: string, + deps: OllamaDoctorCheckDeps = {}, +): DoctorCheck { + const { endpoint, output } = probeOllamaHostInventory(deps); const required = currentProvider === "ollama-local"; - if (result.status !== 0) { + if (!output) { return { group: "Local services", label: "Ollama", @@ -189,7 +192,7 @@ export function ollamaDoctorCheck(currentProvider: string): DoctorCheck { let modelCount = "unknown model count"; try { - const parsed = JSON.parse(result.stdout); + const parsed = JSON.parse(output); if (Array.isArray(parsed.models)) modelCount = `${parsed.models.length} model(s)`; } catch { /* keep generic detail */ diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index ccf1b3b17a2..cac14d4847f 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -10,12 +10,20 @@ */ import { createBearerAuthConfig, createXApiKeyAuthConfig } from "../adapters/http/auth-config"; +import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; import type { CurlProbeOptions, CurlProbeResult } from "../adapters/http/probe"; import { runCurlProbe } from "../adapters/http/probe"; import { normalizeCredentialValue, resolveProviderCredential } from "../credentials/store"; import { getProviderSelectionConfig } from "./config"; -import type { LocalProviderHealthProbeOptions } from "./local"; -import { probeLocalProviderHealth } from "./local"; +import { + createOllamaApiCapture, + getResolvedOllamaHost, + loadPersistedOllamaHost, + type LocalProviderHealthProbeOptions, + OLLAMA_PORT, + probeLocalProviderHealth, + type RunCaptureFn, +} from "./local"; import { MIN_PROBE_REPLY_TOKENS } from "./max-tokens-field"; import { getChatCompletionsProbeCurlArgs } from "./onboard-probes"; import { BUILD_ENDPOINT_URL } from "./provider-models"; @@ -54,6 +62,43 @@ export interface ProviderHealthProbeOptions { isWsl?: boolean; } +export type OllamaHostInventoryProbeOptions = { + getOllamaHost?: () => string; + runCaptureImpl?: RunCaptureFn; + prepareDockerEnvironment?: Parameters[2]; +}; + +/** Probe the persisted raw Ollama daemon through its platform-specific host transport. */ +export function probeOllamaHostInventory(options: OllamaHostInventoryProbeOptions = {}): { + endpoint: string; + output: string; +} { + const host = options.getOllamaHost + ? options.getOllamaHost() + : (loadPersistedOllamaHost() ?? getResolvedOllamaHost()); + const endpoint = `http://${host}:${OLLAMA_PORT}/api/tags`; + const capture = createOllamaApiCapture( + options.runCaptureImpl, + host, + options.prepareDockerEnvironment, + ); + const output = capture( + [ + "curl", + ...buildValidatedCurlCommandArgs([ + "-sS", + "--connect-timeout", + "2", + "--max-time", + "4", + endpoint, + ]), + ], + { ignoreError: true, timeout: 6000 }, + ); + return { endpoint, output }; +} + const COMPATIBLE_PROVIDERS = new Set(["compatible-endpoint", "compatible-anthropic-endpoint"]); const NVIDIA_MANAGED_PROVIDERS = new Set(["nvidia-prod", "nvidia-nim"]); const NVIDIA_HEALTH_CREDENTIAL_ENV = "NVIDIA_INFERENCE_API_KEY"; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 152ce6abc7b..860e1c0e5cb 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -1385,8 +1385,9 @@ export function probeOllamaEndpointInventory( host: string, runCaptureImpl?: RunCaptureFn, timeoutMilliseconds = 5_000, + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, ): string[] | null { - const capture = createOllamaApiCapture(runCaptureImpl, host); + const capture = createOllamaApiCapture(runCaptureImpl, host, prepareDockerEnvironment); const normalizedTimeoutMilliseconds = Number.isFinite(timeoutMilliseconds) && timeoutMilliseconds > 0 ? Math.floor(timeoutMilliseconds) diff --git a/src/lib/inference/ollama-runtime-context.ts b/src/lib/inference/ollama-runtime-context.ts index eadc5f8f2b8..6dbbe87b62e 100644 --- a/src/lib/inference/ollama-runtime-context.ts +++ b/src/lib/inference/ollama-runtime-context.ts @@ -163,6 +163,14 @@ export function probeOllamaRuntimeModelStatus( ], { ignoreError: true, timeout: boundedTimeoutMilliseconds }, ); + return parseOllamaRuntimeModelStatus(model, output); +} + +/** Parse one completed Ollama `/api/ps` response without performing I/O. */ +export function parseOllamaRuntimeModelStatus( + model: string, + output: string, +): OllamaRuntimeModelStatus { if (!output) return { probed: false, loaded: false, cpuOnly: false }; try { diff --git a/src/lib/runner.ts b/src/lib/runner.ts index 9b2e135f909..ccba3bb736f 100644 --- a/src/lib/runner.ts +++ b/src/lib/runner.ts @@ -421,6 +421,7 @@ function validateName(name: string, label = "name"): string { export { ROOT, + buildSubprocessEnv, redact, redactFull, run, From 0643151f57f81c354e5a52f492e4f15eab4b9773 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 03:05:20 -0700 Subject: [PATCH 14/48] fix(doctor): reject invalid Ollama inventory Signed-off-by: Prekshi Vyas --- .../sandbox/doctor-system-checks.test.ts | 20 +++++++++++++++++++ .../actions/sandbox/doctor-system-checks.ts | 6 +++--- src/lib/inference/health.ts | 4 +++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts index 03ce15541e4..0331d1c3aa6 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.test.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -86,4 +86,24 @@ describe("doctor system checks", () => { expect(prepareDockerEnvironment).toHaveBeenCalledOnce(); expect(cleanup).toHaveBeenCalledOnce(); }); + + it.each([ + ["malformed JSON", "not-json"], + ["JSON without a models array", JSON.stringify({ status: "ok" })], + ])("rejects %s from the selected Ollama endpoint", (_name, output) => { + const { ollamaDoctorCheck } = requireDist(modulePath); + + expect( + ollamaDoctorCheck("ollama-local", { + getOllamaHost: () => "127.0.0.1", + runCaptureImpl: () => output, + }), + ).toEqual({ + group: "Local services", + label: "Ollama", + status: "fail", + detail: "invalid response from http://127.0.0.1:11434/api/tags", + hint: "start Ollama or change the sandbox inference provider", + }); + }); }); diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index 47dc59037ef..8d4fae1b436 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -178,14 +178,14 @@ export function ollamaDoctorCheck( currentProvider: string, deps: OllamaDoctorCheckDeps = {}, ): DoctorCheck { - const { endpoint, output } = probeOllamaHostInventory(deps); + const { endpoint, output, valid } = probeOllamaHostInventory(deps); const required = currentProvider === "ollama-local"; - if (!output) { + if (!output || !valid) { return { group: "Local services", label: "Ollama", status: required ? "fail" : "info", - detail: `not reachable at ${endpoint}`, + detail: output ? `invalid response from ${endpoint}` : `not reachable at ${endpoint}`, hint: required ? "start Ollama or change the sandbox inference provider" : undefined, }; } diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index cac14d4847f..af541fb1cd8 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -18,6 +18,7 @@ import { getProviderSelectionConfig } from "./config"; import { createOllamaApiCapture, getResolvedOllamaHost, + isValidOllamaTagsResponseBody, loadPersistedOllamaHost, type LocalProviderHealthProbeOptions, OLLAMA_PORT, @@ -72,6 +73,7 @@ export type OllamaHostInventoryProbeOptions = { export function probeOllamaHostInventory(options: OllamaHostInventoryProbeOptions = {}): { endpoint: string; output: string; + valid: boolean; } { const host = options.getOllamaHost ? options.getOllamaHost() @@ -96,7 +98,7 @@ export function probeOllamaHostInventory(options: OllamaHostInventoryProbeOption ], { ignoreError: true, timeout: 6000 }, ); - return { endpoint, output }; + return { endpoint, output, valid: isValidOllamaTagsResponseBody(output) }; } const COMPATIBLE_PROVIDERS = new Set(["compatible-endpoint", "compatible-anthropic-endpoint"]); From 713b22b8ada362909a4dc0b0e8f744dd1070093e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 03:21:27 -0700 Subject: [PATCH 15/48] test(inference): prove Ollama receipt retirement Signed-off-by: Prekshi Vyas --- .../onboard-inference-reconciliation.test.ts | 81 ++++++++++++------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index 9d25955a331..8adc8f63837 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -8,6 +8,11 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { + clearPersistedOllamaHostIfUnused, + loadPersistedOllamaHost, + persistResolvedOllamaHost, +} from "../../src/lib/inference/local.js"; import { createLocalInferenceRouteApplier } from "../../src/lib/onboard/local-inference-route.js"; import type { SetupInference, SetupInferenceDeps } from "../../src/lib/onboard/setup-inference.js"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; @@ -1087,22 +1092,31 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).toHaveBeenCalledWith(["llama3"]); }); - it("retires the final Ollama route receipt after switching providers", async () => { - const clearPersistedOllamaHostIfUnused = vi.fn(() => true); - const harness = releaseHarness({ - getSandbox: () => priorEntry, - sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }], - unloadOllamaModels: vi.fn(), - clearPersistedOllamaHostIfUnused, - }); + it("retires the final Windows-host Ollama route receipt after switching providers", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-provider-switch-ollama-")); + const finalRoutes = [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }]; + const clearReceipt = vi.fn((routes: readonly ReleaseEntry[]) => + clearPersistedOllamaHostIfUnused(routes, stateRoot), + ); + try { + persistResolvedOllamaHost("host.docker.internal", stateRoot); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: finalRoutes, + unloadOllamaModels: vi.fn(), + loadPersistedOllamaHost: () => loadPersistedOllamaHost(stateRoot), + clearPersistedOllamaHostIfUnused: clearReceipt, + }); - await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual({ - ok: true, - }); + await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual( + { ok: true }, + ); - expect(clearPersistedOllamaHostIfUnused).toHaveBeenCalledWith([ - { ...priorEntry, provider: "vllm-local", model: "vllm-model" }, - ]); + expect(clearReceipt).toHaveBeenCalledWith(finalRoutes); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + } finally { + fs.rmSync(stateRoot, { recursive: true, force: true }); + } }); it("keeps the successful route when the superseded model unload fails (#9110)", async () => { @@ -1277,29 +1291,38 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).not.toHaveBeenCalled(); }); - it("keeps the route and shared model for a compatible local Ollama peer", async () => { + it("keeps the Windows-host route and shared model for a compatible local Ollama peer", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-provider-switch-peer-")); const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); - const clearPersistedOllamaHostIfUnused = vi.fn(() => true); + const clearReceipt = vi.fn((routes: readonly ReleaseEntry[]) => + clearPersistedOllamaHostIfUnused(routes, stateRoot), + ); const peer: ReleaseEntry = { name: "peer", provider: "compatible-endpoint", model: "llama3:latest", - endpointUrl: "http://127.0.0.1:11434/v1", + endpointUrl: "http://host.docker.internal:11434/v1", }; - const harness = releaseHarness({ - getSandbox: () => priorEntry, - sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }, peer], - unloadOllamaModels, - loadPersistedOllamaHost: () => "127.0.0.1", - clearPersistedOllamaHostIfUnused, - }); + try { + persistResolvedOllamaHost("host.docker.internal", stateRoot); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }, peer], + unloadOllamaModels, + loadPersistedOllamaHost: () => loadPersistedOllamaHost(stateRoot), + clearPersistedOllamaHostIfUnused: clearReceipt, + }); - await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual({ - ok: true, - }); + await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual( + { ok: true }, + ); - expect(unloadOllamaModels).not.toHaveBeenCalled(); - expect(clearPersistedOllamaHostIfUnused).not.toHaveBeenCalled(); + expect(unloadOllamaModels).not.toHaveBeenCalled(); + expect(clearReceipt).not.toHaveBeenCalled(); + expect(loadPersistedOllamaHost(stateRoot)).toBe("host.docker.internal"); + } finally { + fs.rmSync(stateRoot, { recursive: true, force: true }); + } }); it("reads the prior route and releases the model inside the sandbox mutation lock (#9110)", async () => { From 3def062fc5ec686a61fe841ab094e309f75fdf49 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 03:48:20 -0700 Subject: [PATCH 16/48] fix(ollama): consolidate isolated recovery probes --- .../agent/ollama-restart-recovery.test.ts | 380 ++++++++---------- .../sandbox/agent/ollama-restart-recovery.ts | 145 +++---- src/lib/onboard/provider-host-state.test.ts | 106 ++--- src/lib/onboard/provider-host-state.ts | 38 +- 4 files changed, 308 insertions(+), 361 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index c28206ee7bb..52f30a83e92 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -14,20 +14,23 @@ import { type OllamaRestartRecoveryDeps, } from "./ollama-restart-recovery"; -const unloadedStatus = { - probed: true, - loaded: false, - cpuOnly: false, -}; - -function successfulWarmResult() { +function successfulRecoveryResult(stdout: string) { return { - stdout: JSON.stringify({ response: "Hello!", done: true }), + stdout, + stderr: "", exitCode: 0, timedOut: false, }; } +function unloadedStatusResult() { + return successfulRecoveryResult(JSON.stringify({ models: [] })); +} + +function successfulWarmResult() { + return successfulRecoveryResult(JSON.stringify({ response: "Hello!", done: true })); +} + function getCommandUrl(command: readonly string[]): string { return command.find((arg) => arg.startsWith("http://")) ?? ""; } @@ -51,24 +54,16 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("uses the persisted direct bridge route for both the default probe and warm-up", async () => { - const cleanup = vi.fn(() => ({ ok: true as const })); + it("uses the persisted direct bridge route for both async recovery requests", async () => { const prepareDockerEnvironment = () => ({ env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, isolatedCredentialConfig: true, - cleanup, + cleanup: () => ({ ok: true as const }), }); - const runCaptureImpl = vi.fn( - (_command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => - options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" - ? JSON.stringify({ models: [] }) - : "", - ); - const runCaptureExImpl = vi.fn((_command: string[], options?: { env?: NodeJS.ProcessEnv }) => - options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" - ? successfulWarmResult() - : { stdout: "", exitCode: 1, timedOut: false }, - ); + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulWarmResult()); await expect( maybeWarmOllamaAfterDaemonRestart( @@ -78,31 +73,31 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, { - runCaptureImpl, - runCaptureExImpl, + runRecoveryCaptureImpl, prepareDockerEnvironment, }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, ); - expect(runCaptureImpl.mock.calls[0][0][0]).toBe("docker"); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); - expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("docker"); - expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({ + expect(getCommandBody(runRecoveryCaptureImpl.mock.calls[1][0])).toMatchObject({ model: "qwen3.6:35b", stream: false, think: false, }); - expect(runCaptureImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe("/tmp/credential-free-docker"); - expect(runCaptureExImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe( - "/tmp/credential-free-docker", - ); - expect(cleanup).toHaveBeenCalledTimes(2); + expect(runRecoveryCaptureImpl.mock.calls[0][1]).toMatchObject({ + host: "host.docker.internal", + prepareDockerEnvironment, + }); + expect(runRecoveryCaptureImpl.mock.calls[1][1]).toMatchObject({ + host: "host.docker.internal", + prepareDockerEnvironment, + }); }); it("runs the production status probe and warm-up through the async capture boundary", async () => { @@ -149,32 +144,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); - it("prepares and cleans one Docker environment for each recovery request", async () => { - const cleanups: Array> = []; - const prepareDockerEnvironment = vi.fn(() => { - const cleanup = vi.fn(() => ({ ok: true as const })); - cleanups.push(cleanup); - return { - env: { DOCKER_CONFIG: `/tmp/credential-free-docker-${cleanups.length}` }, - isolatedCredentialConfig: true, - cleanup, - }; - }); - const commands: string[][] = []; - const runCaptureImpl = vi.fn((command: readonly string[]) => { - commands.push([...command]); - return getCommandUrl(command).endsWith("/api/ps") - ? JSON.stringify({ models: [] }) - : JSON.stringify({ models: [{ name: "llama3.2:1b" }] }); - }); - const runCaptureExImpl = vi.fn((command: string[]) => { - commands.push([...command]); - return { - stdout: JSON.stringify({ error: "model not found" }), - exitCode: 0, - timedOut: false, - }; - }); + it("uses one async capture seam for status, warm-up, and inventory", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulRecoveryResult(JSON.stringify({ error: "model not found" }))) + .mockResolvedValueOnce( + successfulRecoveryResult(JSON.stringify({ models: [{ name: "llama3.2:1b" }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( @@ -183,21 +160,15 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { runCaptureImpl, runCaptureExImpl, prepareDockerEnvironment }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "skipped", reason: "model-absent" }); - expect(commands.map(getCommandUrl)).toEqual([ + expect(runRecoveryCaptureImpl.mock.calls.map(([command]) => getCommandUrl(command))).toEqual([ `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ]); - expect(commands.every((command) => command[0] === "docker")).toBe(true); - expect(prepareDockerEnvironment).toHaveBeenCalledTimes(3); - expect(cleanups).toHaveLength(3); - expect(cleanups[0]).toHaveBeenCalledOnce(); - expect(cleanups[1]).toHaveBeenCalledOnce(); - expect(cleanups[2]).toHaveBeenCalledOnce(); }); it("forwards SIGTERM to an active recovery child and releases its Docker environment", async () => { @@ -261,8 +232,10 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("maps an auth-proxy route back to host loopback", async () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulWarmResult()); await maybeWarmOllamaAfterDaemonRestart( { @@ -270,22 +243,22 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PROXY_PORT}/v1`, }, - { runCaptureImpl, runCaptureExImpl }, + { runRecoveryCaptureImpl }, ); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0][0])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, ); - expect(runCaptureImpl.mock.calls[0][0][0]).toBe("curl"); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1][0])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, ); - expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("curl"); }); it("falls back to an allowlisted host instead of probing an arbitrary registry URL", async () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulWarmResult()); await maybeWarmOllamaAfterDaemonRestart( { @@ -295,18 +268,19 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }, { getOllamaHost: () => "also.example.com", - runCaptureImpl, - runCaptureExImpl, + runRecoveryCaptureImpl, }, ); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1][0])).toContain("http://127.0.0.1:"); }); it("does not map an unrecognized proxy-port host to host loopback (#6039)", async () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulWarmResult()); await maybeWarmOllamaAfterDaemonRestart( { @@ -316,80 +290,85 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }, { getOllamaHost: () => "host.docker.internal", - runCaptureImpl, - runCaptureExImpl, + runRecoveryCaptureImpl, }, ); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, ); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); }); it("skips the warm-up when the selected model is already loaded", async () => { - const probeRuntimeModelStatus = vi.fn(() => ({ - probed: true, - loaded: true, - cpuOnly: false, - })); - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValue( + successfulRecoveryResult(JSON.stringify({ models: [{ name: "qwen3.6:35b" }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { probeRuntimeModelStatus, runCaptureExImpl }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", reason: "already-loaded" }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("skips the warm-up when the daemon probe is unreachable", async () => { - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const runRecoveryCaptureImpl = vi.fn().mockResolvedValue({ + stdout: "", + stderr: "connection refused", + exitCode: 7, + timedOut: false, + }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { runCaptureImpl: () => "", runCaptureExImpl }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", reason: "unreachable", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("skips the warm-up when the daemon status response is malformed", async () => { - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const runRecoveryCaptureImpl = vi.fn().mockResolvedValue(successfulRecoveryResult("not-json")); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { runCaptureImpl: () => "not-json", runCaptureExImpl }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", reason: "unreachable", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("reports a bounded warm-up timeout", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce({ + stdout: "", + stderr: "", + exitCode: 28, + timedOut: true, + }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ - stdout: "", - exitCode: 28, - timedOut: true, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", @@ -402,52 +381,43 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { it("limits warm-up to the command timeout budget remaining after the probe", async () => { let nowMs = 1_000; - const runCaptureExImpl = vi.fn( - (_command: string[], _options?: { env?: NodeJS.ProcessEnv; timeout?: number }) => - successfulWarmResult(), - ); - const probeRuntimeModelStatus = vi.fn(() => { - nowMs = 6_000; - return unloadedStatus; - }); + const runRecoveryCaptureImpl = vi + .fn() + .mockImplementationOnce(async () => { + nowMs = 6_000; + return unloadedStatusResult(); + }) + .mockResolvedValueOnce(successfulWarmResult()); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { - probeRuntimeModelStatus, - runCaptureExImpl, + runRecoveryCaptureImpl, timeoutSeconds: 30, now: () => nowMs, }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); - const warmCommand = runCaptureExImpl.mock.calls[0][0]; + const warmCommand = runRecoveryCaptureImpl.mock.calls[1][0]; expect(warmCommand[warmCommand.indexOf("--max-time") + 1]).toBe("25"); - expect(runCaptureExImpl.mock.calls[0][1]?.timeout).toBe(25_000); - expect(probeRuntimeModelStatus).toHaveBeenCalledWith( - "qwen3.6:35b", - expect.any(Function), - expect.any(Function), - 5_000, - ); + expect(runRecoveryCaptureImpl.mock.calls[0][1]?.timeoutMilliseconds).toBe(5_000); + expect(runRecoveryCaptureImpl.mock.calls[1][1]?.timeoutMilliseconds).toBe(25_000); }); it("skips warm-up when the probe consumes the command timeout budget", async () => { let nowMs = 1_000; - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); - const probeRuntimeModelStatus = vi.fn(() => { + const runRecoveryCaptureImpl = vi.fn().mockImplementationOnce(async () => { nowMs = 31_000; - return unloadedStatus; + return unloadedStatusResult(); }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { - probeRuntimeModelStatus, - runCaptureExImpl, + runRecoveryCaptureImpl, timeoutSeconds: 30, now: () => nowMs, }, @@ -457,47 +427,44 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { reason: "deadline-exhausted", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("bounds the daemon probe and skips warm-up when a short timeout is consumed", async () => { let nowMs = 1_000; - const probeRuntimeModelStatus = vi.fn(() => { + const runRecoveryCaptureImpl = vi.fn().mockImplementationOnce(async () => { nowMs = 3_000; - return unloadedStatus; + return unloadedStatusResult(); }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { probeRuntimeModelStatus, timeoutSeconds: 2, now: () => nowMs }, + { runRecoveryCaptureImpl, timeoutSeconds: 2, now: () => nowMs }, ), ).resolves.toEqual({ kind: "skipped", reason: "deadline-exhausted", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(probeRuntimeModelStatus).toHaveBeenCalledWith( - "qwen3.6:35b", - expect.any(Function), - expect.any(Function), - 2_000, - ); + expect(runRecoveryCaptureImpl.mock.calls[0][1]?.timeoutMilliseconds).toBe(2_000); }); it("does not treat an exit-zero Ollama error body as a successful warm-up", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulRecoveryResult(JSON.stringify({ error: "model not found" }))) + .mockResolvedValueOnce({ + stdout: "", + stderr: "inventory unavailable", + exitCode: 7, + timedOut: false, + }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "missing:latest" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory: () => null, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "model not found" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -509,20 +476,17 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("keeps the warm-up error when the inventory probe throws", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce( + successfulRecoveryResult(JSON.stringify({ error: "runner stopped unexpectedly" })), + ) + .mockRejectedValueOnce(new Error("inventory unavailable")); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory: () => { - throw new Error("inventory unavailable"); - }, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "runner stopped unexpectedly" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -533,7 +497,13 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", async () => { - const probeModelInventory = vi.fn(() => ["llama3.2:1b"]); + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulRecoveryResult(JSON.stringify({ error: "model not found" }))) + .mockResolvedValueOnce( + successfulRecoveryResult(JSON.stringify({ models: [{ name: "llama3.2:1b" }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( @@ -542,15 +512,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "gemma4:26b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "model not found" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", @@ -558,27 +520,29 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, inventoryLabel: "llama3.2:1b", }); - expect(probeModelInventory).toHaveBeenCalledWith( - "host.docker.internal", - undefined, - 5_000, - undefined, + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[2][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ); + expect(runRecoveryCaptureImpl.mock.calls[2][1]).toMatchObject({ + host: "host.docker.internal", + timeoutMilliseconds: 5_000, + }); }); it("keeps the warm failure when the daemon does hold the model (#9455)", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce( + successfulRecoveryResult(JSON.stringify({ error: "runner stopped unexpectedly" })), + ) + .mockResolvedValueOnce( + successfulRecoveryResult(JSON.stringify({ models: [{ name: "qwen3.6:35b" }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory: () => ["qwen3.6:35b"], - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "runner stopped unexpectedly" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -590,17 +554,18 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("accepts a completed thinking-only response from a thinking model", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce( + successfulRecoveryResult( + JSON.stringify({ response: "", thinking: "The model is ready.", done: true }), + ), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ response: "", thinking: "The model is ready.", done: true }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); }); @@ -611,13 +576,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ["missing done marker", JSON.stringify({ response: "Hello!" })], ["empty response", JSON.stringify({ response: "", done: true })], ])("rejects an invalid warm response: %s", async (_name, stdout) => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce(successfulRecoveryResult(stdout)); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -628,13 +594,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("reports a non-zero warm command exit", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockResolvedValueOnce({ stdout: "", stderr: "", exitCode: 7, timedOut: false }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", @@ -646,11 +613,12 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("reports a warm process spawn failure without throwing", async () => { + const runRecoveryCaptureImpl = vi + .fn() + .mockResolvedValueOnce(unloadedStatusResult()) + .mockRejectedValueOnce(new Error("spawn failed")); const deps: OllamaRestartRecoveryDeps = { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => { - throw new Error("spawn failed"); - }, + runRecoveryCaptureImpl, }; await expect( diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 637dace2eeb..f1c0313e73a 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -20,22 +20,16 @@ import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args" import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, - createOllamaApiCapture, - createOllamaApiCaptureEx, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, parseOllamaModelInventory, prepareOllamaApiExecution, - probeOllamaEndpointInventory, - type RunCaptureFn, - type RunCaptureExFn, } from "../../../inference/local"; import { type OllamaRuntimeModelStatus, parseOllamaRuntimeModelStatus, - probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; import { buildSubprocessEnv, redact, redactFull } from "../../../runner"; import type { SandboxExecSignalSource } from "../exec"; @@ -45,6 +39,10 @@ import { runAgentDispatch, } from "./passthrough-dispatch"; +type PrepareOllamaDockerEnvironment = NonNullable< + Parameters[2] +>["prepareDockerEnvironment"]; + export interface OllamaRestartRecoveryRoute { provider?: string | null; model?: string | null; @@ -56,22 +54,8 @@ export interface OllamaRestartRecoveryOptions { } export interface OllamaRestartRecoveryDeps extends OllamaRestartRecoveryOptions { - probeRuntimeModelStatus?: ( - model: string, - getOllamaHost: () => string, - runCaptureImpl?: RunCaptureFn, - timeoutMilliseconds?: number, - ) => OllamaRuntimeModelStatus; - probeModelInventory?: ( - host: string, - runCaptureImpl?: RunCaptureFn, - timeoutMilliseconds?: number, - prepareDockerEnvironment?: Parameters[2], - ) => string[] | null; - runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; - runCaptureImpl?: RunCaptureFn; - prepareDockerEnvironment?: Parameters[2]; + prepareDockerEnvironment?: PrepareOllamaDockerEnvironment; runRecoveryCaptureImpl?: OllamaRecoveryCaptureFn; signalSource?: SandboxExecSignalSource; spawnRecoveryChild?: OllamaRecoverySpawner; @@ -186,7 +170,7 @@ export type OllamaRecoveryCaptureFn = ( options: { host: string; timeoutMilliseconds: number; - prepareDockerEnvironment?: Parameters[2]; + prepareDockerEnvironment?: PrepareOllamaDockerEnvironment; signalSource?: SandboxExecSignalSource; spawnRecoveryChild?: OllamaRecoverySpawner; }, @@ -376,7 +360,6 @@ export async function maybeWarmOllamaAfterDaemonRestart( } const now = deps.now ?? Date.now; const recoveryDeadline = recoveryDeadlineMilliseconds(deps.timeoutSeconds, now); - const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; const probeBudgetMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); if (probeBudgetMilliseconds === 0) { return { kind: "skipped", reason: "deadline-exhausted", endpoint: rawEndpoint }; @@ -387,33 +370,24 @@ export async function maybeWarmOllamaAfterDaemonRestart( OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, probeBudgetMilliseconds, ); - if (deps.probeRuntimeModelStatus || deps.runCaptureImpl) { - const rawCapture = createOllamaApiCapture( - deps.runCaptureImpl, - rawHost, - deps.prepareDockerEnvironment, - ); - status = probe(model, () => rawHost, rawCapture, statusTimeoutMilliseconds); - } else { - const capture = deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture; - const result = await capture( - buildOllamaProbeCommand(rawHost, "/api/ps", statusTimeoutMilliseconds), - { - host: rawHost, - timeoutMilliseconds: statusTimeoutMilliseconds, - prepareDockerEnvironment: deps.prepareDockerEnvironment, - signalSource: deps.signalSource, - spawnRecoveryChild: deps.spawnRecoveryChild, - }, - ); - if (result.signal && !result.timedOut) { - return { kind: "cancelled", signal: result.signal }; - } - status = - result.exitCode === 0 && !result.error - ? parseOllamaRuntimeModelStatus(model, result.stdout) - : { probed: false, loaded: false, cpuOnly: false }; + const capture = deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture; + const result = await capture( + buildOllamaProbeCommand(rawHost, "/api/ps", statusTimeoutMilliseconds), + { + host: rawHost, + timeoutMilliseconds: statusTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }, + ); + if (result.signal && !result.timedOut) { + return { kind: "cancelled", signal: result.signal }; } + status = + result.exitCode === 0 && !result.error + ? parseOllamaRuntimeModelStatus(model, result.stdout) + : { probed: false, loaded: false, cpuOnly: false }; } catch { return { kind: "skipped", reason: "unreachable", endpoint: rawEndpoint }; } @@ -432,22 +406,15 @@ export async function maybeWarmOllamaAfterDaemonRestart( try { const command = buildWarmCommand(model, rawHost, warmupTimeoutSeconds); - const result = deps.runCaptureExImpl - ? createOllamaApiCaptureEx( - deps.runCaptureExImpl, - rawHost, - deps.prepareDockerEnvironment, - )(command, { timeout: warmupTimeoutMilliseconds }) - : await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)(command, { - host: rawHost, - timeoutMilliseconds: warmupTimeoutMilliseconds, - prepareDockerEnvironment: deps.prepareDockerEnvironment, - signalSource: deps.signalSource, - spawnRecoveryChild: deps.spawnRecoveryChild, - }); - const asyncResult = result as Partial; - if (asyncResult.signal && !result.timedOut) { - return { kind: "cancelled", signal: asyncResult.signal }; + const result = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)(command, { + host: rawHost, + timeoutMilliseconds: warmupTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }); + if (result.signal && !result.timedOut) { + return { kind: "cancelled", signal: result.signal }; } if (result.timedOut) { return { @@ -461,16 +428,13 @@ export async function maybeWarmOllamaAfterDaemonRestart( ), }; } - if (asyncResult.error) { + if (result.error) { return { kind: "warmed", ok: false, reason: "spawn-failed", endpoint: rawEndpoint, - detail: boundedOllamaRestartRecoveryDetail( - asyncResult.error, - "warm-up process could not start", - ), + detail: boundedOllamaRestartRecoveryDetail(result.error, "warm-up process could not start"), }; } if (result.exitCode !== 0) { @@ -500,32 +464,23 @@ export async function maybeWarmOllamaAfterDaemonRestart( OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, inventoryBudgetMilliseconds, ); - if (deps.probeModelInventory || deps.runCaptureImpl) { - inventory = (deps.probeModelInventory ?? probeOllamaEndpointInventory)( - rawHost, - deps.runCaptureImpl, - inventoryTimeoutMilliseconds, - deps.prepareDockerEnvironment, - ); - } else { - const inventoryResult = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)( - buildOllamaProbeCommand(rawHost, "/api/tags", inventoryTimeoutMilliseconds), - { - host: rawHost, - timeoutMilliseconds: inventoryTimeoutMilliseconds, - prepareDockerEnvironment: deps.prepareDockerEnvironment, - signalSource: deps.signalSource, - spawnRecoveryChild: deps.spawnRecoveryChild, - }, - ); - if (inventoryResult.signal && !inventoryResult.timedOut) { - return { kind: "cancelled", signal: inventoryResult.signal }; - } - inventory = - inventoryResult.exitCode === 0 && !inventoryResult.error - ? parseOllamaModelInventory(inventoryResult.stdout) - : null; + const inventoryResult = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)( + buildOllamaProbeCommand(rawHost, "/api/tags", inventoryTimeoutMilliseconds), + { + host: rawHost, + timeoutMilliseconds: inventoryTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }, + ); + if (inventoryResult.signal && !inventoryResult.timedOut) { + return { kind: "cancelled", signal: inventoryResult.signal }; } + inventory = + inventoryResult.exitCode === 0 && !inventoryResult.error + ? parseOllamaModelInventory(inventoryResult.stdout) + : null; } catch { // Inventory only refines the original warm-up error. } diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 911cdcf39b8..274467d042a 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -13,6 +13,7 @@ import { const WINDOWS_OLLAMA_TAGS_URL = "http://host.docker.internal:11434/api/tags"; const VALID_OLLAMA_TAGS_BODY = '{"models": [{"name": "llama3.2:latest"}]}'; +const ISOLATED_DOCKER_CONFIG = "/tmp/nemoclaw-provider-host-state-docker"; const SUPPORTED_WINDOWS_OLLAMA = { supported: true, @@ -43,10 +44,28 @@ function buildDeps( detectVllmProfile: vi.fn(() => null), getLocalProviderAvailabilityEndpoint: vi.fn(() => "http://127.0.0.1:8000/v1/models"), detectLocalTcpListener: vi.fn(() => null), + prepareDockerEnvironment: vi.fn(() => ({ + env: { DOCKER_CONFIG: ISOLATED_DOCKER_CONFIG }, + isolatedCredentialConfig: true, + cleanup: () => ({ ok: true as const }), + })), ...overrides, }; } +function windowsProbeRunCapture( + tagsBody: string, + networkingMode = "", +): DetectInferenceProviderHostStateDeps["runCapture"] { + return vi.fn((command) => + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL + ? tagsBody + : command.join(" ").includes("wslinfo --networking-mode") + ? networkingMode + : "", + ); +} + function detectWithDeps( deps: DetectInferenceProviderHostStateDeps, gpu: InferenceProviderHostGpu | null = null, @@ -189,8 +208,15 @@ describe("detectInferenceProviderHostState", () => { it("detects Windows-host Ollama from Docker Desktop when WSL cannot reach it (#8127)", () => { const logs: string[] = []; - const dockerCapture = vi.fn((command: string[]) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", + const cleanup = vi.fn(() => ({ ok: true as const })); + const runCapture = vi.fn( + (command, options) => + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL && + options?.env?.DOCKER_CONFIG === ISOLATED_DOCKER_CONFIG + ? VALID_OLLAMA_TAGS_BODY + : command.join(" ").includes("wslinfo --networking-mode") + ? "nat\n" + : "", ); const deps = buildDeps({ isWsl: vi.fn(() => true), @@ -200,12 +226,12 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Users\\me\\AppData\\Local\\Programs\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => { - const joined = command.join(" "); - if (joined.includes("wslinfo --networking-mode")) return "nat\n"; - return ""; + runCapture, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: ISOLATED_DOCKER_CONFIG }, + isolatedCredentialConfig: true, + cleanup, }), - dockerCapture, }); const state = detectInferenceProviderHostState({ @@ -225,8 +251,9 @@ describe("detectInferenceProviderHostState", () => { expect(state.winOllamaInstalledPath).toMatch(/ollama\.exe$/); expect(logs.join("\n")).toContain("Ollama is running on both WSL and the Windows host"); expect(deps.getWindowsHostOllamaDockerRequirement).toHaveBeenCalledWith("docker-desktop"); - expect(dockerCapture).toHaveBeenCalledWith( + expect(runCapture).toHaveBeenCalledWith( [ + "docker", "run", "--rm", "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", @@ -237,8 +264,12 @@ describe("detectInferenceProviderHostState", () => { "5", WINDOWS_OLLAMA_TAGS_URL, ], - { ignoreError: true }, + { + env: { DOCKER_CONFIG: ISOLATED_DOCKER_CONFIG }, + ignoreError: true, + }, ); + expect(cleanup).toHaveBeenCalledOnce(); }); it("keeps WSL-local install available when Docker Desktop cannot reach Windows-host Ollama (#8199)", () => { @@ -279,7 +310,7 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Users\\me\\AppData\\Local\\Programs\\Ollama\\ollama.exe", loopbackOnly: false, })), - dockerCapture: vi.fn(() => body), + runCapture: windowsProbeRunCapture(body), }); const state = detectWithDeps(deps); @@ -290,10 +321,10 @@ describe("detectInferenceProviderHostState", () => { }); it("does not run the Windows-host probe without Docker Desktop WSL integration (#8127)", () => { - const dockerCapture = vi.fn(() => "{}"); + const runCapture = vi.fn(() => "{}"); const deps = buildDeps({ isWsl: vi.fn(() => true), - dockerCapture, + runCapture, getWindowsHostOllamaDockerRequirement: vi.fn(() => getWindowsHostOllamaDockerRequirement("docker"), ), @@ -302,7 +333,10 @@ describe("detectInferenceProviderHostState", () => { const state = detectWithDeps(deps); expect(state.windowsOllamaReachable).toBe(false); - expect(dockerCapture).not.toHaveBeenCalled(); + expect(runCapture).not.toHaveBeenCalledWith( + expect.arrayContaining([WINDOWS_OLLAMA_TAGS_URL]), + expect.anything(), + ); }); it("passes injected platform and env through WSL detection", () => { @@ -335,14 +369,7 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => { - const joined = command.join(" "); - if (joined.includes("wslinfo --networking-mode")) return "mirrored\n"; - return ""; - }), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", - ), + runCapture: windowsProbeRunCapture(VALID_OLLAMA_TAGS_BODY, "mirrored\n"), detectLocalTcpListener: vi.fn(() => false), }); @@ -374,12 +401,7 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => - command.join(" ").includes("wslinfo --networking-mode") ? "mirrored\n" : "", - ), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", - ), + runCapture: windowsProbeRunCapture(VALID_OLLAMA_TAGS_BODY, "mirrored\n"), detectLocalTcpListener: vi.fn(() => true), }); @@ -410,12 +432,7 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => - command.join(" ").includes("wslinfo --networking-mode") ? "mirrored\n" : "", - ), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", - ), + runCapture: windowsProbeRunCapture(VALID_OLLAMA_TAGS_BODY, "mirrored\n"), detectLocalTcpListener: vi.fn(() => null), }); @@ -435,12 +452,7 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => - command.join(" ").includes("wslinfo --networking-mode") ? "future-mode\n" : "", - ), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", - ), + runCapture: windowsProbeRunCapture(VALID_OLLAMA_TAGS_BODY, "future-mode\n"), detectLocalTcpListener, }); @@ -463,7 +475,6 @@ describe("detectInferenceProviderHostState", () => { it("probes Docker reachability when WSL can reach Windows-host Ollama (#10100)", () => { const runCapture = vi.fn(() => ""); - const dockerCapture = vi.fn(() => ""); const deps = buildDeps({ isWsl: vi.fn(() => true), findReachableOllamaHost: vi.fn(() => "host.docker.internal"), @@ -473,23 +484,20 @@ describe("detectInferenceProviderHostState", () => { loopbackOnly: true, })), runCapture, - dockerCapture, }); const state = detectWithDeps(deps); expect(state.isWindowsHostOllama).toBe(true); expect(state.windowsOllamaReachable).toBe(false); - expect(dockerCapture).toHaveBeenCalledWith( + expect(runCapture).toHaveBeenCalledWith( expect.arrayContaining([WINDOWS_OLLAMA_TAGS_URL]), - { ignoreError: true }, + expect.objectContaining({ ignoreError: true }), ); }); it("reuses Windows-host Ollama only after Docker reachability succeeds (#10100)", () => { - const dockerCapture = vi.fn( - (command) => (command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : ""), - ); + const runCapture = vi.fn(windowsProbeRunCapture(VALID_OLLAMA_TAGS_BODY)); const deps = buildDeps({ isWsl: vi.fn(() => true), findReachableOllamaHost: vi.fn(() => "host.docker.internal"), @@ -498,16 +506,16 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - dockerCapture, + runCapture, }); const state = detectWithDeps(deps); expect(state.isWindowsHostOllama).toBe(true); expect(state.windowsOllamaReachable).toBe(true); - expect(dockerCapture).toHaveBeenCalledWith( + expect(runCapture).toHaveBeenCalledWith( expect.arrayContaining([WINDOWS_OLLAMA_TAGS_URL]), - { ignoreError: true }, + expect.objectContaining({ ignoreError: true }), ); }); }); diff --git a/src/lib/onboard/provider-host-state.ts b/src/lib/onboard/provider-host-state.ts index fc9457241a4..d6c46c0b469 100644 --- a/src/lib/onboard/provider-host-state.ts +++ b/src/lib/onboard/provider-host-state.ts @@ -5,13 +5,14 @@ import fs from "node:fs"; import { dockerCapture as defaultDockerCapture } from "../adapters/docker"; import { + createOllamaApiCapture, findReachableOllamaHost, getLocalProviderAvailabilityEndpoint, - getWindowsHostOllamaDockerReachabilityArgs, isLocalProviderProbeOutputHealthy, isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_PORT, + type RunCaptureFn, } from "../inference/local"; import type { NvidiaPlatform } from "../inference/nim"; import { detectVllmProfile, type VllmProfile } from "../inference/vllm"; @@ -32,11 +33,8 @@ import { type OllamaInstallMenuResult, resolveOllamaInstallMenuEntry } from "./o import { buildVllmMenuEntries, type VllmMenuEntry } from "./vllm-menu"; import { detectWindowsHostOllama, type WindowsHostOllamaState } from "./windows-host-ollama"; -type RunCapture = (args: string[], options?: { ignoreError?: boolean }) => string; -type DockerCapture = ( - args: string[], - options?: { env?: NodeJS.ProcessEnv; ignoreError?: boolean; timeout?: number }, -) => string; +type RunCapture = RunCaptureFn; +type DockerCapture = RunCaptureFn; type ReadTextFile = (filePath: string) => string | null; export interface InferenceProviderHostGpu { @@ -92,6 +90,7 @@ export interface DetectInferenceProviderHostStateDeps { detectVllmProfile: (gpu: InferenceProviderHostGpu | null | undefined) => VllmProfile | null; getLocalProviderAvailabilityEndpoint: (provider: string) => string | null; detectLocalTcpListener: (port: number) => boolean | null; + prepareDockerEnvironment?: Parameters[2]; } const LOCAL_PROVIDER_PROBE_CURL_ARGS = ["--connect-timeout", "2", "--max-time", "5"] as const; @@ -164,6 +163,7 @@ function buildDeps( getLocalProviderAvailabilityEndpoint: overrides.getLocalProviderAvailabilityEndpoint ?? getLocalProviderAvailabilityEndpoint, detectLocalTcpListener: overrides.detectLocalTcpListener ?? detectLocalTcpListener, + prepareDockerEnvironment: overrides.prepareDockerEnvironment, }; } @@ -190,15 +190,30 @@ function probeVllmRunning(deps: DetectInferenceProviderHostStateDeps): boolean { function probeWindowsOllamaReachable(input: { isWsl: boolean; dockerRequirementSupported: boolean; - dockerCapture: DockerCapture; + runCapture: RunCapture; + prepareDockerEnvironment?: Parameters[2]; }): boolean { if (!input.isWsl || !input.dockerRequirementSupported) return false; // A successful Docker run is not enough: a captive proxy, a stale listener, or // a stub on host.docker.internal can all answer with arbitrary 2xx bodies. Only // a body in the Ollama `/api/tags` wire format proves the Windows daemon is live. - const body = input.dockerCapture(getWindowsHostOllamaDockerReachabilityArgs(), { - ignoreError: true, - }); + const capture = createOllamaApiCapture( + input.runCapture, + OLLAMA_HOST_DOCKER_INTERNAL, + input.prepareDockerEnvironment, + ); + const body = capture( + [ + "curl", + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + `http://${OLLAMA_HOST_DOCKER_INTERNAL}:${OLLAMA_PORT}/api/tags`, + ], + { ignoreError: true }, + ); return isValidOllamaTagsResponseBody(body); } @@ -263,7 +278,8 @@ export function detectInferenceProviderHostState( : probeWindowsOllamaReachable({ isWsl, dockerRequirementSupported: windowsHostOllamaDockerRequirement.supported, - dockerCapture: deps.dockerCapture, + runCapture: deps.runCapture, + prepareDockerEnvironment: deps.prepareDockerEnvironment, }); const wslNetworkingMode = From b7645e417649204576748cf5194a1c8414bbb13b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 04:02:31 -0700 Subject: [PATCH 17/48] fix(inference): unify Windows Ollama probes Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 390 +++++++++--------- .../sandbox/agent/ollama-restart-recovery.ts | 145 +++---- src/lib/onboard/provider-host-state.test.ts | 112 +++-- src/lib/onboard/provider-host-state.ts | 32 +- 4 files changed, 334 insertions(+), 345 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index c28206ee7bb..aee5388a149 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -11,23 +11,82 @@ import type { AgentDispatchChild } from "./passthrough-dispatch"; import { maybeWarmOllamaAfterDaemonRestart, runOllamaRecoveryCapture, + type OllamaRecoveryCaptureFn, + type OllamaRecoveryCaptureResult, + type OllamaRecoverySpawner, type OllamaRestartRecoveryDeps, } from "./ollama-restart-recovery"; -const unloadedStatus = { - probed: true, - loaded: false, - cpuOnly: false, -}; - -function successfulWarmResult() { +function recoveryResult( + stdout: string, + overrides: Partial = {}, +): OllamaRecoveryCaptureResult { return { - stdout: JSON.stringify({ response: "Hello!", done: true }), + stdout, + stderr: "", exitCode: 0, timedOut: false, + ...overrides, }; } +function successfulWarmResult(): OllamaRecoveryCaptureResult { + return recoveryResult(JSON.stringify({ response: "Hello!", done: true })); +} + +function unloadedProbeResult(): OllamaRecoveryCaptureResult { + return recoveryResult(JSON.stringify({ models: [] })); +} + +function scriptedRecoveryCapture( + ...responses: OllamaRecoveryCaptureResult[] +): ReturnType> { + const pending = [...responses]; + return vi.fn(async () => + Promise.resolve(pending.shift() ?? Promise.reject(new Error("unexpected recovery request"))), + ); +} + +function failingRecoveryCapture( + error: Error, + ...responses: OllamaRecoveryCaptureResult[] +): ReturnType> { + const pending = [ + ...responses.map((response) => () => Promise.resolve(response)), + () => Promise.reject(error), + ]; + return vi.fn(async () => + (pending.shift() ?? (() => Promise.reject(new Error("unexpected recovery request"))))(), + ); +} + +function completingRecoverySpawner( + responses: readonly string[], +): ReturnType> { + const pending = [...responses]; + return vi.fn((_binary, _args, _stdio, _env) => { + const childEvents = new EventEmitter(); + const stderr = new EventEmitter(); + const stdout = new EventEmitter(); + const child: AgentDispatchChild = { + exitCode: null, + signalCode: null, + kill: vi.fn(() => true), + once: ((event: string, listener: (...args: unknown[]) => void) => + childEvents.once(event, listener)) as AgentDispatchChild["once"], + stderr, + stdout, + }; + const response = pending.shift() ?? ""; + queueMicrotask(() => { + stdout.emit("data", response); + child.exitCode = 0; + childEvents.emit("close", 0, null); + }); + return child; + }); +} + function getCommandUrl(command: readonly string[]): string { return command.find((arg) => arg.startsWith("http://")) ?? ""; } @@ -52,22 +111,9 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("uses the persisted direct bridge route for both the default probe and warm-up", async () => { - const cleanup = vi.fn(() => ({ ok: true as const })); - const prepareDockerEnvironment = () => ({ - env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, - isolatedCredentialConfig: true, - cleanup, - }); - const runCaptureImpl = vi.fn( - (_command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => - options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" - ? JSON.stringify({ models: [] }) - : "", - ); - const runCaptureExImpl = vi.fn((_command: string[], options?: { env?: NodeJS.ProcessEnv }) => - options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" - ? successfulWarmResult() - : { stdout: "", exitCode: 1, timedOut: false }, + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + successfulWarmResult(), ); await expect( @@ -77,32 +123,25 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { - runCaptureImpl, - runCaptureExImpl, - prepareDockerEnvironment, - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0]?.[0] ?? [])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, ); - expect(runCaptureImpl.mock.calls[0][0][0]).toBe("docker"); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? [])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); - expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("docker"); - expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({ + expect(getCommandBody(runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? [])).toMatchObject({ model: "qwen3.6:35b", stream: false, think: false, }); - expect(runCaptureImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe("/tmp/credential-free-docker"); - expect(runCaptureExImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe( - "/tmp/credential-free-docker", - ); - expect(cleanup).toHaveBeenCalledTimes(2); + expect(runRecoveryCaptureImpl.mock.calls.map(([, options]) => options.host)).toEqual([ + "host.docker.internal", + "host.docker.internal", + ]); }); it("runs the production status probe and warm-up through the async capture boundary", async () => { @@ -160,21 +199,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { cleanup, }; }); - const commands: string[][] = []; - const runCaptureImpl = vi.fn((command: readonly string[]) => { - commands.push([...command]); - return getCommandUrl(command).endsWith("/api/ps") - ? JSON.stringify({ models: [] }) - : JSON.stringify({ models: [{ name: "llama3.2:1b" }] }); - }); - const runCaptureExImpl = vi.fn((command: string[]) => { - commands.push([...command]); - return { - stdout: JSON.stringify({ error: "model not found" }), - exitCode: 0, - timedOut: false, - }; - }); + const spawnRecoveryChild = completingRecoverySpawner([ + JSON.stringify({ models: [] }), + JSON.stringify({ error: "model not found" }), + JSON.stringify({ models: [{ name: "llama3.2:1b" }] }), + ]); await expect( maybeWarmOllamaAfterDaemonRestart( @@ -183,16 +212,22 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { runCaptureImpl, runCaptureExImpl, prepareDockerEnvironment }, + { spawnRecoveryChild, prepareDockerEnvironment }, ), ).resolves.toMatchObject({ kind: "skipped", reason: "model-absent" }); + const commands = spawnRecoveryChild.mock.calls.map(([binary, args]) => [binary, ...args]); expect(commands.map(getCommandUrl)).toEqual([ `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ]); expect(commands.every((command) => command[0] === "docker")).toBe(true); + expect(spawnRecoveryChild.mock.calls.map(([, , , env]) => env.DOCKER_CONFIG)).toEqual([ + "/tmp/credential-free-docker-1", + "/tmp/credential-free-docker-2", + "/tmp/credential-free-docker-3", + ]); expect(prepareDockerEnvironment).toHaveBeenCalledTimes(3); expect(cleanups).toHaveLength(3); expect(cleanups[0]).toHaveBeenCalledOnce(); @@ -261,8 +296,10 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("maps an auth-proxy route back to host loopback", async () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + successfulWarmResult(), + ); await maybeWarmOllamaAfterDaemonRestart( { @@ -270,22 +307,22 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PROXY_PORT}/v1`, }, - { runCaptureImpl, runCaptureExImpl }, + { runRecoveryCaptureImpl }, ); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0]?.[0] ?? [])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, ); - expect(runCaptureImpl.mock.calls[0][0][0]).toBe("curl"); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? [])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, ); - expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("curl"); }); it("falls back to an allowlisted host instead of probing an arbitrary registry URL", async () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + successfulWarmResult(), + ); await maybeWarmOllamaAfterDaemonRestart( { @@ -295,18 +332,23 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }, { getOllamaHost: () => "also.example.com", - runCaptureImpl, - runCaptureExImpl, + runRecoveryCaptureImpl, }, ); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0]?.[0] ?? [])).toContain( + "http://127.0.0.1:", + ); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? [])).toContain( + "http://127.0.0.1:", + ); }); it("does not map an unrecognized proxy-port host to host loopback (#6039)", async () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + successfulWarmResult(), + ); await maybeWarmOllamaAfterDaemonRestart( { @@ -316,80 +358,73 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }, { getOllamaHost: () => "host.docker.internal", - runCaptureImpl, - runCaptureExImpl, + runRecoveryCaptureImpl, }, ); - expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[0]?.[0] ?? [])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, ); - expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? [])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); }); it("skips the warm-up when the selected model is already loaded", async () => { - const probeRuntimeModelStatus = vi.fn(() => ({ - probed: true, - loaded: true, - cpuOnly: false, - })); - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + recoveryResult(JSON.stringify({ models: [{ name: "qwen3.6:35b", size_vram: 1 }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { probeRuntimeModelStatus, runCaptureExImpl }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", reason: "already-loaded" }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("skips the warm-up when the daemon probe is unreachable", async () => { - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const runRecoveryCaptureImpl = scriptedRecoveryCapture(recoveryResult("", { exitCode: 7 })); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { runCaptureImpl: () => "", runCaptureExImpl }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", reason: "unreachable", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("skips the warm-up when the daemon status response is malformed", async () => { - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + const runRecoveryCaptureImpl = scriptedRecoveryCapture(recoveryResult("not-json")); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { runCaptureImpl: () => "not-json", runCaptureExImpl }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", reason: "unreachable", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("reports a bounded warm-up timeout", async () => { + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult("", { exitCode: 28, timedOut: true }), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ - stdout: "", - exitCode: 28, - timedOut: true, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", @@ -402,52 +437,44 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { it("limits warm-up to the command timeout budget remaining after the probe", async () => { let nowMs = 1_000; - const runCaptureExImpl = vi.fn( - (_command: string[], _options?: { env?: NodeJS.ProcessEnv; timeout?: number }) => - successfulWarmResult(), - ); - const probeRuntimeModelStatus = vi.fn(() => { - nowMs = 6_000; - return unloadedStatus; - }); + const responses = [ + () => { + nowMs = 6_000; + return unloadedProbeResult(); + }, + () => successfulWarmResult(), + ]; + const runRecoveryCaptureImpl = vi.fn(async () => responses.shift()!()); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { - probeRuntimeModelStatus, - runCaptureExImpl, + runRecoveryCaptureImpl, timeoutSeconds: 30, now: () => nowMs, }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); - const warmCommand = runCaptureExImpl.mock.calls[0][0]; + const warmCommand = runRecoveryCaptureImpl.mock.calls[1]?.[0] ?? []; expect(warmCommand[warmCommand.indexOf("--max-time") + 1]).toBe("25"); - expect(runCaptureExImpl.mock.calls[0][1]?.timeout).toBe(25_000); - expect(probeRuntimeModelStatus).toHaveBeenCalledWith( - "qwen3.6:35b", - expect.any(Function), - expect.any(Function), - 5_000, - ); + expect(runRecoveryCaptureImpl.mock.calls[0]?.[1].timeoutMilliseconds).toBe(5_000); + expect(runRecoveryCaptureImpl.mock.calls[1]?.[1].timeoutMilliseconds).toBe(25_000); }); it("skips warm-up when the probe consumes the command timeout budget", async () => { let nowMs = 1_000; - const runCaptureExImpl = vi.fn(() => successfulWarmResult()); - const probeRuntimeModelStatus = vi.fn(() => { + const runRecoveryCaptureImpl = vi.fn(async () => { nowMs = 31_000; - return unloadedStatus; + return unloadedProbeResult(); }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, { - probeRuntimeModelStatus, - runCaptureExImpl, + runRecoveryCaptureImpl, timeoutSeconds: 30, now: () => nowMs, }, @@ -457,47 +484,39 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { reason: "deadline-exhausted", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(runCaptureExImpl).not.toHaveBeenCalled(); + expect(runRecoveryCaptureImpl).toHaveBeenCalledOnce(); }); it("bounds the daemon probe and skips warm-up when a short timeout is consumed", async () => { let nowMs = 1_000; - const probeRuntimeModelStatus = vi.fn(() => { + const runRecoveryCaptureImpl = vi.fn(async () => { nowMs = 3_000; - return unloadedStatus; + return unloadedProbeResult(); }); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { probeRuntimeModelStatus, timeoutSeconds: 2, now: () => nowMs }, + { runRecoveryCaptureImpl, timeoutSeconds: 2, now: () => nowMs }, ), ).resolves.toEqual({ kind: "skipped", reason: "deadline-exhausted", endpoint: `http://127.0.0.1:${OLLAMA_PORT}`, }); - expect(probeRuntimeModelStatus).toHaveBeenCalledWith( - "qwen3.6:35b", - expect.any(Function), - expect.any(Function), - 2_000, - ); + expect(runRecoveryCaptureImpl.mock.calls[0]?.[1].timeoutMilliseconds).toBe(2_000); }); it("does not treat an exit-zero Ollama error body as a successful warm-up", async () => { + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult(JSON.stringify({ error: "model not found" })), + recoveryResult("not-json"), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "missing:latest" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory: () => null, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "model not found" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -509,20 +528,15 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("keeps the warm-up error when the inventory probe throws", async () => { + const runRecoveryCaptureImpl = failingRecoveryCapture( + new Error("inventory unavailable"), + unloadedProbeResult(), + recoveryResult(JSON.stringify({ error: "runner stopped unexpectedly" })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory: () => { - throw new Error("inventory unavailable"); - }, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "runner stopped unexpectedly" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -533,7 +547,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", async () => { - const probeModelInventory = vi.fn(() => ["llama3.2:1b"]); + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult(JSON.stringify({ error: "model not found" })), + recoveryResult(JSON.stringify({ models: [{ name: "llama3.2:1b" }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( @@ -542,15 +560,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "gemma4:26b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "model not found" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", @@ -558,27 +568,22 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, inventoryLabel: "llama3.2:1b", }); - expect(probeModelInventory).toHaveBeenCalledWith( - "host.docker.internal", - undefined, - 5_000, - undefined, + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[2]?.[0] ?? [])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ); + expect(runRecoveryCaptureImpl.mock.calls[2]?.[1].timeoutMilliseconds).toBe(5_000); }); it("keeps the warm failure when the daemon does hold the model (#9455)", async () => { + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult(JSON.stringify({ error: "runner stopped unexpectedly" })), + recoveryResult(JSON.stringify({ models: [{ name: "qwen3.6:35b" }] })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - probeModelInventory: () => ["qwen3.6:35b"], - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ error: "runner stopped unexpectedly" }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -590,17 +595,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("accepts a completed thinking-only response from a thinking model", async () => { + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult(JSON.stringify({ response: "", thinking: "The model is ready.", done: true })), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ - stdout: JSON.stringify({ response: "", thinking: "The model is ready.", done: true }), - exitCode: 0, - timedOut: false, - }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); }); @@ -611,13 +613,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ["missing done marker", JSON.stringify({ response: "Hello!" })], ["empty response", JSON.stringify({ response: "", done: true })], ])("rejects an invalid warm response: %s", async (_name, stdout) => { + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult(stdout), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toMatchObject({ kind: "warmed", @@ -628,13 +631,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("reports a non-zero warm command exit", async () => { + const runRecoveryCaptureImpl = scriptedRecoveryCapture( + unloadedProbeResult(), + recoveryResult("", { exitCode: 7 }), + ); await expect( maybeWarmOllamaAfterDaemonRestart( { provider: "ollama-local", model: "qwen3.6:35b" }, - { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), - }, + { runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", @@ -647,10 +651,10 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { it("reports a warm process spawn failure without throwing", async () => { const deps: OllamaRestartRecoveryDeps = { - probeRuntimeModelStatus: () => unloadedStatus, - runCaptureExImpl: () => { - throw new Error("spawn failed"); - }, + runRecoveryCaptureImpl: failingRecoveryCapture( + new Error("spawn failed"), + unloadedProbeResult(), + ), }; await expect( diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 637dace2eeb..b0aed6a66b7 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -20,22 +20,16 @@ import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args" import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, - createOllamaApiCapture, - createOllamaApiCaptureEx, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, parseOllamaModelInventory, prepareOllamaApiExecution, - probeOllamaEndpointInventory, - type RunCaptureFn, - type RunCaptureExFn, } from "../../../inference/local"; import { type OllamaRuntimeModelStatus, parseOllamaRuntimeModelStatus, - probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; import { buildSubprocessEnv, redact, redactFull } from "../../../runner"; import type { SandboxExecSignalSource } from "../exec"; @@ -55,23 +49,13 @@ export interface OllamaRestartRecoveryOptions { timeoutSeconds?: number; } +type PrepareOllamaDockerEnvironment = NonNullable< + Parameters[2] +>["prepareDockerEnvironment"]; + export interface OllamaRestartRecoveryDeps extends OllamaRestartRecoveryOptions { - probeRuntimeModelStatus?: ( - model: string, - getOllamaHost: () => string, - runCaptureImpl?: RunCaptureFn, - timeoutMilliseconds?: number, - ) => OllamaRuntimeModelStatus; - probeModelInventory?: ( - host: string, - runCaptureImpl?: RunCaptureFn, - timeoutMilliseconds?: number, - prepareDockerEnvironment?: Parameters[2], - ) => string[] | null; - runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; - runCaptureImpl?: RunCaptureFn; - prepareDockerEnvironment?: Parameters[2]; + prepareDockerEnvironment?: PrepareOllamaDockerEnvironment; runRecoveryCaptureImpl?: OllamaRecoveryCaptureFn; signalSource?: SandboxExecSignalSource; spawnRecoveryChild?: OllamaRecoverySpawner; @@ -186,7 +170,7 @@ export type OllamaRecoveryCaptureFn = ( options: { host: string; timeoutMilliseconds: number; - prepareDockerEnvironment?: Parameters[2]; + prepareDockerEnvironment?: PrepareOllamaDockerEnvironment; signalSource?: SandboxExecSignalSource; spawnRecoveryChild?: OllamaRecoverySpawner; }, @@ -376,7 +360,6 @@ export async function maybeWarmOllamaAfterDaemonRestart( } const now = deps.now ?? Date.now; const recoveryDeadline = recoveryDeadlineMilliseconds(deps.timeoutSeconds, now); - const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; const probeBudgetMilliseconds = remainingRecoveryMilliseconds(recoveryDeadline, now); if (probeBudgetMilliseconds === 0) { return { kind: "skipped", reason: "deadline-exhausted", endpoint: rawEndpoint }; @@ -387,33 +370,24 @@ export async function maybeWarmOllamaAfterDaemonRestart( OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, probeBudgetMilliseconds, ); - if (deps.probeRuntimeModelStatus || deps.runCaptureImpl) { - const rawCapture = createOllamaApiCapture( - deps.runCaptureImpl, - rawHost, - deps.prepareDockerEnvironment, - ); - status = probe(model, () => rawHost, rawCapture, statusTimeoutMilliseconds); - } else { - const capture = deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture; - const result = await capture( - buildOllamaProbeCommand(rawHost, "/api/ps", statusTimeoutMilliseconds), - { - host: rawHost, - timeoutMilliseconds: statusTimeoutMilliseconds, - prepareDockerEnvironment: deps.prepareDockerEnvironment, - signalSource: deps.signalSource, - spawnRecoveryChild: deps.spawnRecoveryChild, - }, - ); - if (result.signal && !result.timedOut) { - return { kind: "cancelled", signal: result.signal }; - } - status = - result.exitCode === 0 && !result.error - ? parseOllamaRuntimeModelStatus(model, result.stdout) - : { probed: false, loaded: false, cpuOnly: false }; + const capture = deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture; + const result = await capture( + buildOllamaProbeCommand(rawHost, "/api/ps", statusTimeoutMilliseconds), + { + host: rawHost, + timeoutMilliseconds: statusTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }, + ); + if (result.signal && !result.timedOut) { + return { kind: "cancelled", signal: result.signal }; } + status = + result.exitCode === 0 && !result.error + ? parseOllamaRuntimeModelStatus(model, result.stdout) + : { probed: false, loaded: false, cpuOnly: false }; } catch { return { kind: "skipped", reason: "unreachable", endpoint: rawEndpoint }; } @@ -432,22 +406,15 @@ export async function maybeWarmOllamaAfterDaemonRestart( try { const command = buildWarmCommand(model, rawHost, warmupTimeoutSeconds); - const result = deps.runCaptureExImpl - ? createOllamaApiCaptureEx( - deps.runCaptureExImpl, - rawHost, - deps.prepareDockerEnvironment, - )(command, { timeout: warmupTimeoutMilliseconds }) - : await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)(command, { - host: rawHost, - timeoutMilliseconds: warmupTimeoutMilliseconds, - prepareDockerEnvironment: deps.prepareDockerEnvironment, - signalSource: deps.signalSource, - spawnRecoveryChild: deps.spawnRecoveryChild, - }); - const asyncResult = result as Partial; - if (asyncResult.signal && !result.timedOut) { - return { kind: "cancelled", signal: asyncResult.signal }; + const result = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)(command, { + host: rawHost, + timeoutMilliseconds: warmupTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }); + if (result.signal && !result.timedOut) { + return { kind: "cancelled", signal: result.signal }; } if (result.timedOut) { return { @@ -461,16 +428,13 @@ export async function maybeWarmOllamaAfterDaemonRestart( ), }; } - if (asyncResult.error) { + if (result.error) { return { kind: "warmed", ok: false, reason: "spawn-failed", endpoint: rawEndpoint, - detail: boundedOllamaRestartRecoveryDetail( - asyncResult.error, - "warm-up process could not start", - ), + detail: boundedOllamaRestartRecoveryDetail(result.error, "warm-up process could not start"), }; } if (result.exitCode !== 0) { @@ -500,32 +464,23 @@ export async function maybeWarmOllamaAfterDaemonRestart( OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS, inventoryBudgetMilliseconds, ); - if (deps.probeModelInventory || deps.runCaptureImpl) { - inventory = (deps.probeModelInventory ?? probeOllamaEndpointInventory)( - rawHost, - deps.runCaptureImpl, - inventoryTimeoutMilliseconds, - deps.prepareDockerEnvironment, - ); - } else { - const inventoryResult = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)( - buildOllamaProbeCommand(rawHost, "/api/tags", inventoryTimeoutMilliseconds), - { - host: rawHost, - timeoutMilliseconds: inventoryTimeoutMilliseconds, - prepareDockerEnvironment: deps.prepareDockerEnvironment, - signalSource: deps.signalSource, - spawnRecoveryChild: deps.spawnRecoveryChild, - }, - ); - if (inventoryResult.signal && !inventoryResult.timedOut) { - return { kind: "cancelled", signal: inventoryResult.signal }; - } - inventory = - inventoryResult.exitCode === 0 && !inventoryResult.error - ? parseOllamaModelInventory(inventoryResult.stdout) - : null; + const inventoryResult = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)( + buildOllamaProbeCommand(rawHost, "/api/tags", inventoryTimeoutMilliseconds), + { + host: rawHost, + timeoutMilliseconds: inventoryTimeoutMilliseconds, + prepareDockerEnvironment: deps.prepareDockerEnvironment, + signalSource: deps.signalSource, + spawnRecoveryChild: deps.spawnRecoveryChild, + }, + ); + if (inventoryResult.signal && !inventoryResult.timedOut) { + return { kind: "cancelled", signal: inventoryResult.signal }; } + inventory = + inventoryResult.exitCode === 0 && !inventoryResult.error + ? parseOllamaModelInventory(inventoryResult.stdout) + : null; } catch { // Inventory only refines the original warm-up error. } diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 911cdcf39b8..b668b039a85 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -43,6 +43,11 @@ function buildDeps( detectVllmProfile: vi.fn(() => null), getLocalProviderAvailabilityEndpoint: vi.fn(() => "http://127.0.0.1:8000/v1/models"), detectLocalTcpListener: vi.fn(() => null), + prepareDockerEnvironment: vi.fn(() => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true as const }), + })), ...overrides, }; } @@ -189,8 +194,17 @@ describe("detectInferenceProviderHostState", () => { it("detects Windows-host Ollama from Docker Desktop when WSL cannot reach it (#8127)", () => { const logs: string[] = []; - const dockerCapture = vi.fn((command: string[]) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", + const cleanup = vi.fn(() => ({ ok: true as const })); + const runCapture = vi.fn( + (command, options) => { + const isNetworkingModeProbe = command.join(" ").includes("wslinfo --networking-mode"); + const isIsolatedWindowsOllamaProbe = + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL && + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? VALID_OLLAMA_TAGS_BODY + : ""; + return isNetworkingModeProbe ? "nat\n" : isIsolatedWindowsOllamaProbe; + }, ); const deps = buildDeps({ isWsl: vi.fn(() => true), @@ -200,12 +214,12 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Users\\me\\AppData\\Local\\Programs\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => { - const joined = command.join(" "); - if (joined.includes("wslinfo --networking-mode")) return "nat\n"; - return ""; + runCapture, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, }), - dockerCapture, }); const state = detectInferenceProviderHostState({ @@ -225,20 +239,26 @@ describe("detectInferenceProviderHostState", () => { expect(state.winOllamaInstalledPath).toMatch(/ollama\.exe$/); expect(logs.join("\n")).toContain("Ollama is running on both WSL and the Windows host"); expect(deps.getWindowsHostOllamaDockerRequirement).toHaveBeenCalledWith("docker-desktop"); - expect(dockerCapture).toHaveBeenCalledWith( + expect(runCapture).toHaveBeenCalledWith( [ + "docker", "run", "--rm", "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", - "2", + "3", "--max-time", "5", WINDOWS_OLLAMA_TAGS_URL, ], - { ignoreError: true }, + expect.objectContaining({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + ignoreError: true, + timeout: 5_000, + }), ); + expect(cleanup).toHaveBeenCalledOnce(); }); it("keeps WSL-local install available when Docker Desktop cannot reach Windows-host Ollama (#8199)", () => { @@ -279,7 +299,9 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Users\\me\\AppData\\Local\\Programs\\Ollama\\ollama.exe", loopbackOnly: false, })), - dockerCapture: vi.fn(() => body), + runCapture: vi.fn((command) => + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? body : "", + ), }); const state = detectWithDeps(deps); @@ -290,10 +312,10 @@ describe("detectInferenceProviderHostState", () => { }); it("does not run the Windows-host probe without Docker Desktop WSL integration (#8127)", () => { - const dockerCapture = vi.fn(() => "{}"); + const runCapture = vi.fn(() => "{}"); const deps = buildDeps({ isWsl: vi.fn(() => true), - dockerCapture, + runCapture, getWindowsHostOllamaDockerRequirement: vi.fn(() => getWindowsHostOllamaDockerRequirement("docker"), ), @@ -302,7 +324,9 @@ describe("detectInferenceProviderHostState", () => { const state = detectWithDeps(deps); expect(state.windowsOllamaReachable).toBe(false); - expect(dockerCapture).not.toHaveBeenCalled(); + expect( + runCapture.mock.calls.some(([command]) => command.includes(WINDOWS_OLLAMA_TAGS_URL)), + ).toBe(false); }); it("passes injected platform and env through WSL detection", () => { @@ -335,13 +359,12 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - runCapture: vi.fn((command) => { - const joined = command.join(" "); - if (joined.includes("wslinfo --networking-mode")) return "mirrored\n"; - return ""; - }), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", + runCapture: vi.fn((command) => + command.join(" ").includes("wslinfo --networking-mode") + ? "mirrored\n" + : command.at(-1) === WINDOWS_OLLAMA_TAGS_URL + ? VALID_OLLAMA_TAGS_BODY + : "", ), detectLocalTcpListener: vi.fn(() => false), }); @@ -375,10 +398,11 @@ describe("detectInferenceProviderHostState", () => { loopbackOnly: false, })), runCapture: vi.fn((command) => - command.join(" ").includes("wslinfo --networking-mode") ? "mirrored\n" : "", - ), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", + command.join(" ").includes("wslinfo --networking-mode") + ? "mirrored\n" + : command.at(-1) === WINDOWS_OLLAMA_TAGS_URL + ? VALID_OLLAMA_TAGS_BODY + : "", ), detectLocalTcpListener: vi.fn(() => true), }); @@ -411,10 +435,11 @@ describe("detectInferenceProviderHostState", () => { loopbackOnly: false, })), runCapture: vi.fn((command) => - command.join(" ").includes("wslinfo --networking-mode") ? "mirrored\n" : "", - ), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", + command.join(" ").includes("wslinfo --networking-mode") + ? "mirrored\n" + : command.at(-1) === WINDOWS_OLLAMA_TAGS_URL + ? VALID_OLLAMA_TAGS_BODY + : "", ), detectLocalTcpListener: vi.fn(() => null), }); @@ -436,10 +461,11 @@ describe("detectInferenceProviderHostState", () => { loopbackOnly: false, })), runCapture: vi.fn((command) => - command.join(" ").includes("wslinfo --networking-mode") ? "future-mode\n" : "", - ), - dockerCapture: vi.fn((command) => - command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", + command.join(" ").includes("wslinfo --networking-mode") + ? "future-mode\n" + : command.at(-1) === WINDOWS_OLLAMA_TAGS_URL + ? VALID_OLLAMA_TAGS_BODY + : "", ), detectLocalTcpListener, }); @@ -463,7 +489,6 @@ describe("detectInferenceProviderHostState", () => { it("probes Docker reachability when WSL can reach Windows-host Ollama (#10100)", () => { const runCapture = vi.fn(() => ""); - const dockerCapture = vi.fn(() => ""); const deps = buildDeps({ isWsl: vi.fn(() => true), findReachableOllamaHost: vi.fn(() => "host.docker.internal"), @@ -473,22 +498,21 @@ describe("detectInferenceProviderHostState", () => { loopbackOnly: true, })), runCapture, - dockerCapture, }); const state = detectWithDeps(deps); expect(state.isWindowsHostOllama).toBe(true); expect(state.windowsOllamaReachable).toBe(false); - expect(dockerCapture).toHaveBeenCalledWith( - expect.arrayContaining([WINDOWS_OLLAMA_TAGS_URL]), - { ignoreError: true }, + expect(runCapture).toHaveBeenCalledWith( + expect.arrayContaining(["docker", WINDOWS_OLLAMA_TAGS_URL]), + expect.objectContaining({ ignoreError: true, timeout: 5_000 }), ); }); it("reuses Windows-host Ollama only after Docker reachability succeeds (#10100)", () => { - const dockerCapture = vi.fn( - (command) => (command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : ""), + const runCapture = vi.fn((command) => + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL ? VALID_OLLAMA_TAGS_BODY : "", ); const deps = buildDeps({ isWsl: vi.fn(() => true), @@ -498,16 +522,16 @@ describe("detectInferenceProviderHostState", () => { installedPath: "C:\\Ollama\\ollama.exe", loopbackOnly: false, })), - dockerCapture, + runCapture, }); const state = detectWithDeps(deps); expect(state.isWindowsHostOllama).toBe(true); expect(state.windowsOllamaReachable).toBe(true); - expect(dockerCapture).toHaveBeenCalledWith( - expect.arrayContaining([WINDOWS_OLLAMA_TAGS_URL]), - { ignoreError: true }, + expect(runCapture).toHaveBeenCalledWith( + expect.arrayContaining(["docker", WINDOWS_OLLAMA_TAGS_URL]), + expect.objectContaining({ ignoreError: true, timeout: 5_000 }), ); }); }); diff --git a/src/lib/onboard/provider-host-state.ts b/src/lib/onboard/provider-host-state.ts index fc9457241a4..dedf39e5cbb 100644 --- a/src/lib/onboard/provider-host-state.ts +++ b/src/lib/onboard/provider-host-state.ts @@ -5,13 +5,14 @@ import fs from "node:fs"; import { dockerCapture as defaultDockerCapture } from "../adapters/docker"; import { + createOllamaApiCapture, findReachableOllamaHost, getLocalProviderAvailabilityEndpoint, - getWindowsHostOllamaDockerReachabilityArgs, isLocalProviderProbeOutputHealthy, - isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_PORT, + probeOllamaEndpointInventory, + type RunCaptureFn, } from "../inference/local"; import type { NvidiaPlatform } from "../inference/nim"; import { detectVllmProfile, type VllmProfile } from "../inference/vllm"; @@ -32,11 +33,8 @@ import { type OllamaInstallMenuResult, resolveOllamaInstallMenuEntry } from "./o import { buildVllmMenuEntries, type VllmMenuEntry } from "./vllm-menu"; import { detectWindowsHostOllama, type WindowsHostOllamaState } from "./windows-host-ollama"; -type RunCapture = (args: string[], options?: { ignoreError?: boolean }) => string; -type DockerCapture = ( - args: string[], - options?: { env?: NodeJS.ProcessEnv; ignoreError?: boolean; timeout?: number }, -) => string; +type RunCapture = RunCaptureFn; +type DockerCapture = RunCaptureFn; type ReadTextFile = (filePath: string) => string | null; export interface InferenceProviderHostGpu { @@ -92,6 +90,7 @@ export interface DetectInferenceProviderHostStateDeps { detectVllmProfile: (gpu: InferenceProviderHostGpu | null | undefined) => VllmProfile | null; getLocalProviderAvailabilityEndpoint: (provider: string) => string | null; detectLocalTcpListener: (port: number) => boolean | null; + prepareDockerEnvironment?: Parameters[2]; } const LOCAL_PROVIDER_PROBE_CURL_ARGS = ["--connect-timeout", "2", "--max-time", "5"] as const; @@ -164,6 +163,7 @@ function buildDeps( getLocalProviderAvailabilityEndpoint: overrides.getLocalProviderAvailabilityEndpoint ?? getLocalProviderAvailabilityEndpoint, detectLocalTcpListener: overrides.detectLocalTcpListener ?? detectLocalTcpListener, + prepareDockerEnvironment: overrides.prepareDockerEnvironment, }; } @@ -190,16 +190,21 @@ function probeVllmRunning(deps: DetectInferenceProviderHostStateDeps): boolean { function probeWindowsOllamaReachable(input: { isWsl: boolean; dockerRequirementSupported: boolean; - dockerCapture: DockerCapture; + runCapture: RunCapture; + prepareDockerEnvironment?: Parameters[2]; }): boolean { if (!input.isWsl || !input.dockerRequirementSupported) return false; // A successful Docker run is not enough: a captive proxy, a stale listener, or // a stub on host.docker.internal can all answer with arbitrary 2xx bodies. Only // a body in the Ollama `/api/tags` wire format proves the Windows daemon is live. - const body = input.dockerCapture(getWindowsHostOllamaDockerReachabilityArgs(), { - ignoreError: true, - }); - return isValidOllamaTagsResponseBody(body); + return ( + probeOllamaEndpointInventory( + OLLAMA_HOST_DOCKER_INTERNAL, + input.runCapture, + 5_000, + input.prepareDockerEnvironment, + ) !== null + ); } function maybeWarnAboutDuplicateOllamaDaemons(input: { @@ -263,7 +268,8 @@ export function detectInferenceProviderHostState( : probeWindowsOllamaReachable({ isWsl, dockerRequirementSupported: windowsHostOllamaDockerRequirement.supported, - dockerCapture: deps.dockerCapture, + runCapture: deps.runCapture, + prepareDockerEnvironment: deps.prepareDockerEnvironment, }); const wslNetworkingMode = From ca8a31d60347142e9a7c5e08447b8cf63839fd71 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 04:08:48 -0700 Subject: [PATCH 18/48] fix(inference): remove redundant status fallback Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama-runtime-context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/inference/ollama-runtime-context.ts b/src/lib/inference/ollama-runtime-context.ts index 6dbbe87b62e..fa11c2512af 100644 --- a/src/lib/inference/ollama-runtime-context.ts +++ b/src/lib/inference/ollama-runtime-context.ts @@ -174,7 +174,7 @@ export function parseOllamaRuntimeModelStatus( if (!output) return { probed: false, loaded: false, cpuOnly: false }; try { - const parsed = JSON.parse(String(output || "")) as { models?: unknown } | null; + const parsed = JSON.parse(output) as { models?: unknown } | null; if (!parsed || !Array.isArray(parsed.models)) { return { probed: false, loaded: false, cpuOnly: false }; } From 8168f2c130a078a82e7f6c9dc782346fa6ce47c9 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 04:32:01 -0700 Subject: [PATCH 19/48] fix(ollama): restore Windows state after failed rebind Signed-off-by: Prekshi Vyas --- .../agent/passthrough-dispatch.test.ts | 4 - src/lib/inference/ollama/windows.test.ts | 120 ++++++++++++++-- src/lib/inference/ollama/windows.ts | 128 +++++++++++++++--- 3 files changed, 216 insertions(+), 36 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts b/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts index a28273e0359..bd6bd5830ef 100644 --- a/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts @@ -243,10 +243,6 @@ describe("agentDispatchDeadlineSeconds", () => { expect(agentDispatchDeadlineSeconds(["openclaw", "agent", "-m", "hi"])).toBeUndefined(); }); - it("holds the deadline buffer above the longest aborted-run finish measured (#8723)", () => { - expect(AGENT_DISPATCH_DEADLINE_BUFFER_SECONDS).toBeGreaterThan(20); - }); - it("stays unbounded when the buffered deadline leaves the safe-integer range (#8723)", () => { const ceiling = String(Number.MAX_SAFE_INTEGER); expect(requestedAgentTimeoutSeconds(["openclaw", "agent", "--timeout", ceiling])).toBe( diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 82631c96b9b..c4c65d1a9e9 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -35,6 +35,18 @@ function isDockerTagsRequest(command: string | string[]): boolean { return Array.isArray(command) && command[0] === "docker" && command.some(isWindowsOllamaTagsUrl); } +function successfulRun(stdout = "") { + return { status: 0, stdout, stderr: "" }; +} + +function hostSnapshotRun( + userHost: string | null, + watcherPath: string | null, + daemonPath: string | null, +) { + return successfulRun(JSON.stringify({ userHost, watcherPath, daemonPath })); +} + function loadWindowsOllamaWithMocks( run: ReturnType, runCapture: ReturnType, @@ -114,27 +126,30 @@ describe("Windows Ollama helper", () => { const run = vi.fn((command: string[]) => { const launch = commandText(command); + const capturesHostSnapshot = launch.includes("ConvertTo-Json -Compress"); + const persistsNewBinding = launch.includes( + "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", + ); const isWatcherLaunch = launch.includes(watcherPath); const isInstalledLaunch = launch.includes(installedPath); watcherLaunchAttempted ||= isWatcherLaunch; installedLaunchAttempted ||= isInstalledLaunch; - return isWatcherLaunch - ? { status: 1, stderr: "stale watcher path" } - : isInstalledLaunch - ? { status: 0, stderr: "" } - : { status: 1, stderr: "unexpected launch target" }; + return capturesHostSnapshot + ? hostSnapshotRun("127.0.0.1:11434", watcherPath, installedPath) + : persistsNewBinding + ? successfulRun() + : isWatcherLaunch + ? { status: 1, stderr: "stale watcher path" } + : isInstalledLaunch + ? successfulRun() + : { status: 1, stderr: "unexpected launch target" }; }); const runCapture = vi.fn((command: string | string[]) => { - const cmd = commandText(command); - const capturesWatcherPath = - cmd.includes("Get-Process 'ollama app'") && cmd.includes("ExpandProperty Path"); const probesDockerReadiness = isDockerTagsRequest(command); dockerReadinessObserved ||= probesDockerReadiness; - return capturesWatcherPath - ? watcherPath - : probesDockerReadiness && installedLaunchAttempted - ? JSON.stringify({ models: [] }) - : ""; + return probesDockerReadiness && installedLaunchAttempted + ? JSON.stringify({ models: [] }) + : ""; }); const localInference = require(LOCAL_INFERENCE_PATH); localInference.resetOllamaHostCache(); @@ -156,6 +171,85 @@ describe("Windows Ollama helper", () => { expect(dockerReadinessObserved).toBe(true); }); + it("restores the prior binding and watcher when every rebound launch fails", () => { + const priorHost = "127.0.0.1:11434"; + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + let rollbackScript = ""; + const run = vi.fn((command: string[]) => { + const script = commandText(command); + const capturesHostSnapshot = script.includes("ConvertTo-Json -Compress"); + const persistsNewBinding = script.includes( + "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", + ); + const restoresPriorState = script.includes("$previousHost ="); + rollbackScript = restoresPriorState ? script : rollbackScript; + return capturesHostSnapshot + ? hostSnapshotRun(priorHost, watcherPath, daemonPath) + : persistsNewBinding || restoresPriorState + ? successfulRun() + : { status: 1, stderr: "launch unavailable" }; + }); + const runCapture = vi.fn(() => ""); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + + try { + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(rollbackScript).toContain("SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')"); + expect(rollbackScript).toContain(Buffer.from(priorHost, "utf8").toString("base64")); + expect(rollbackScript).toContain(Buffer.from(watcherPath, "utf8").toString("base64")); + expect(rollbackScript).toContain("Start-Process -FilePath $previousWatcher"); + }); + + it("prints a direct recovery command when prior Windows state rollback fails", () => { + const priorHost = "127.0.0.1:11434"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + const run = vi.fn((command: string[]) => { + const script = commandText(command); + const capturesHostSnapshot = script.includes("ConvertTo-Json -Compress"); + const persistsNewBinding = script.includes( + "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", + ); + const restoresPriorState = script.includes("$previousHost ="); + return capturesHostSnapshot + ? hostSnapshotRun(priorHost, null, daemonPath) + : persistsNewBinding + ? successfulRun() + : restoresPriorState + ? { status: 1, stderr: "rollback denied" } + : { status: 1, stderr: "launch unavailable" }; + }); + const runCapture = vi.fn(() => ""); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + + try { + expect(windows.setupWindowsOllamaWith0000Binding()).toBe(false); + } finally { + restore(); + logSpy.mockRestore(); + } + + const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + errorSpy.mockRestore(); + const encodedCommand = diagnostic.match(/-EncodedCommand ([A-Za-z0-9+/=]+)/u)?.[1] ?? ""; + const recoveryScript = Buffer.from(encodedCommand, "base64").toString("utf16le"); + expect(diagnostic).toContain("Failed to restore the previous Windows Ollama state"); + expect(diagnostic).toContain(`Previous User-scope OLLAMA_HOST: ${JSON.stringify(priorHost)}`); + expect(diagnostic).toContain("powershell.exe -NoProfile -EncodedCommand"); + expect(recoveryScript).toContain("SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')"); + expect(recoveryScript).toContain("Start-Process -FilePath $previousDaemon"); + }); + it("isolates Docker credentials while waiting for the Windows-host daemon", () => { const run = vi.fn(); const cleanup = vi.fn(() => ({ ok: true as const })); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index acbc4340055..d0109b12e58 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -64,32 +64,110 @@ async function installOllamaOnWindowsHost(): Promise<{ ok: boolean; path: string return { ok: true, path: installedPath }; } -// Capture the watcher path so we can relaunch from the same exe after kill, -// preserving the tray icon and the watcher's auto-restart behavior. -function captureWindowsOllamaWatcherPath(): string { - return runCapture( +type WindowsOllamaHostSnapshot = { + userHost: string | null; + watcherPath: string | null; + daemonPath: string | null; +}; + +function optionalSnapshotPath(value: unknown): string | null | undefined { + if (value === null) return null; + if (typeof value !== "string") return undefined; + return value.trim() || null; +} + +// Capture every state item that setup mutates before changing the User-scope +// binding or stopping processes. A failed snapshot leaves the host untouched. +function captureWindowsOllamaHostSnapshot(): WindowsOllamaHostSnapshot | null { + const result = run( [ "powershell.exe", "-Command", - "Get-Process 'ollama app' -EA SilentlyContinue | Select-Object -First 1 -ExpandProperty Path", + "$userHost = [Environment]::GetEnvironmentVariable('OLLAMA_HOST','User'); " + + "$watcherPath = Get-Process 'ollama app' -EA SilentlyContinue | Select-Object -First 1 -ExpandProperty Path; " + + "$daemonPath = Get-Process ollama -EA SilentlyContinue | Select-Object -First 1 -ExpandProperty Path; " + + "[PSCustomObject]@{userHost=$userHost;watcherPath=$watcherPath;daemonPath=$daemonPath} | ConvertTo-Json -Compress", ], - { ignoreError: true }, - ).trim(); + { ignoreError: true, suppressOutput: true }, + ); + if (result.error || result.status !== 0) return null; + try { + const parsed = JSON.parse(String(result.stdout || "")) as Record | null; + if (!parsed || (parsed.userHost !== null && typeof parsed.userHost !== "string")) return null; + const watcherPath = optionalSnapshotPath(parsed.watcherPath); + const daemonPath = optionalSnapshotPath(parsed.daemonPath); + if (watcherPath === undefined || daemonPath === undefined) return null; + return { userHost: parsed.userHost, watcherPath, daemonPath }; + } catch { + return null; + } +} + +function psUtf8Expression(value: string): string { + const encoded = Buffer.from(value, "utf8").toString("base64"); + return `[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encoded}'))`; +} + +function psNullableUtf8Expression(value: string | null): string { + return value === null ? "$null" : psUtf8Expression(value); +} + +function runWindowsOllamaStateScript(script: string): boolean { + const result = run(["powershell.exe", "-Command", `$ErrorActionPreference='Stop'; ${script}`], { + ignoreError: true, + suppressOutput: true, + }); + return !result.error && result.status === 0; } // User-scope so the next login-time tray launch keeps the 0.0.0.0 binding // without NemoClaw being involved. -function persistOllamaHostEnvVar(): void { - runCapture( - [ - "powershell.exe", - "-Command", - "[Environment]::SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", - ], - { ignoreError: true }, +function persistOllamaHostEnvVar(): boolean { + return runWindowsOllamaStateScript( + "[Environment]::SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", ); } +function buildWindowsOllamaRestoreScript(snapshot: WindowsOllamaHostSnapshot): string { + const script = [ + `$previousHost = ${psNullableUtf8Expression(snapshot.userHost)}`, + "[Environment]::SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')", + "$env:OLLAMA_HOST = $previousHost", + ]; + if (snapshot.watcherPath) { + script.push( + `$previousWatcher = ${psUtf8Expression(snapshot.watcherPath)}`, + "Start-Process -FilePath $previousWatcher -WindowStyle Hidden -ErrorAction Stop", + ); + } else if (snapshot.daemonPath) { + script.push( + `$previousDaemon = ${psUtf8Expression(snapshot.daemonPath)}`, + "Start-Process -FilePath $previousDaemon -ArgumentList 'serve' -WindowStyle Hidden -ErrorAction Stop", + ); + } + return script.join("; "); +} + +function previousOllamaHostLabel(value: string | null): string { + if (value === null) return "unset"; + const terminalSafe = value.replace(/[\u202A-\u202E\u2066-\u2069]/gu, "").slice(0, 200); + return JSON.stringify(terminalSafe); +} + +function restoreWindowsOllamaHostSnapshot(snapshot: WindowsOllamaHostSnapshot): boolean { + const script = buildWindowsOllamaRestoreScript(snapshot); + if (runWindowsOllamaStateScript(script)) return true; + const encodedCommand = Buffer.from( + `$ErrorActionPreference='Stop'; ${script}`, + "utf16le", + ).toString("base64"); + console.error(" Failed to restore the previous Windows Ollama state."); + console.error(` Previous User-scope OLLAMA_HOST: ${previousOllamaHostLabel(snapshot.userHost)}`); + console.error(" Restore it and relaunch the previous Ollama process with:"); + console.error(` powershell.exe -NoProfile -EncodedCommand ${encodedCommand}`); + return false; +} + // Order matters: kill 'ollama app' (the tray watcher) before 'ollama' // (the daemon). The watcher auto-respawns the daemon as soon as it dies. // If the daemon goes first, the watcher can launch a fresh daemon with @@ -203,17 +281,29 @@ function launchAndAwaitWindowsOllama( function setupWindowsOllamaWith0000Binding( opts: { announceStop?: boolean; installedPath?: string } = {}, ): boolean { - const watcherPath = captureWindowsOllamaWatcherPath(); - persistOllamaHostEnvVar(); + const snapshot = captureWindowsOllamaHostSnapshot(); + if (!snapshot) { + console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); + return false; + } + if (!persistOllamaHostEnvVar()) { + console.error( + " Could not persist the Windows Ollama host binding; leaving processes running.", + ); + return false; + } if (opts.announceStop) { console.log(" Stopping existing Ollama on Windows host..."); } killWindowsOllamaProcesses(); sleepSeconds(1); - return launchAndAwaitWindowsOllama({ - watcherPath: watcherPath || undefined, + const launched = launchAndAwaitWindowsOllama({ + watcherPath: snapshot.watcherPath || undefined, installedPath: opts.installedPath, }); + if (launched) return true; + restoreWindowsOllamaHostSnapshot(snapshot); + return false; } function switchToWindowsOllamaHost(): void { From b778698699bc3dfb69498e731a8f43efc7f563a5 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 04:39:51 -0700 Subject: [PATCH 20/48] test(ollama): verify restart recovery boundaries --- .../agent/passthrough-dispatch.test.ts | 58 ++++++++++++++++++- src/lib/inference/ollama/windows.test.ts | 2 + src/lib/inference/ollama/windows.ts | 2 + 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts b/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts index bd6bd5830ef..690e4274430 100644 --- a/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-dispatch.test.ts @@ -18,7 +18,7 @@ import { SILENT_AGENT_DISPATCH_EXIT_CODE, TIMED_OUT_AGENT_TURN_EXIT_CODE, } from "./passthrough-dispatch"; -import { computeExitCode, type SandboxExecSignalSource } from "../exec"; +import { buildOpenshellExecArgs, computeExitCode, type SandboxExecSignalSource } from "../exec"; function dispatchHarness() { const childEvents = new EventEmitter(); @@ -42,7 +42,7 @@ function dispatchHarness() { add: (signal, listener) => signalEvents.on(signal, listener), remove: (signal, listener) => signalEvents.off(signal, listener), }; - return { child, signalEvents, signalSource, stderr, stdout }; + return { child, childEvents, signalEvents, signalSource, stderr, stdout }; } describe("runAgentDispatch", () => { @@ -115,6 +115,60 @@ describe("runAgentDispatch", () => { expect(result.stdout).toBe("1234"); expect(result.stderr).toBe(""); }); + + it("delivers a turn timeout reported 20.8 seconds after its requested deadline (#8723)", async () => { + vi.useFakeTimers(); + const harness = dispatchHarness(); + const requestedDeadlineSeconds = 30; + const delayedFinishMilliseconds = (requestedDeadlineSeconds + 20.8) * 1000; + const timeoutReport = "Request timed out before a response was generated.\n"; + const command = [ + "openclaw", + "agent", + "--timeout", + String(requestedDeadlineSeconds), + "-m", + "ping", + ]; + const args = buildOpenshellExecArgs("alpha", command, { + tty: false, + timeoutSeconds: agentDispatchDeadlineSeconds(command), + }); + + try { + const pending = runAgentDispatch( + "openshell", + args, + { stdinIsTty: true }, + { + signalSource: harness.signalSource, + spawnChild: (_binary, spawnArgs) => { + const hostTimeoutIndex = spawnArgs.indexOf("--timeout"); + const hostTimeoutMilliseconds = Number(spawnArgs[hostTimeoutIndex + 1]) * 1000; + setTimeout(() => harness.child.kill("SIGTERM"), hostTimeoutMilliseconds); + setTimeout(() => { + harness.stdout.emit("data", timeoutReport); + harness.child.exitCode = 0; + harness.childEvents.emit("close", 0, null); + }, delayedFinishMilliseconds); + return harness.child; + }, + }, + ); + + await vi.advanceTimersByTimeAsync(delayedFinishMilliseconds); + expect(await pending).toMatchObject({ + status: 0, + signal: null, + stdout: timeoutReport, + stderr: "", + }); + expect(harness.child.kill).not.toHaveBeenCalled(); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); }); describe("isSilentAgentDispatch", () => { diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index c4c65d1a9e9..aa54b941e20 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -204,6 +204,8 @@ describe("Windows Ollama helper", () => { } expect(rollbackScript).toContain("SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')"); + expect(rollbackScript).toContain("Get-Process 'ollama app'"); + expect(rollbackScript).toContain("Get-Process ollama"); expect(rollbackScript).toContain(Buffer.from(priorHost, "utf8").toString("base64")); expect(rollbackScript).toContain(Buffer.from(watcherPath, "utf8").toString("base64")); expect(rollbackScript).toContain("Start-Process -FilePath $previousWatcher"); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index d0109b12e58..3f043da37cc 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -130,6 +130,8 @@ function persistOllamaHostEnvVar(): boolean { function buildWindowsOllamaRestoreScript(snapshot: WindowsOllamaHostSnapshot): string { const script = [ + "Get-Process 'ollama app' -EA SilentlyContinue | Stop-Process -Force", + "Get-Process ollama -EA SilentlyContinue | Stop-Process -Force", `$previousHost = ${psNullableUtf8Expression(snapshot.userHost)}`, "[Environment]::SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')", "$env:OLLAMA_HOST = $previousHost", From 72685a98e9758eb214c5f9a1d0e57acad8e76ed7 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 04:58:38 -0700 Subject: [PATCH 21/48] test(ollama): model Windows rollback state Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 198 ++++++++++++++++++----- 1 file changed, 156 insertions(+), 42 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index aa54b941e20..ce66e695fe9 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -47,6 +47,114 @@ function hostSnapshotRun( return successfulRun(JSON.stringify({ userHost, watcherPath, daemonPath })); } +type WindowsPowerShellState = { + userHost: string | null; + watcherRunning: boolean; + daemonRunning: boolean; + events: string[]; +}; + +function createWindowsPowerShellBoundary(options: { + userHost: string | null; + watcherPath: string | null; + daemonPath: string | null; + launchStatuses?: number[]; + readinessResponse?: string; + rollbackStatus?: number; +}) { + const original = { + userHost: options.userHost, + watcherPath: options.watcherPath, + daemonPath: options.daemonPath, + }; + const state: WindowsPowerShellState = { + userHost: original.userHost, + watcherRunning: Boolean(original.watcherPath), + daemonRunning: Boolean(original.daemonPath), + events: [], + }; + const launchStatuses = options.launchStatuses ?? [1, 1, 1]; + const rollbackStatus = options.rollbackStatus ?? 0; + let launchIndex = 0; + const run = vi.fn((command: string[]) => { + const script = commandText(command); + const capturesHostSnapshot = script.includes("ConvertTo-Json -Compress"); + const persistsNewBinding = script.includes( + "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", + ); + const restoresPriorState = script.includes("$previousHost ="); + const launchesReplacement = !restoresPriorState && script.includes("Start-Process"); + const rollbackStopsWatcher = restoresPriorState && script.includes("Get-Process 'ollama app'"); + const rollbackStopsDaemon = restoresPriorState && script.includes("Get-Process ollama -EA"); + const launchStatus = launchStatuses[launchIndex] ?? 1; + const launchesWatcher = + launchesReplacement && + Boolean(original.watcherPath) && + script.includes(String(original.watcherPath)); + const event = capturesHostSnapshot + ? "snapshot" + : persistsNewBinding + ? "persist" + : restoresPriorState + ? "restore" + : launchesReplacement + ? "launch" + : "unexpected"; + state.events.push( + ...(restoresPriorState + ? [ + ...(rollbackStopsWatcher ? ["stop-watcher"] : []), + ...(rollbackStopsDaemon ? ["stop-daemon"] : []), + "restore", + ] + : [event]), + ); + state.watcherRunning = rollbackStopsWatcher ? false : state.watcherRunning; + state.daemonRunning = rollbackStopsDaemon ? false : state.daemonRunning; + state.userHost = persistsNewBinding + ? "0.0.0.0:11434" + : restoresPriorState && rollbackStatus === 0 + ? original.userHost + : state.userHost; + state.watcherRunning = + restoresPriorState && rollbackStatus === 0 + ? Boolean(original.watcherPath) + : launchesWatcher && launchStatus === 0 + ? true + : state.watcherRunning; + state.daemonRunning = + restoresPriorState && rollbackStatus === 0 + ? Boolean(original.daemonPath) && !original.watcherPath + : launchesReplacement && launchStatus === 0 && !launchesWatcher + ? true + : state.daemonRunning; + launchIndex += launchesReplacement ? 1 : 0; + return capturesHostSnapshot + ? hostSnapshotRun(original.userHost, original.watcherPath, original.daemonPath) + : persistsNewBinding + ? successfulRun() + : restoresPriorState + ? rollbackStatus === 0 + ? successfulRun() + : { status: rollbackStatus, stderr: "rollback denied" } + : launchesReplacement + ? launchStatus === 0 + ? successfulRun() + : { status: launchStatus, stderr: "launch unavailable" } + : { status: 1, stderr: "unexpected PowerShell operation" }; + }); + const runCapture = vi.fn((command: string | string[]) => { + const script = commandText(command); + const stopsWatcher = script.includes("Get-Process 'ollama app'"); + const stopsDaemon = script.includes("Get-Process ollama -EA"); + state.watcherRunning = stopsWatcher ? false : state.watcherRunning; + state.daemonRunning = stopsDaemon ? false : state.daemonRunning; + state.events.push(stopsWatcher ? "stop-watcher" : stopsDaemon ? "stop-daemon" : "probe"); + return isDockerTagsRequest(command) ? (options.readinessResponse ?? "") : ""; + }); + return { run, runCapture, state }; +} + function loadWindowsOllamaWithMocks( run: ReturnType, runCapture: ReturnType, @@ -175,25 +283,14 @@ describe("Windows Ollama helper", () => { const priorHost = "127.0.0.1:11434"; const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; - let rollbackScript = ""; - const run = vi.fn((command: string[]) => { - const script = commandText(command); - const capturesHostSnapshot = script.includes("ConvertTo-Json -Compress"); - const persistsNewBinding = script.includes( - "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", - ); - const restoresPriorState = script.includes("$previousHost ="); - rollbackScript = restoresPriorState ? script : rollbackScript; - return capturesHostSnapshot - ? hostSnapshotRun(priorHost, watcherPath, daemonPath) - : persistsNewBinding || restoresPriorState - ? successfulRun() - : { status: 1, stderr: "launch unavailable" }; + const boundary = createWindowsPowerShellBoundary({ + userHost: priorHost, + watcherPath, + daemonPath, }); - const runCapture = vi.fn(() => ""); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(boundary.run, boundary.runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); @@ -203,36 +300,24 @@ describe("Windows Ollama helper", () => { errorSpy.mockRestore(); } - expect(rollbackScript).toContain("SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')"); - expect(rollbackScript).toContain("Get-Process 'ollama app'"); - expect(rollbackScript).toContain("Get-Process ollama"); - expect(rollbackScript).toContain(Buffer.from(priorHost, "utf8").toString("base64")); - expect(rollbackScript).toContain(Buffer.from(watcherPath, "utf8").toString("base64")); - expect(rollbackScript).toContain("Start-Process -FilePath $previousWatcher"); + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(true); + expect(boundary.state.daemonRunning).toBe(false); + expect(boundary.state.events.at(-1)).toBe("restore"); }); it("prints a direct recovery command when prior Windows state rollback fails", () => { const priorHost = "127.0.0.1:11434"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; - const run = vi.fn((command: string[]) => { - const script = commandText(command); - const capturesHostSnapshot = script.includes("ConvertTo-Json -Compress"); - const persistsNewBinding = script.includes( - "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", - ); - const restoresPriorState = script.includes("$previousHost ="); - return capturesHostSnapshot - ? hostSnapshotRun(priorHost, null, daemonPath) - : persistsNewBinding - ? successfulRun() - : restoresPriorState - ? { status: 1, stderr: "rollback denied" } - : { status: 1, stderr: "launch unavailable" }; + const boundary = createWindowsPowerShellBoundary({ + userHost: priorHost, + watcherPath: null, + daemonPath, + rollbackStatus: 1, }); - const runCapture = vi.fn(() => ""); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(boundary.run, boundary.runCapture); try { expect(windows.setupWindowsOllamaWith0000Binding()).toBe(false); @@ -243,13 +328,42 @@ describe("Windows Ollama helper", () => { const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); errorSpy.mockRestore(); - const encodedCommand = diagnostic.match(/-EncodedCommand ([A-Za-z0-9+/=]+)/u)?.[1] ?? ""; - const recoveryScript = Buffer.from(encodedCommand, "base64").toString("utf16le"); expect(diagnostic).toContain("Failed to restore the previous Windows Ollama state"); expect(diagnostic).toContain(`Previous User-scope OLLAMA_HOST: ${JSON.stringify(priorHost)}`); + expect(diagnostic).toContain("Restore it and relaunch the previous Ollama process with:"); expect(diagnostic).toContain("powershell.exe -NoProfile -EncodedCommand"); - expect(recoveryScript).toContain("SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')"); - expect(recoveryScript).toContain("Start-Process -FilePath $previousDaemon"); + }); + + it("stops a final unready replacement before restoring the prior Windows state", () => { + const priorHost = "127.0.0.1:11434"; + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + const boundary = createWindowsPowerShellBoundary({ + userHost: priorHost, + watcherPath, + daemonPath, + launchStatuses: [1, 1, 0], + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(boundary.run, boundary.runCapture); + + try { + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + const finalLaunch = boundary.state.events.lastIndexOf("launch"); + const finalStop = boundary.state.events.lastIndexOf("stop-daemon"); + const rollback = boundary.state.events.lastIndexOf("restore"); + expect(finalLaunch).toBeLessThan(finalStop); + expect(finalStop).toBeLessThan(rollback); + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(true); + expect(boundary.state.daemonRunning).toBe(false); }); it("isolates Docker credentials while waiting for the Windows-host daemon", () => { From 802e3531d353e75d403c240550eccb2f8097dace Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 05:19:41 -0700 Subject: [PATCH 22/48] fix(ollama): restore Windows state on interrupt Signed-off-by: Prekshi Vyas --- .../agent/passthrough-ollama-recovery.test.ts | 16 +- .../agent/passthrough-ollama-recovery.ts | 10 +- src/lib/inference/ollama/windows.test.ts | 236 ++++++++++-------- src/lib/inference/ollama/windows.ts | 131 +++++++--- 4 files changed, 249 insertions(+), 144 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 0443377ded5..02a293bc441 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -72,7 +72,9 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("at http://host.docker.internal:11434"); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); - expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).toContain( + "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + ); expect(stderr).not.toContain("rerun this command"); }); @@ -102,7 +104,9 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("http://host.docker.internal:11434"); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); - expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).toContain( + "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + ); expect(stderr).not.toContain("rerun this command"); }); @@ -142,7 +146,9 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("qwen3.6:35b"); expect(stderr).toContain("Restore Ollama access"); expect(stderr).toContain("confirm that it serves"); - expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).toContain( + "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + ); expect(stderr).not.toContain("rerun this command"); }); @@ -227,7 +233,9 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("OPENAI_API_KEY="); expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("Restore Ollama access to that endpoint"); - expect(stderr).toContain("NemoClaw will retry the warm-up before the next agent command"); + expect(stderr).toContain( + "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + ); expect(stderr).not.toContain("rerun this command"); expect(stderr).not.toContain(exposedToken); expect(stderr).not.toContain("END-OF-DETAIL"); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 71cdc3f1764..168da34a6f3 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -64,8 +64,8 @@ function reportRecovery( proc.stderr.write( ` Ollama warm-up for '${model}' at ${endpoint} ${describeWarmFailure(result.reason)} ` + `(${detail}). OpenClaw dispatch will continue. Restore Ollama access to ${endpoint} ` + - `and confirm that it serves '${model}'. NemoClaw will retry the warm-up before the ` + - `next agent command.\n`, + `and confirm that it serves '${model}'. NemoClaw will check the model before the next ` + + `OpenClaw agent command and warm it if necessary.\n`, ); return; } @@ -104,7 +104,8 @@ function reportRecovery( proc.stderr.write( ` Ollama at ${endpoint} was unreachable while checking '${model}'; continuing to ` + `OpenClaw dispatch. Restore Ollama access to ${endpoint}, confirm that it serves ` + - `'${model}'. NemoClaw will retry the warm-up before the next agent command.\n`, + `'${model}'. NemoClaw will check the model before the next OpenClaw agent command ` + + `and warm it if necessary.\n`, ); break; } @@ -141,7 +142,8 @@ export async function runOllamaRestartRecovery( proc.stderr.write( ` Ollama restart recovery for '${model}' ${endpoint} failed unexpectedly: ${detail}. ` + `OpenClaw dispatch will continue. Restore Ollama access to that endpoint, confirm it ` + - `serves '${model}'. NemoClaw will retry the warm-up before the next agent command.\n`, + `serves '${model}'. NemoClaw will check the model before the next OpenClaw agent ` + + `command and warm it if necessary.\n`, ); } return null; diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index ce66e695fe9..d7228b0ec20 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -47,112 +47,82 @@ function hostSnapshotRun( return successfulRun(JSON.stringify({ userHost, watcherPath, daemonPath })); } -type WindowsPowerShellState = { +type WindowsSetupState = { userHost: string | null; watcherRunning: boolean; daemonRunning: boolean; events: string[]; }; -function createWindowsPowerShellBoundary(options: { +class PreservedWindowsInterrupt extends Error { + constructor(readonly signal: NodeJS.Signals) { + super(`preserved ${signal}`); + } +} + +function createWindowsSetupBoundary(options: { userHost: string | null; watcherPath: string | null; daemonPath: string | null; - launchStatuses?: number[]; - readinessResponse?: string; + replacementRemainsRunning?: boolean; rollbackStatus?: number; + interruptSignal?: "SIGINT" | "SIGTERM"; }) { - const original = { + const snapshot = { userHost: options.userHost, watcherPath: options.watcherPath, daemonPath: options.daemonPath, }; - const state: WindowsPowerShellState = { - userHost: original.userHost, - watcherRunning: Boolean(original.watcherPath), - daemonRunning: Boolean(original.daemonPath), + const state: WindowsSetupState = { + userHost: snapshot.userHost, + watcherRunning: Boolean(snapshot.watcherPath), + daemonRunning: Boolean(snapshot.daemonPath), events: [], }; - const launchStatuses = options.launchStatuses ?? [1, 1, 1]; const rollbackStatus = options.rollbackStatus ?? 0; - let launchIndex = 0; - const run = vi.fn((command: string[]) => { - const script = commandText(command); - const capturesHostSnapshot = script.includes("ConvertTo-Json -Compress"); - const persistsNewBinding = script.includes( - "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", - ); - const restoresPriorState = script.includes("$previousHost ="); - const launchesReplacement = !restoresPriorState && script.includes("Start-Process"); - const rollbackStopsWatcher = restoresPriorState && script.includes("Get-Process 'ollama app'"); - const rollbackStopsDaemon = restoresPriorState && script.includes("Get-Process ollama -EA"); - const launchStatus = launchStatuses[launchIndex] ?? 1; - const launchesWatcher = - launchesReplacement && - Boolean(original.watcherPath) && - script.includes(String(original.watcherPath)); - const event = capturesHostSnapshot - ? "snapshot" - : persistsNewBinding - ? "persist" - : restoresPriorState - ? "restore" - : launchesReplacement - ? "launch" - : "unexpected"; - state.events.push( - ...(restoresPriorState - ? [ - ...(rollbackStopsWatcher ? ["stop-watcher"] : []), - ...(rollbackStopsDaemon ? ["stop-daemon"] : []), - "restore", - ] - : [event]), - ); - state.watcherRunning = rollbackStopsWatcher ? false : state.watcherRunning; - state.daemonRunning = rollbackStopsDaemon ? false : state.daemonRunning; - state.userHost = persistsNewBinding - ? "0.0.0.0:11434" - : restoresPriorState && rollbackStatus === 0 - ? original.userHost - : state.userHost; - state.watcherRunning = - restoresPriorState && rollbackStatus === 0 - ? Boolean(original.watcherPath) - : launchesWatcher && launchStatus === 0 - ? true - : state.watcherRunning; - state.daemonRunning = - restoresPriorState && rollbackStatus === 0 - ? Boolean(original.daemonPath) && !original.watcherPath - : launchesReplacement && launchStatus === 0 && !launchesWatcher - ? true - : state.daemonRunning; - launchIndex += launchesReplacement ? 1 : 0; - return capturesHostSnapshot - ? hostSnapshotRun(original.userHost, original.watcherPath, original.daemonPath) - : persistsNewBinding - ? successfulRun() - : restoresPriorState - ? rollbackStatus === 0 - ? successfulRun() - : { status: rollbackStatus, stderr: "rollback denied" } - : launchesReplacement - ? launchStatus === 0 - ? successfulRun() - : { status: launchStatus, stderr: "launch unavailable" } - : { status: 1, stderr: "unexpected PowerShell operation" }; - }); - const runCapture = vi.fn((command: string | string[]) => { - const script = commandText(command); - const stopsWatcher = script.includes("Get-Process 'ollama app'"); - const stopsDaemon = script.includes("Get-Process ollama -EA"); - state.watcherRunning = stopsWatcher ? false : state.watcherRunning; - state.daemonRunning = stopsDaemon ? false : state.daemonRunning; - state.events.push(stopsWatcher ? "stop-watcher" : stopsDaemon ? "stop-daemon" : "probe"); - return isDockerTagsRequest(command) ? (options.readinessResponse ?? "") : ""; - }); - return { run, runCapture, state }; + let interruptHandler = (_signal: "SIGINT" | "SIGTERM") => {}; + const operations = { + captureSnapshot: vi.fn(() => { + state.events.push("snapshot"); + return snapshot; + }), + persistBinding: vi.fn(() => { + state.events.push("persist"); + state.userHost = "0.0.0.0:11434"; + return true; + }), + stopProcesses: vi.fn(() => { + state.events.push("stop-existing"); + state.watcherRunning = false; + state.daemonRunning = false; + }), + wait: vi.fn(), + launch: vi.fn(() => { + state.events.push("launch"); + state.daemonRunning = options.replacementRemainsRunning ?? false; + options.interruptSignal ? interruptHandler(options.interruptSignal) : undefined; + return false; + }), + rollbackSnapshot: vi.fn(() => { + state.events.push("stop-replacement", "restore"); + const restored = rollbackStatus === 0; + state.userHost = restored ? snapshot.userHost : state.userHost; + state.watcherRunning = restored ? Boolean(snapshot.watcherPath) : false; + state.daemonRunning = restored + ? Boolean(snapshot.daemonPath) && !snapshot.watcherPath + : false; + return restored; + }), + registerInterruptHandler: vi.fn((handler: (signal: "SIGINT" | "SIGTERM") => void) => { + interruptHandler = handler; + return vi.fn(); + }), + preserveInterrupt: vi.fn((signal: "SIGINT" | "SIGTERM") => { + state.events.push(`signal:${signal}`); + throw new PreservedWindowsInterrupt(signal); + }), + }; + return { operations, state }; } function loadWindowsOllamaWithMocks( @@ -283,17 +253,22 @@ describe("Windows Ollama helper", () => { const priorHost = "127.0.0.1:11434"; const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; - const boundary = createWindowsPowerShellBoundary({ + const boundary = createWindowsSetupBoundary({ userHost: priorHost, watcherPath, daemonPath, }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(boundary.run, boundary.runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); + expect( + windows.setupWindowsOllamaWith0000Binding( + { installedPath: daemonPath }, + boundary.operations, + ), + ).toBe(false); } finally { restore(); logSpy.mockRestore(); @@ -306,10 +281,10 @@ describe("Windows Ollama helper", () => { expect(boundary.state.events.at(-1)).toBe("restore"); }); - it("prints a direct recovery command when prior Windows state rollback fails", () => { - const priorHost = "127.0.0.1:11434"; + it("redacts the prior binding from a failed Windows rollback diagnostic", () => { + const priorHost = "https://operator:private-token@ollama.example:11434"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; - const boundary = createWindowsPowerShellBoundary({ + const boundary = createWindowsSetupBoundary({ userHost: priorHost, watcherPath: null, daemonPath, @@ -317,10 +292,10 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(boundary.run, boundary.runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - expect(windows.setupWindowsOllamaWith0000Binding()).toBe(false); + expect(windows.setupWindowsOllamaWith0000Binding({}, boundary.operations)).toBe(false); } finally { restore(); logSpy.mockRestore(); @@ -329,27 +304,34 @@ describe("Windows Ollama helper", () => { const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); errorSpy.mockRestore(); expect(diagnostic).toContain("Failed to restore the previous Windows Ollama state"); - expect(diagnostic).toContain(`Previous User-scope OLLAMA_HOST: ${JSON.stringify(priorHost)}`); - expect(diagnostic).toContain("Restore it and relaunch the previous Ollama process with:"); - expect(diagnostic).toContain("powershell.exe -NoProfile -EncodedCommand"); + expect(diagnostic).toContain("restore your previous User-scope OLLAMA_HOST value"); + expect(diagnostic).toContain("relaunch the previous Ollama app or daemon"); + expect(diagnostic).not.toContain(priorHost); + expect(diagnostic).not.toContain("private-token"); + expect(diagnostic).not.toContain(Buffer.from(priorHost, "utf8").toString("base64")); }); it("stops a final unready replacement before restoring the prior Windows state", () => { const priorHost = "127.0.0.1:11434"; const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; - const boundary = createWindowsPowerShellBoundary({ + const boundary = createWindowsSetupBoundary({ userHost: priorHost, watcherPath, daemonPath, - launchStatuses: [1, 1, 0], + replacementRemainsRunning: true, }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(boundary.run, boundary.runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); + expect( + windows.setupWindowsOllamaWith0000Binding( + { installedPath: daemonPath }, + boundary.operations, + ), + ).toBe(false); } finally { restore(); logSpy.mockRestore(); @@ -357,7 +339,7 @@ describe("Windows Ollama helper", () => { } const finalLaunch = boundary.state.events.lastIndexOf("launch"); - const finalStop = boundary.state.events.lastIndexOf("stop-daemon"); + const finalStop = boundary.state.events.lastIndexOf("stop-replacement"); const rollback = boundary.state.events.lastIndexOf("restore"); expect(finalLaunch).toBeLessThan(finalStop); expect(finalStop).toBeLessThan(rollback); @@ -366,6 +348,52 @@ describe("Windows Ollama helper", () => { expect(boundary.state.daemonRunning).toBe(false); }); + it.each(["SIGINT", "SIGTERM"] as const)( + "restores the prior Windows state before preserving %s", + (signal) => { + const priorHost = "127.0.0.1:11434"; + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + const boundary = createWindowsSetupBoundary({ + userHost: priorHost, + watcherPath, + daemonPath, + replacementRemainsRunning: true, + interruptSignal: signal, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + expect(() => + windows.setupWindowsOllamaWith0000Binding( + { installedPath: daemonPath }, + boundary.operations, + ), + ).toThrow(PreservedWindowsInterrupt); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(boundary.state.events).toEqual([ + "snapshot", + "persist", + "stop-existing", + "launch", + "stop-replacement", + "restore", + `signal:${signal}`, + ]); + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(true); + expect(boundary.state.daemonRunning).toBe(false); + expect(boundary.operations.preserveInterrupt).toHaveBeenCalledWith(signal); + }, + ); + it("isolates Docker credentials while waiting for the Windows-host daemon", () => { const run = vi.fn(); const cleanup = vi.fn(() => ({ ok: true as const })); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 3f043da37cc..78d658804d2 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -150,24 +150,17 @@ function buildWindowsOllamaRestoreScript(snapshot: WindowsOllamaHostSnapshot): s return script.join("; "); } -function previousOllamaHostLabel(value: string | null): string { - if (value === null) return "unset"; - const terminalSafe = value.replace(/[\u202A-\u202E\u2066-\u2069]/gu, "").slice(0, 200); - return JSON.stringify(terminalSafe); +function rollbackWindowsOllamaHostSnapshot(snapshot: WindowsOllamaHostSnapshot): boolean { + const script = buildWindowsOllamaRestoreScript(snapshot); + return runWindowsOllamaStateScript(script); } -function restoreWindowsOllamaHostSnapshot(snapshot: WindowsOllamaHostSnapshot): boolean { - const script = buildWindowsOllamaRestoreScript(snapshot); - if (runWindowsOllamaStateScript(script)) return true; - const encodedCommand = Buffer.from( - `$ErrorActionPreference='Stop'; ${script}`, - "utf16le", - ).toString("base64"); +function reportWindowsOllamaRollbackFailure(): void { console.error(" Failed to restore the previous Windows Ollama state."); - console.error(` Previous User-scope OLLAMA_HOST: ${previousOllamaHostLabel(snapshot.userHost)}`); - console.error(" Restore it and relaunch the previous Ollama process with:"); - console.error(` powershell.exe -NoProfile -EncodedCommand ${encodedCommand}`); - return false; + console.error( + " In Windows PowerShell, stop Ollama, restore your previous User-scope OLLAMA_HOST " + + "value, and relaunch the previous Ollama app or daemon.", + ); } // Order matters: kill 'ollama app' (the tray watcher) before 'ollama' @@ -277,35 +270,109 @@ function launchAndAwaitWindowsOllama( return false; } +type WindowsOllamaInterruptSignal = "SIGINT" | "SIGTERM"; + +type WindowsOllamaSetupOperations = { + captureSnapshot: () => WindowsOllamaHostSnapshot | null; + persistBinding: () => boolean; + stopProcesses: () => void; + wait: (seconds: number) => void; + launch: (opts: { watcherPath?: string; installedPath?: string }) => boolean; + rollbackSnapshot: (snapshot: WindowsOllamaHostSnapshot) => boolean; + registerInterruptHandler: (handler: (signal: WindowsOllamaInterruptSignal) => void) => () => void; + preserveInterrupt: (signal: WindowsOllamaInterruptSignal) => void; +}; + +function registerWindowsOllamaInterruptHandler( + handler: (signal: WindowsOllamaInterruptSignal) => void, +): () => void { + const onSigint = () => handler("SIGINT"); + const onSigterm = () => handler("SIGTERM"); + process.once("SIGINT", onSigint); + process.once("SIGTERM", onSigterm); + return () => { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + }; +} + +const WINDOWS_OLLAMA_SETUP_OPERATIONS: WindowsOllamaSetupOperations = { + captureSnapshot: captureWindowsOllamaHostSnapshot, + persistBinding: persistOllamaHostEnvVar, + stopProcesses: killWindowsOllamaProcesses, + wait: sleepSeconds, + launch: launchAndAwaitWindowsOllama, + rollbackSnapshot: rollbackWindowsOllamaHostSnapshot, + registerInterruptHandler: registerWindowsOllamaInterruptHandler, + preserveInterrupt: (signal) => { + process.kill(process.pid, signal); + }, +}; + +function rollbackWindowsOllamaSetup( + snapshot: WindowsOllamaHostSnapshot, + operations: WindowsOllamaSetupOperations, +): void { + if (!operations.rollbackSnapshot(snapshot)) reportWindowsOllamaRollbackFailure(); +} + // Used by start and restart paths to force a 0.0.0.0 binding on an already // installed Ollama. Fresh install fallback passes installedPath to avoid // relying on a newly-mutated Windows PATH from this process. function setupWindowsOllamaWith0000Binding( opts: { announceStop?: boolean; installedPath?: string } = {}, + operations: WindowsOllamaSetupOperations = WINDOWS_OLLAMA_SETUP_OPERATIONS, ): boolean { - const snapshot = captureWindowsOllamaHostSnapshot(); + const snapshot = operations.captureSnapshot(); if (!snapshot) { console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); return false; } - if (!persistOllamaHostEnvVar()) { - console.error( - " Could not persist the Windows Ollama host binding; leaving processes running.", - ); + let rollbackRequired = false; + let removeInterruptHandler = () => {}; + const handleInterrupt = (signal: WindowsOllamaInterruptSignal) => { + const shouldRollback = rollbackRequired; + rollbackRequired = false; + try { + if (shouldRollback) rollbackWindowsOllamaSetup(snapshot, operations); + } finally { + removeInterruptHandler(); + operations.preserveInterrupt(signal); + } + }; + removeInterruptHandler = operations.registerInterruptHandler(handleInterrupt); + try { + if (!operations.persistBinding()) { + console.error( + " Could not persist the Windows Ollama host binding; leaving processes running.", + ); + return false; + } + rollbackRequired = true; + if (opts.announceStop) { + console.log(" Stopping existing Ollama on Windows host..."); + } + operations.stopProcesses(); + operations.wait(1); + const launched = operations.launch({ + watcherPath: snapshot.watcherPath || undefined, + installedPath: opts.installedPath, + }); + if (launched) { + rollbackRequired = false; + return true; + } + rollbackRequired = false; + rollbackWindowsOllamaSetup(snapshot, operations); return false; + } catch (error) { + const shouldRollback = rollbackRequired; + rollbackRequired = false; + if (shouldRollback) rollbackWindowsOllamaSetup(snapshot, operations); + throw error; + } finally { + removeInterruptHandler(); } - if (opts.announceStop) { - console.log(" Stopping existing Ollama on Windows host..."); - } - killWindowsOllamaProcesses(); - sleepSeconds(1); - const launched = launchAndAwaitWindowsOllama({ - watcherPath: snapshot.watcherPath || undefined, - installedPath: opts.installedPath, - }); - if (launched) return true; - restoreWindowsOllamaHostSnapshot(snapshot); - return false; } function switchToWindowsOllamaHost(): void { From 6690fb96d32c7d133713b1c84fc98c246e63e072 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 05:36:48 -0700 Subject: [PATCH 23/48] refactor(ollama): name Windows launch attempts Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 107 +++++++++++++---------- src/lib/inference/ollama/windows.ts | 60 ++++++++++--- 2 files changed, 105 insertions(+), 62 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index d7228b0ec20..e6a22141fc7 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -64,7 +64,8 @@ function createWindowsSetupBoundary(options: { userHost: string | null; watcherPath: string | null; daemonPath: string | null; - replacementRemainsRunning?: boolean; + launchStatuses?: number[]; + readinessResults?: boolean[]; rollbackStatus?: number; interruptSignal?: "SIGINT" | "SIGTERM"; }) { @@ -79,7 +80,11 @@ function createWindowsSetupBoundary(options: { daemonRunning: Boolean(snapshot.daemonPath), events: [], }; + const launchStatuses = options.launchStatuses ?? [1, 1, 1]; + const readinessResults = options.readinessResults ?? []; const rollbackStatus = options.rollbackStatus ?? 0; + let launchIndex = 0; + let readinessIndex = 0; let interruptHandler = (_signal: "SIGINT" | "SIGTERM") => {}; const operations = { captureSnapshot: vi.fn(() => { @@ -97,12 +102,33 @@ function createWindowsSetupBoundary(options: { state.daemonRunning = false; }), wait: vi.fn(), - launch: vi.fn(() => { - state.events.push("launch"); - state.daemonRunning = options.replacementRemainsRunning ?? false; - options.interruptSignal ? interruptHandler(options.interruptSignal) : undefined; - return false; - }), + launchOperations: { + runAttempt: vi.fn((attempt: { kind: "watcher" | "installed" | "path" }) => { + const status = launchStatuses[launchIndex] ?? 1; + launchIndex += 1; + state.events.push(`launch:${attempt.kind}`); + state.daemonRunning = status === 0; + options.interruptSignal ? interruptHandler(options.interruptSignal) : undefined; + return status === 0 + ? successfulRun() + : { status, stderr: `${attempt.kind} launch unavailable` }; + }), + awaitReady: vi.fn(() => { + const ready = readinessResults[readinessIndex] ?? false; + readinessIndex += 1; + state.events.push("readiness"); + ready + ? require(LOCAL_INFERENCE_PATH).setResolvedOllamaHost("host.docker.internal") + : undefined; + return ready; + }), + stopProcesses: vi.fn(() => { + state.events.push("stop-launch-attempt"); + state.watcherRunning = false; + state.daemonRunning = false; + }), + wait: vi.fn(), + }, rollbackSnapshot: vi.fn(() => { state.events.push("stop-replacement", "restore"); const restored = rollbackStatus === 0; @@ -195,58 +221,42 @@ describe("Windows Ollama helper", () => { } }); - it("falls back from a stale watcher path and checks readiness from Docker Desktop (#8127)", () => { + it("falls back from a stale watcher to the verified executable and selects Windows route", () => { const watcherPath = "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama app.exe"; const installedPath = "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe"; - let watcherLaunchAttempted = false; - let installedLaunchAttempted = false; - let dockerReadinessObserved = false; - - const run = vi.fn((command: string[]) => { - const launch = commandText(command); - const capturesHostSnapshot = launch.includes("ConvertTo-Json -Compress"); - const persistsNewBinding = launch.includes( - "SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User')", - ); - const isWatcherLaunch = launch.includes(watcherPath); - const isInstalledLaunch = launch.includes(installedPath); - watcherLaunchAttempted ||= isWatcherLaunch; - installedLaunchAttempted ||= isInstalledLaunch; - return capturesHostSnapshot - ? hostSnapshotRun("127.0.0.1:11434", watcherPath, installedPath) - : persistsNewBinding - ? successfulRun() - : isWatcherLaunch - ? { status: 1, stderr: "stale watcher path" } - : isInstalledLaunch - ? successfulRun() - : { status: 1, stderr: "unexpected launch target" }; - }); - const runCapture = vi.fn((command: string | string[]) => { - const probesDockerReadiness = isDockerTagsRequest(command); - dockerReadinessObserved ||= probesDockerReadiness; - return probesDockerReadiness && installedLaunchAttempted - ? JSON.stringify({ models: [] }) - : ""; + const boundary = createWindowsSetupBoundary({ + userHost: "127.0.0.1:11434", + watcherPath, + daemonPath: installedPath, + launchStatuses: [1, 0], + readinessResults: [true], }); const localInference = require(LOCAL_INFERENCE_PATH); localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); + expect( + windows.setupWindowsOllamaWith0000Binding({ installedPath }, boundary.operations), + ).toBe(true); + expect(boundary.state.events).toEqual([ + "snapshot", + "persist", + "stop-existing", + "launch:watcher", + "stop-launch-attempt", + "launch:installed", + "readiness", + ]); + expect(localInference.getResolvedOllamaHost()).toBe("host.docker.internal"); } finally { localInference.resetOllamaHostCache(); restore(); logSpy.mockRestore(); errorSpy.mockRestore(); } - - expect(watcherLaunchAttempted).toBe(true); - expect(installedLaunchAttempted).toBe(true); - expect(dockerReadinessObserved).toBe(true); }); it("restores the prior binding and watcher when every rebound launch fails", () => { @@ -319,7 +329,8 @@ describe("Windows Ollama helper", () => { userHost: priorHost, watcherPath, daemonPath, - replacementRemainsRunning: true, + launchStatuses: [1, 1, 0], + readinessResults: [false], }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); @@ -338,7 +349,7 @@ describe("Windows Ollama helper", () => { errorSpy.mockRestore(); } - const finalLaunch = boundary.state.events.lastIndexOf("launch"); + const finalLaunch = boundary.state.events.lastIndexOf("launch:path"); const finalStop = boundary.state.events.lastIndexOf("stop-replacement"); const rollback = boundary.state.events.lastIndexOf("restore"); expect(finalLaunch).toBeLessThan(finalStop); @@ -358,7 +369,7 @@ describe("Windows Ollama helper", () => { userHost: priorHost, watcherPath, daemonPath, - replacementRemainsRunning: true, + launchStatuses: [0], interruptSignal: signal, }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); @@ -382,7 +393,7 @@ describe("Windows Ollama helper", () => { "snapshot", "persist", "stop-existing", - "launch", + "launch:watcher", "stop-replacement", "restore", `signal:${signal}`, diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 78d658804d2..e5497dc8750 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -212,18 +212,48 @@ function awaitWindowsOllamaReady(opts: { prepareDockerEnvironment?: () => unknow return false; } +type WindowsOllamaLaunchAttempt = { + kind: "watcher" | "installed" | "path"; + label: string; + script: string; +}; + +type WindowsOllamaLaunchOperations = { + runAttempt: (attempt: WindowsOllamaLaunchAttempt) => { + status: number | null; + stderr?: string; + error?: Error; + }; + awaitReady: () => boolean; + stopProcesses: () => void; + wait: (seconds: number) => void; +}; + +const WINDOWS_OLLAMA_LAUNCH_OPERATIONS: WindowsOllamaLaunchOperations = { + runAttempt: (attempt) => + run(["powershell.exe", "-Command", attempt.script], { + ignoreError: true, + suppressOutput: true, + }), + awaitReady: awaitWindowsOllamaReady, + stopProcesses: killWindowsOllamaProcesses, + wait: sleepSeconds, +}; + // Relaunch via the watcher path when available so the tray icon and the // watcher's auto-restart survive; fall back through the verified installed // path and finally refreshed PATH because stale watcher paths are possible. function launchAndAwaitWindowsOllama( opts: { watcherPath?: string; installedPath?: string } = {}, + operations: WindowsOllamaLaunchOperations = WINDOWS_OLLAMA_LAUNCH_OPERATIONS, ): boolean { console.log(" Starting Ollama on Windows host via WSL interop..."); const watcherPath = typeof opts.watcherPath === "string" ? opts.watcherPath.trim() : ""; const installedPath = typeof opts.installedPath === "string" ? opts.installedPath.trim() : ""; - const launchAttempts: Array<{ label: string; script: string }> = []; + const launchAttempts: WindowsOllamaLaunchAttempt[] = []; if (watcherPath) { launchAttempts.push({ + kind: "watcher", label: "Ollama tray app", script: `$env:OLLAMA_HOST='0.0.0.0:11434'; Start-Process -FilePath ${psSingleQuote(watcherPath)} ` + @@ -232,6 +262,7 @@ function launchAndAwaitWindowsOllama( } if (installedPath) { launchAttempts.push({ + kind: "installed", label: "verified ollama.exe", script: `$env:OLLAMA_HOST='0.0.0.0:11434'; Start-Process -FilePath ${psSingleQuote(installedPath)} ` + @@ -239,6 +270,7 @@ function launchAndAwaitWindowsOllama( }); } launchAttempts.push({ + kind: "path", label: "refreshed Windows PATH", script: "$env:PATH = [Environment]::GetEnvironmentVariable('PATH','Machine') + ';' + [Environment]::GetEnvironmentVariable('PATH','User'); " + @@ -247,11 +279,8 @@ function launchAndAwaitWindowsOllama( for (let i = 0; i < launchAttempts.length; i++) { const attempt = launchAttempts[i]; - const result = run(["powershell.exe", "-Command", attempt.script], { - ignoreError: true, - suppressOutput: true, - }); - if (result.status === 0 && awaitWindowsOllamaReady()) { + const result = operations.runAttempt(attempt); + if (result.status === 0 && operations.awaitReady()) { return true; } @@ -263,8 +292,8 @@ function launchAndAwaitWindowsOllama( : error || `exit ${result.status}${stderr ? `: ${stderr}` : ""}`; console.error(` PowerShell launch via ${attempt.label} failed: ${detail}`); if (i < launchAttempts.length - 1) { - killWindowsOllamaProcesses(); - sleepSeconds(1); + operations.stopProcesses(); + operations.wait(1); } } return false; @@ -277,7 +306,7 @@ type WindowsOllamaSetupOperations = { persistBinding: () => boolean; stopProcesses: () => void; wait: (seconds: number) => void; - launch: (opts: { watcherPath?: string; installedPath?: string }) => boolean; + launchOperations: WindowsOllamaLaunchOperations; rollbackSnapshot: (snapshot: WindowsOllamaHostSnapshot) => boolean; registerInterruptHandler: (handler: (signal: WindowsOllamaInterruptSignal) => void) => () => void; preserveInterrupt: (signal: WindowsOllamaInterruptSignal) => void; @@ -301,7 +330,7 @@ const WINDOWS_OLLAMA_SETUP_OPERATIONS: WindowsOllamaSetupOperations = { persistBinding: persistOllamaHostEnvVar, stopProcesses: killWindowsOllamaProcesses, wait: sleepSeconds, - launch: launchAndAwaitWindowsOllama, + launchOperations: WINDOWS_OLLAMA_LAUNCH_OPERATIONS, rollbackSnapshot: rollbackWindowsOllamaHostSnapshot, registerInterruptHandler: registerWindowsOllamaInterruptHandler, preserveInterrupt: (signal) => { @@ -354,10 +383,13 @@ function setupWindowsOllamaWith0000Binding( } operations.stopProcesses(); operations.wait(1); - const launched = operations.launch({ - watcherPath: snapshot.watcherPath || undefined, - installedPath: opts.installedPath, - }); + const launched = launchAndAwaitWindowsOllama( + { + watcherPath: snapshot.watcherPath || undefined, + installedPath: opts.installedPath, + }, + operations.launchOperations, + ); if (launched) { rollbackRequired = false; return true; From 957776b606c38d39d8ec98a182705b0a30d6980b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 05:53:37 -0700 Subject: [PATCH 24/48] refactor(inference): reuse validated doctor inventory Signed-off-by: Prekshi Vyas --- .../agent/passthrough-ollama-recovery.test.ts | 8 ++--- .../agent/passthrough-ollama-recovery.ts | 10 +++---- .../sandbox/doctor-system-checks.test.ts | 2 +- .../actions/sandbox/doctor-system-checks.ts | 15 +++------- src/lib/inference/health.ts | 30 +++++-------------- 5 files changed, 21 insertions(+), 44 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 02a293bc441..6a8361b6bda 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -73,7 +73,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); expect(stderr).toContain( - "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + "NemoClaw will check the model before the next `nemoclaw agent` command and warm it if necessary", ); expect(stderr).not.toContain("rerun this command"); }); @@ -105,7 +105,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); expect(stderr).toContain( - "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + "NemoClaw will check the model before the next `nemoclaw agent` command and warm it if necessary", ); expect(stderr).not.toContain("rerun this command"); }); @@ -147,7 +147,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("Restore Ollama access"); expect(stderr).toContain("confirm that it serves"); expect(stderr).toContain( - "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + "NemoClaw will check the model before the next `nemoclaw agent` command and warm it if necessary", ); expect(stderr).not.toContain("rerun this command"); }); @@ -234,7 +234,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).toContain("OpenClaw dispatch will continue"); expect(stderr).toContain("Restore Ollama access to that endpoint"); expect(stderr).toContain( - "NemoClaw will check the model before the next OpenClaw agent command and warm it if necessary", + "NemoClaw will check the model before the next `nemoclaw agent` command and warm it if necessary", ); expect(stderr).not.toContain("rerun this command"); expect(stderr).not.toContain(exposedToken); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 168da34a6f3..1e8b1a622ff 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -65,7 +65,7 @@ function reportRecovery( ` Ollama warm-up for '${model}' at ${endpoint} ${describeWarmFailure(result.reason)} ` + `(${detail}). OpenClaw dispatch will continue. Restore Ollama access to ${endpoint} ` + `and confirm that it serves '${model}'. NemoClaw will check the model before the next ` + - `OpenClaw agent command and warm it if necessary.\n`, + `\`nemoclaw agent\` command and warm it if necessary.\n`, ); return; } @@ -104,8 +104,8 @@ function reportRecovery( proc.stderr.write( ` Ollama at ${endpoint} was unreachable while checking '${model}'; continuing to ` + `OpenClaw dispatch. Restore Ollama access to ${endpoint}, confirm that it serves ` + - `'${model}'. NemoClaw will check the model before the next OpenClaw agent command ` + - `and warm it if necessary.\n`, + `'${model}'. NemoClaw will check the model before the next ` + + `\`nemoclaw agent\` command and warm it if necessary.\n`, ); break; } @@ -142,8 +142,8 @@ export async function runOllamaRestartRecovery( proc.stderr.write( ` Ollama restart recovery for '${model}' ${endpoint} failed unexpectedly: ${detail}. ` + `OpenClaw dispatch will continue. Restore Ollama access to that endpoint, confirm it ` + - `serves '${model}'. NemoClaw will check the model before the next OpenClaw agent ` + - `command and warm it if necessary.\n`, + `serves '${model}'. NemoClaw will check the model before the next ` + + `\`nemoclaw agent\` command and warm it if necessary.\n`, ); } return null; diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts index 0331d1c3aa6..14bc626bae9 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.test.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -102,7 +102,7 @@ describe("doctor system checks", () => { group: "Local services", label: "Ollama", status: "fail", - detail: "invalid response from http://127.0.0.1:11434/api/tags", + detail: "not reachable or invalid response at http://127.0.0.1:11434/api/tags", hint: "start Ollama or change the sandbox inference provider", }); }); diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index 8d4fae1b436..53ebb370514 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -178,30 +178,23 @@ export function ollamaDoctorCheck( currentProvider: string, deps: OllamaDoctorCheckDeps = {}, ): DoctorCheck { - const { endpoint, output, valid } = probeOllamaHostInventory(deps); + const { endpoint, inventory } = probeOllamaHostInventory(deps); const required = currentProvider === "ollama-local"; - if (!output || !valid) { + if (inventory === null) { return { group: "Local services", label: "Ollama", status: required ? "fail" : "info", - detail: output ? `invalid response from ${endpoint}` : `not reachable at ${endpoint}`, + detail: `not reachable or invalid response at ${endpoint}`, hint: required ? "start Ollama or change the sandbox inference provider" : undefined, }; } - let modelCount = "unknown model count"; - try { - const parsed = JSON.parse(output); - if (Array.isArray(parsed.models)) modelCount = `${parsed.models.length} model(s)`; - } catch { - /* keep generic detail */ - } return { group: "Local services", label: "Ollama", status: "ok", - detail: `reachable at ${endpoint} (${modelCount})`, + detail: `reachable at ${endpoint} (${inventory.length} model(s))`, }; } diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index af541fb1cd8..db25d68ac81 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -10,18 +10,16 @@ */ import { createBearerAuthConfig, createXApiKeyAuthConfig } from "../adapters/http/auth-config"; -import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; import type { CurlProbeOptions, CurlProbeResult } from "../adapters/http/probe"; import { runCurlProbe } from "../adapters/http/probe"; import { normalizeCredentialValue, resolveProviderCredential } from "../credentials/store"; import { getProviderSelectionConfig } from "./config"; import { - createOllamaApiCapture, getResolvedOllamaHost, - isValidOllamaTagsResponseBody, loadPersistedOllamaHost, type LocalProviderHealthProbeOptions, OLLAMA_PORT, + probeOllamaEndpointInventory, probeLocalProviderHealth, type RunCaptureFn, } from "./local"; @@ -66,39 +64,25 @@ export interface ProviderHealthProbeOptions { export type OllamaHostInventoryProbeOptions = { getOllamaHost?: () => string; runCaptureImpl?: RunCaptureFn; - prepareDockerEnvironment?: Parameters[2]; + prepareDockerEnvironment?: Parameters[3]; }; /** Probe the persisted raw Ollama daemon through its platform-specific host transport. */ export function probeOllamaHostInventory(options: OllamaHostInventoryProbeOptions = {}): { endpoint: string; - output: string; - valid: boolean; + inventory: string[] | null; } { const host = options.getOllamaHost ? options.getOllamaHost() : (loadPersistedOllamaHost() ?? getResolvedOllamaHost()); const endpoint = `http://${host}:${OLLAMA_PORT}/api/tags`; - const capture = createOllamaApiCapture( - options.runCaptureImpl, + const inventory = probeOllamaEndpointInventory( host, + options.runCaptureImpl, + 5_000, options.prepareDockerEnvironment, ); - const output = capture( - [ - "curl", - ...buildValidatedCurlCommandArgs([ - "-sS", - "--connect-timeout", - "2", - "--max-time", - "4", - endpoint, - ]), - ], - { ignoreError: true, timeout: 6000 }, - ); - return { endpoint, output, valid: isValidOllamaTagsResponseBody(output) }; + return { endpoint, inventory }; } const COMPATIBLE_PROVIDERS = new Set(["compatible-endpoint", "compatible-anthropic-endpoint"]); From 8e8f3b572787ab56e5cab6ff30e3ba521c24d68a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 06:14:02 -0700 Subject: [PATCH 25/48] fix(inference): validate discovered Ollama inventory Signed-off-by: Prekshi Vyas --- .../local-windows-ollama-transport.test.ts | 57 +++++++++++++++++++ src/lib/inference/local.test.ts | 30 ---------- src/lib/inference/local.ts | 2 +- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 63b8bb3ab98..59a964e1ceb 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -233,6 +233,63 @@ describe("Windows-host Ollama transport", () => { } }); + it("probes WSL loopback before Windows-host Ollama", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-discovery-")); + const commands: (readonly string[])[] = []; + const endpoints: string[] = []; + const capture = vi.fn((command: readonly string[]) => { + commands.push(command); + const endpoint = command.at(-1) ?? ""; + endpoints.push(endpoint); + return endpoint.includes("host.docker.internal") ? JSON.stringify({ models: [] }) : ""; + }); + + try { + expect(findReachableOllamaHost(capture, { isWsl: true }, stateRoot)).toBe( + OLLAMA_HOST_DOCKER_INTERNAL, + ); + expect(endpoints).toEqual([ + "http://127.0.0.1:11434/api/tags", + "http://host.docker.internal:11434/api/tags", + ]); + expect(commands.map((command) => command[0])).toEqual(["curl", "docker"]); + expect(commands[0]).toEqual( + expect.arrayContaining(["--connect-timeout", "3", "--max-time", "5"]), + ); + expect(commands[1]).toEqual( + expect.arrayContaining([ + CONTAINER_REACHABILITY_IMAGE, + "--connect-timeout", + "3", + "--max-time", + "5", + ]), + ); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("rejects a nonempty invalid Windows-host inventory during WSL discovery", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-invalid-discovery-")); + const invalidCapture = vi.fn((command: readonly string[]) => + (command.at(-1) ?? "").includes("host.docker.internal") ? "proxy error" : "", + ); + const validCapture = vi.fn((command: readonly string[]) => + (command.at(-1) ?? "").includes("host.docker.internal") ? JSON.stringify({ models: [] }) : "", + ); + + try { + expect(findReachableOllamaHost(invalidCapture, { isWsl: true }, stateRoot)).toBeNull(); + expect(findReachableOllamaHost(validCapture, { isWsl: true }, stateRoot)).toBe( + OLLAMA_HOST_DOCKER_INTERNAL, + ); + expect(validCapture).toHaveBeenCalledTimes(2); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("rejects an untrusted persisted host", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-invalid-")); try { diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 79f0b392e8c..47d0805424b 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -38,7 +38,6 @@ import { buildOllamaProbeOptions, CONTAINER_REACHABILITY_IMAGE, DEFAULT_OLLAMA_MODEL, - findReachableOllamaHost, getBootstrapOllamaModelOptions, getDefaultOllamaModel, getLocalProviderBaseUrl, @@ -154,35 +153,6 @@ describe("local inference helpers", () => { ]); }); - it("probes WSL loopback before Windows-host Ollama", () => { - vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); - const commands: string[][] = []; - const endpoints: string[] = []; - - const host = findReachableOllamaHost( - (command) => { - commands.push([...command]); - const endpoint = command.at(-1) ?? ""; - endpoints.push(endpoint); - return endpoint.includes("host.docker.internal") ? "ollama" : ""; - }, - // Pin the WSL decision: isWsl answers false off Linux before it reads - // WSL_DISTRO_NAME, so the stub above cannot reach the WSL candidate order. - { isWsl: true }, - ); - - expect(host).toBe("host.docker.internal"); - expect(endpoints).toEqual([ - "http://127.0.0.1:11434/api/tags", - "http://host.docker.internal:11434/api/tags", - ]); - expect(commands.map((command) => command[0])).toEqual(["curl", "docker"]); - expect(commands[0]).toEqual(expect.arrayContaining(["--connect-timeout", "3", "--max-time", "5"])); - expect(commands[1]).toEqual( - expect.arrayContaining([CONTAINER_REACHABILITY_IMAGE, "--connect-timeout", "3", "--max-time", "5"]), - ); - }); - it("returns the expected base URL for vllm-local", () => { expect(getLocalProviderBaseUrl("vllm-local")).toBe("http://host.openshell.internal:8000/v1"); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 860e1c0e5cb..45efb5f6677 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -221,7 +221,7 @@ export function findReachableOllamaHost( ], { ignoreError: true }, ); - if (result) { + if (isValidOllamaTagsResponseBody(result)) { _resolvedOllamaHost = host; return host; } From ca75e6cc5dd3c60cea52fb818bd4b0c16bb712b6 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 06:33:56 -0700 Subject: [PATCH 26/48] fix(cli): keep Ollama rollback state out of argv Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 72 ++++++++++++++++++++++++ src/lib/inference/ollama/windows.ts | 53 +++++++++-------- 2 files changed, 98 insertions(+), 27 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index e6a22141fc7..14684e06c30 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -176,6 +176,40 @@ function loadWindowsOllamaWithMocks( }; } +function captureDefaultWindowsRollbackInvocation(userHost: string | null) { + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + const launchFailure = { status: 1, stdout: "", stderr: "launch unavailable" }; + const runResults = [ + hostSnapshotRun(userHost, watcherPath, daemonPath), + successfulRun(), + launchFailure, + launchFailure, + launchFailure, + successfulRun(), + ]; + const run = vi.fn( + (_command: string | string[], _options?: { env?: NodeJS.ProcessEnv }) => + runResults.shift() ?? successfulRun(), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks( + run, + vi.fn(() => ""), + ); + + try { + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + return { daemonPath, run, watcherPath }; +} + describe("Windows Ollama helper", () => { it("continues probing after a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); @@ -321,6 +355,44 @@ describe("Windows Ollama helper", () => { expect(diagnostic).not.toContain(Buffer.from(priorHost, "utf8").toString("base64")); }); + it("keeps a credential-bearing prior binding out of Windows rollback argv", () => { + const priorHost = "https://operator:private-token@ollama.example:11434"; + const { daemonPath, run, watcherPath } = captureDefaultWindowsRollbackInvocation(priorHost); + expect(run).toHaveBeenCalledTimes(6); + const rollbackCall = run.mock.calls.at(-1); + expect(rollbackCall).toBeDefined(); + const [rollbackCommand, rollbackOptions] = rollbackCall!; + const argv = commandText(rollbackCommand); + + expect(argv).toContain("$env:NEMOCLAW_OLLAMA_RESTORE_HOST"); + expect(argv).toContain("Remove-Item Env:NEMOCLAW_OLLAMA_RESTORE_HOST"); + expect(argv).not.toContain(priorHost); + expect(argv).not.toContain(Buffer.from(priorHost, "utf8").toString("base64")); + expect(argv).not.toContain(watcherPath); + expect(argv).not.toContain(daemonPath); + expect(rollbackOptions?.env).toMatchObject({ + NEMOCLAW_OLLAMA_RESTORE_HOST: priorHost, + NEMOCLAW_OLLAMA_RESTORE_HOST_PRESENT: "1", + NEMOCLAW_OLLAMA_RESTORE_WATCHER: watcherPath, + NEMOCLAW_OLLAMA_RESTORE_DAEMON: daemonPath, + }); + }); + + it("preserves a missing prior binding distinctly during Windows rollback", () => { + const { run } = captureDefaultWindowsRollbackInvocation(null); + const rollbackCall = run.mock.calls.at(-1); + expect(rollbackCall).toBeDefined(); + const [rollbackCommand, rollbackOptions] = rollbackCall!; + + expect(commandText(rollbackCommand)).toContain( + "if ($previousHostPresent -ne '1') { $previousHost = $null }", + ); + expect(rollbackOptions?.env).toMatchObject({ + NEMOCLAW_OLLAMA_RESTORE_HOST: "", + NEMOCLAW_OLLAMA_RESTORE_HOST_PRESENT: "0", + }); + }); + it("stops a final unready replacement before restoring the prior Windows state", () => { const priorHost = "127.0.0.1:11434"; const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index e5497dc8750..9fc91335994 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -103,17 +103,14 @@ function captureWindowsOllamaHostSnapshot(): WindowsOllamaHostSnapshot | null { } } -function psUtf8Expression(value: string): string { - const encoded = Buffer.from(value, "utf8").toString("base64"); - return `[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encoded}'))`; -} - -function psNullableUtf8Expression(value: string | null): string { - return value === null ? "$null" : psUtf8Expression(value); -} +const WINDOWS_OLLAMA_RESTORE_HOST_ENV = "NEMOCLAW_OLLAMA_RESTORE_HOST"; +const WINDOWS_OLLAMA_RESTORE_HOST_PRESENT_ENV = "NEMOCLAW_OLLAMA_RESTORE_HOST_PRESENT"; +const WINDOWS_OLLAMA_RESTORE_WATCHER_ENV = "NEMOCLAW_OLLAMA_RESTORE_WATCHER"; +const WINDOWS_OLLAMA_RESTORE_DAEMON_ENV = "NEMOCLAW_OLLAMA_RESTORE_DAEMON"; -function runWindowsOllamaStateScript(script: string): boolean { +function runWindowsOllamaStateScript(script: string, env?: NodeJS.ProcessEnv): boolean { const result = run(["powershell.exe", "-Command", `$ErrorActionPreference='Stop'; ${script}`], { + env, ignoreError: true, suppressOutput: true, }); @@ -128,31 +125,33 @@ function persistOllamaHostEnvVar(): boolean { ); } -function buildWindowsOllamaRestoreScript(snapshot: WindowsOllamaHostSnapshot): string { - const script = [ +function buildWindowsOllamaRestoreScript(): string { + return [ + `$previousHostPresent = $env:${WINDOWS_OLLAMA_RESTORE_HOST_PRESENT_ENV}`, + `$previousHost = $env:${WINDOWS_OLLAMA_RESTORE_HOST_ENV}`, + `$previousWatcher = $env:${WINDOWS_OLLAMA_RESTORE_WATCHER_ENV}`, + `$previousDaemon = $env:${WINDOWS_OLLAMA_RESTORE_DAEMON_ENV}`, + `Remove-Item Env:${WINDOWS_OLLAMA_RESTORE_HOST_PRESENT_ENV} -EA SilentlyContinue`, + `Remove-Item Env:${WINDOWS_OLLAMA_RESTORE_HOST_ENV} -EA SilentlyContinue`, + `Remove-Item Env:${WINDOWS_OLLAMA_RESTORE_WATCHER_ENV} -EA SilentlyContinue`, + `Remove-Item Env:${WINDOWS_OLLAMA_RESTORE_DAEMON_ENV} -EA SilentlyContinue`, + "if ($previousHostPresent -ne '1') { $previousHost = $null }", "Get-Process 'ollama app' -EA SilentlyContinue | Stop-Process -Force", "Get-Process ollama -EA SilentlyContinue | Stop-Process -Force", - `$previousHost = ${psNullableUtf8Expression(snapshot.userHost)}`, "[Environment]::SetEnvironmentVariable('OLLAMA_HOST',$previousHost,'User')", "$env:OLLAMA_HOST = $previousHost", - ]; - if (snapshot.watcherPath) { - script.push( - `$previousWatcher = ${psUtf8Expression(snapshot.watcherPath)}`, - "Start-Process -FilePath $previousWatcher -WindowStyle Hidden -ErrorAction Stop", - ); - } else if (snapshot.daemonPath) { - script.push( - `$previousDaemon = ${psUtf8Expression(snapshot.daemonPath)}`, - "Start-Process -FilePath $previousDaemon -ArgumentList 'serve' -WindowStyle Hidden -ErrorAction Stop", - ); - } - return script.join("; "); + "if ($previousWatcher) { Start-Process -FilePath $previousWatcher -WindowStyle Hidden -ErrorAction Stop } " + + "elseif ($previousDaemon) { Start-Process -FilePath $previousDaemon -ArgumentList 'serve' -WindowStyle Hidden -ErrorAction Stop }", + ].join("; "); } function rollbackWindowsOllamaHostSnapshot(snapshot: WindowsOllamaHostSnapshot): boolean { - const script = buildWindowsOllamaRestoreScript(snapshot); - return runWindowsOllamaStateScript(script); + return runWindowsOllamaStateScript(buildWindowsOllamaRestoreScript(), { + [WINDOWS_OLLAMA_RESTORE_HOST_PRESENT_ENV]: snapshot.userHost === null ? "0" : "1", + [WINDOWS_OLLAMA_RESTORE_HOST_ENV]: snapshot.userHost ?? "", + [WINDOWS_OLLAMA_RESTORE_WATCHER_ENV]: snapshot.watcherPath ?? "", + [WINDOWS_OLLAMA_RESTORE_DAEMON_ENV]: snapshot.daemonPath ?? "", + }); } function reportWindowsOllamaRollbackFailure(): void { From bd77a2157f043e2a30ddd06ba75598697233d123 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 06:46:35 -0700 Subject: [PATCH 27/48] refactor(cli): reuse Ollama transport owner Signed-off-by: Prekshi Vyas --- .../local-windows-ollama-transport.test.ts | 13 ++++++++++++- src/lib/inference/local.test.ts | 15 --------------- src/lib/inference/local.ts | 15 --------------- src/lib/inference/ollama/windows.test.ts | 5 +---- src/lib/inference/ollama/windows.ts | 16 ++++++++++++++-- 5 files changed, 27 insertions(+), 37 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 59a964e1ceb..b5724cbdfa1 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -51,7 +51,14 @@ describe("Windows-host Ollama transport", () => { it("selects Docker Desktop only for the Windows-host transport owner", () => { expect( getOllamaApiCommand( - ["-sf", "http://host.docker.internal:11434/api/tags"], + [ + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + "http://host.docker.internal:11434/api/tags", + ], OLLAMA_HOST_DOCKER_INTERNAL, ), ).toEqual([ @@ -60,6 +67,10 @@ describe("Windows-host Ollama transport", () => { "--rm", CONTAINER_REACHABILITY_IMAGE, "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", "http://host.docker.internal:11434/api/tags", ]); expect(getOllamaApiCommand(["-sf", "http://127.0.0.1:11434/api/tags"], "127.0.0.1")).toEqual([ diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 47d0805424b..e75e2c9bea3 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -50,7 +50,6 @@ import { getOllamaModelOptions, getOllamaProbeCommand, getOllamaWarmupCommand, - getWindowsHostOllamaDockerReachabilityArgs, isLocalProviderProbeOutputHealthy, isOllamaRunnerCrash, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, @@ -139,20 +138,6 @@ describe("local inference helpers", () => { }); }); - it("builds a credential-free Docker Desktop probe for Windows-host Ollama (#8127)", () => { - expect(getWindowsHostOllamaDockerReachabilityArgs()).toEqual([ - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "-sf", - "--connect-timeout", - "2", - "--max-time", - "5", - "http://host.docker.internal:11434/api/tags", - ]); - }); - it("returns the expected base URL for vllm-local", () => { expect(getLocalProviderBaseUrl("vllm-local")).toBe("http://host.openshell.internal:8000/v1"); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 45efb5f6677..0f1a3079a66 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -149,21 +149,6 @@ export { OLLAMA_LOCALHOST, } from "./local-adapter-lifecycle"; -/** Build the credential-free Docker Desktop probe for Windows-host Ollama. */ -export function getWindowsHostOllamaDockerReachabilityArgs(): string[] { - return [ - "run", - "--rm", - CONTAINER_REACHABILITY_IMAGE, - "-sf", - "--connect-timeout", - "2", - "--max-time", - "5", - `http://${OLLAMA_HOST_DOCKER_INTERNAL}:${OLLAMA_PORT}/api/tags`, - ]; -} - let _resolvedOllamaHost: string | null = null; const OLLAMA_HOST_RECEIPT_NAME = "ollama-host.json"; diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 14684e06c30..47ddcd4cd85 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -382,11 +382,8 @@ describe("Windows Ollama helper", () => { const { run } = captureDefaultWindowsRollbackInvocation(null); const rollbackCall = run.mock.calls.at(-1); expect(rollbackCall).toBeDefined(); - const [rollbackCommand, rollbackOptions] = rollbackCall!; + const [, rollbackOptions] = rollbackCall!; - expect(commandText(rollbackCommand)).toContain( - "if ($previousHostPresent -ne '1') { $previousHost = $null }", - ); expect(rollbackOptions?.env).toMatchObject({ NEMOCLAW_OLLAMA_RESTORE_HOST: "", NEMOCLAW_OLLAMA_RESTORE_HOST_PRESENT: "0", diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 9fc91335994..b4b5a6debde 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -9,7 +9,7 @@ const { spawn } = require("child_process"); const { run, runCapture } = require("../../runner"); const { createOllamaApiCapture, - getWindowsHostOllamaDockerReachabilityArgs, + getOllamaApiCommand, isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, setResolvedOllamaHost, @@ -418,7 +418,19 @@ function printWindowsOllamaTimeoutDiagnostics(): void { console.error( ' powershell.exe -Command "Get-NetTCPConnection -LocalPort 11434 -State Listen -ErrorAction SilentlyContinue"', ); - console.error(` docker ${getWindowsHostOllamaDockerReachabilityArgs().join(" ")}`); + console.error( + ` ${getOllamaApiCommand( + [ + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + `http://${OLLAMA_HOST_DOCKER_INTERNAL}:${OLLAMA_PORT}/api/tags`, + ], + OLLAMA_HOST_DOCKER_INTERNAL, + ).join(" ")}`, + ); } module.exports = { From 3a952ef0974825096150a13b925f9714df54d1e1 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 07:31:35 -0700 Subject: [PATCH 28/48] fix(cli): roll back failed Windows installs Signed-off-by: Prekshi Vyas --- ci/test-file-size-budget.json | 2 +- src/lib/inference/ollama/windows.test.ts | 173 +++++++++++++- src/lib/inference/ollama/windows.ts | 223 +++++++++++++----- src/lib/onboard.ts | 26 +- src/lib/onboard/setup-nim-ollama.test.ts | 98 +++++++- src/lib/onboard/setup-nim-ollama.ts | 84 ++++--- ...board-ollama-upgrade-version-floor.test.ts | 8 +- test/onboarding/onboard-selection.test.ts | 194 ++++++++------- .../support/onboard-selection-test-helpers.ts | 7 +- 9 files changed, 594 insertions(+), 221 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 56e22943bf6..2cf75ba3d4c 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -9,6 +9,6 @@ "test/installer-integration/install-preflight.test.ts": 3025, "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 4625, "test/onboarding/onboard-messaging.test.ts": 1971, - "test/onboarding/onboard-selection.test.ts": 4176 + "test/onboarding/onboard-selection.test.ts": 4170 } } diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 47ddcd4cd85..088f059d062 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -148,7 +148,11 @@ function createWindowsSetupBoundary(options: { throw new PreservedWindowsInterrupt(signal); }), }; - return { operations, state }; + return { + operations, + state, + triggerInterrupt: (signal: "SIGINT" | "SIGTERM") => interruptHandler(signal), + }; } function loadWindowsOllamaWithMocks( @@ -210,6 +214,43 @@ function captureDefaultWindowsRollbackInvocation(userHost: string | null) { return { daemonPath, run, watcherPath }; } +function createWindowsInstallBoundary(options: { + userHost: string | null; + watcherPath: string | null; + daemonPath: string | null; + installedPath?: string; + initialReady?: boolean; + launchStatuses?: number[]; + readinessResults?: boolean[]; + rollbackStatus?: number; + installerInterrupt?: "SIGINT" | "SIGTERM"; +}) { + const boundary = createWindowsSetupBoundary(options); + const cancelInstaller = vi.fn(() => boundary.state.events.push("cancel-installer")); + const completion = options.installerInterrupt + ? Promise.resolve().then(() => boundary.triggerInterrupt(options.installerInterrupt!)) + : Promise.resolve(); + const operations = { + ...boundary.operations, + startInstaller: vi.fn(() => { + boundary.state.events.push("install"); + boundary.state.userHost = "0.0.0.0:11434"; + boundary.state.watcherRunning = true; + boundary.state.daemonRunning = true; + return { completion, cancel: cancelInstaller }; + }), + resolveInstalledPath: vi.fn(() => { + boundary.state.events.push("resolve-path"); + return options.installedPath ?? "C:\\Users\\tester\\Ollama\\ollama.exe"; + }), + awaitReady: vi.fn(() => { + boundary.state.events.push("installer-readiness"); + return options.initialReady ?? false; + }), + }; + return { ...boundary, cancelInstaller, operations }; +} + describe("Windows Ollama helper", () => { it("continues probing after a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); @@ -390,6 +431,136 @@ describe("Windows Ollama helper", () => { }); }); + it("restores the prior Windows state when install cannot resolve ollama.exe", async () => { + const priorHost = "127.0.0.1:11434"; + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const boundary = createWindowsInstallBoundary({ + userHost: priorHost, + watcherPath, + daemonPath: null, + installedPath: "", + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + await expect(windows.installOllamaOnWindowsHost({}, boundary.operations)).resolves.toEqual({ + ok: false, + path: "", + reason: "install", + }); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(true); + expect(boundary.state.daemonRunning).toBe(false); + expect(boundary.state.events).toEqual([ + "snapshot", + "install", + "resolve-path", + "stop-replacement", + "restore", + ]); + }); + + it("restores the prior Windows state when installed Ollama remains unreachable", async () => { + const priorHost = "127.0.0.1:11434"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + const boundary = createWindowsInstallBoundary({ + userHost: priorHost, + watcherPath: null, + daemonPath, + launchStatuses: [1, 1], + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + await expect(windows.installOllamaOnWindowsHost({}, boundary.operations)).resolves.toEqual({ + ok: false, + path: daemonPath, + reason: "readiness", + }); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(false); + expect(boundary.state.daemonRunning).toBe(true); + expect(boundary.state.events.at(-2)).toBe("stop-replacement"); + expect(boundary.state.events.at(-1)).toBe("restore"); + }); + + it("cancels install and restores prior Windows state before preserving SIGTERM", async () => { + const priorHost = "127.0.0.1:11434"; + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const boundary = createWindowsInstallBoundary({ + userHost: priorHost, + watcherPath, + daemonPath: null, + installerInterrupt: "SIGTERM", + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + await expect(windows.installOllamaOnWindowsHost({}, boundary.operations)).rejects.toThrow( + PreservedWindowsInterrupt, + ); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(boundary.state.events).toEqual([ + "snapshot", + "install", + "cancel-installer", + "stop-replacement", + "restore", + "signal:SIGTERM", + ]); + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(true); + expect(boundary.cancelInstaller).toHaveBeenCalledOnce(); + }); + + it("reports manual recovery when install rollback fails", async () => { + const boundary = createWindowsInstallBoundary({ + userHost: "127.0.0.1:11434", + watcherPath: null, + daemonPath: null, + installedPath: "", + rollbackStatus: 1, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + await windows.installOllamaOnWindowsHost({}, boundary.operations); + } finally { + restore(); + logSpy.mockRestore(); + } + + const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + errorSpy.mockRestore(); + expect(diagnostic).toContain("Failed to restore the previous Windows Ollama state"); + expect(diagnostic).toContain("restore your previous User-scope OLLAMA_HOST value"); + }); + it("stops a final unready replacement before restoring the prior Windows state", () => { const priorHost = "127.0.0.1:11434"; const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index b4b5a6debde..f2c9dca6bf3 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -21,6 +21,11 @@ function psSingleQuote(value: string): string { return `'${String(value).replace(/'/g, "''")}'`; } +type WindowsOllamaInstallerProcess = { + completion: Promise; + cancel: () => void; +}; + // Pre-set OLLAMA_HOST in both User scope (persists across logins) and the // current PowerShell session (inherited by the installer's auto-spawned // ollama_app + daemon) so the new daemon binds 0.0.0.0 from the start. @@ -29,18 +34,16 @@ function psSingleQuote(value: string): string { // holds output in an internal buffer and the user sees long silent gaps. // Reading the pipe from Node and re-writing to our own TTY shows progress // as soon as PowerShell flushes a chunk. -async function installOllamaOnWindowsHost(): Promise<{ ok: boolean; path: string }> { - console.log(" Installing Ollama on Windows host..."); - console.log(" This can take several minutes. Output may pause silently"); - await new Promise((resolve) => { - const child = spawn( - "powershell.exe", - [ - "-Command", - "[Environment]::SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User'); $env:OLLAMA_HOST='0.0.0.0:11434'; irm https://ollama.com/install.ps1 | iex", - ], - { stdio: ["ignore", "pipe", "pipe"] }, - ); +function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { + const child = spawn( + "powershell.exe", + [ + "-Command", + "[Environment]::SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User'); $env:OLLAMA_HOST='0.0.0.0:11434'; irm https://ollama.com/install.ps1 | iex", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + const completion = new Promise((resolve) => { child.stdout?.on("data", (chunk: Buffer) => process.stdout.write(chunk)); child.stderr?.on("data", (chunk: Buffer) => process.stderr.write(chunk)); child.on("close", () => resolve()); @@ -49,7 +52,20 @@ async function installOllamaOnWindowsHost(): Promise<{ ok: boolean; path: string resolve(); }); }); - const installedPath = runCapture( + return { + completion, + cancel: () => { + try { + child.kill("SIGTERM"); + } catch { + // Rollback below still stops any installer-created Ollama processes. + } + }, + }; +} + +function resolveWindowsOllamaInstalledPath(): string { + return runCapture( [ "powershell.exe", "-Command", @@ -57,11 +73,6 @@ async function installOllamaOnWindowsHost(): Promise<{ ok: boolean; path: string ], { ignoreError: true }, ).trim(); - if (!installedPath) { - return { ok: false, path: "" }; - } - console.log(` ✓ Installed: ${installedPath}`); - return { ok: true, path: installedPath }; } type WindowsOllamaHostSnapshot = { @@ -311,6 +322,16 @@ type WindowsOllamaSetupOperations = { preserveInterrupt: (signal: WindowsOllamaInterruptSignal) => void; }; +type WindowsOllamaInstallOperations = WindowsOllamaSetupOperations & { + startInstaller: () => WindowsOllamaInstallerProcess; + resolveInstalledPath: () => string; + awaitReady: () => boolean; +}; + +type WindowsOllamaInstallResult = + | { ok: false; path: string; reason: "install" | "readiness" } + | { ok: true; path: string; commit: () => void; rollback: () => void }; + function registerWindowsOllamaInterruptHandler( handler: (signal: WindowsOllamaInterruptSignal) => void, ): () => void { @@ -337,6 +358,13 @@ const WINDOWS_OLLAMA_SETUP_OPERATIONS: WindowsOllamaSetupOperations = { }, }; +const WINDOWS_OLLAMA_INSTALL_OPERATIONS: WindowsOllamaInstallOperations = { + ...WINDOWS_OLLAMA_SETUP_OPERATIONS, + startInstaller: startWindowsOllamaInstaller, + resolveInstalledPath: resolveWindowsOllamaInstalledPath, + awaitReady: awaitWindowsOllamaReady, +}; + function rollbackWindowsOllamaSetup( snapshot: WindowsOllamaHostSnapshot, operations: WindowsOllamaSetupOperations, @@ -344,6 +372,117 @@ function rollbackWindowsOllamaSetup( if (!operations.rollbackSnapshot(snapshot)) reportWindowsOllamaRollbackFailure(); } +function beginWindowsOllamaMutation( + snapshot: WindowsOllamaHostSnapshot, + operations: WindowsOllamaSetupOperations, +) { + let active = true; + let mutated = false; + let cancelActiveOperation = () => {}; + let removeInterruptHandler = () => {}; + const commit = () => { + if (!active) return; + active = false; + cancelActiveOperation = () => {}; + removeInterruptHandler(); + }; + const rollback = () => { + if (!active) return; + active = false; + removeInterruptHandler(); + const cancel = cancelActiveOperation; + cancelActiveOperation = () => {}; + try { + cancel(); + } finally { + if (mutated) rollbackWindowsOllamaSetup(snapshot, operations); + } + }; + removeInterruptHandler = operations.registerInterruptHandler((signal) => { + try { + rollback(); + } finally { + operations.preserveInterrupt(signal); + } + }); + return { + commit, + markMutated: () => { + mutated = true; + }, + rollback, + setInterruptCancellation: (cancel: () => void) => { + if (active) cancelActiveOperation = cancel; + }, + }; +} + +function applyWindowsOllamaBinding( + opts: { announceStop?: boolean; installedPath?: string } = {}, + snapshot: WindowsOllamaHostSnapshot, + operations: WindowsOllamaSetupOperations, + markMutated: () => void, +): boolean { + if (!operations.persistBinding()) { + console.error(" Could not persist the Windows Ollama host binding."); + return false; + } + markMutated(); + if (opts.announceStop) { + console.log(" Stopping existing Ollama on Windows host..."); + } + operations.stopProcesses(); + operations.wait(1); + return launchAndAwaitWindowsOllama( + { + watcherPath: snapshot.watcherPath || undefined, + installedPath: opts.installedPath, + }, + operations.launchOperations, + ); +} + +async function installOllamaOnWindowsHost( + opts: { beforeRestart?: () => void } = {}, + operations: WindowsOllamaInstallOperations = WINDOWS_OLLAMA_INSTALL_OPERATIONS, +): Promise { + const snapshot = operations.captureSnapshot(); + if (!snapshot) { + console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); + return { ok: false, path: "", reason: "install" }; + } + const mutation = beginWindowsOllamaMutation(snapshot, operations); + mutation.markMutated(); + console.log(" Installing Ollama on Windows host..."); + console.log(" This can take several minutes. Output may pause silently"); + try { + const installer = operations.startInstaller(); + mutation.setInterruptCancellation(installer.cancel); + await installer.completion; + mutation.setInterruptCancellation(() => {}); + const installedPath = operations.resolveInstalledPath(); + if (!installedPath) { + mutation.rollback(); + return { ok: false, path: "", reason: "install" }; + } + console.log(` ✓ Installed: ${installedPath}`); + if (!operations.awaitReady()) { + console.log(" Installer did not leave a reachable Ollama daemon; restarting it..."); + opts.beforeRestart?.(); + if ( + !applyWindowsOllamaBinding({ installedPath }, snapshot, operations, mutation.markMutated) + ) { + mutation.rollback(); + return { ok: false, path: installedPath, reason: "readiness" }; + } + } + return { ok: true, path: installedPath, commit: mutation.commit, rollback: mutation.rollback }; + } catch (error) { + mutation.rollback(); + throw error; + } +} + // Used by start and restart paths to force a 0.0.0.0 binding on an already // installed Ollama. Fresh install fallback passes installedPath to avoid // relying on a newly-mutated Windows PATH from this process. @@ -356,53 +495,17 @@ function setupWindowsOllamaWith0000Binding( console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); return false; } - let rollbackRequired = false; - let removeInterruptHandler = () => {}; - const handleInterrupt = (signal: WindowsOllamaInterruptSignal) => { - const shouldRollback = rollbackRequired; - rollbackRequired = false; - try { - if (shouldRollback) rollbackWindowsOllamaSetup(snapshot, operations); - } finally { - removeInterruptHandler(); - operations.preserveInterrupt(signal); - } - }; - removeInterruptHandler = operations.registerInterruptHandler(handleInterrupt); + const mutation = beginWindowsOllamaMutation(snapshot, operations); try { - if (!operations.persistBinding()) { - console.error( - " Could not persist the Windows Ollama host binding; leaving processes running.", - ); + if (!applyWindowsOllamaBinding(opts, snapshot, operations, mutation.markMutated)) { + mutation.rollback(); return false; } - rollbackRequired = true; - if (opts.announceStop) { - console.log(" Stopping existing Ollama on Windows host..."); - } - operations.stopProcesses(); - operations.wait(1); - const launched = launchAndAwaitWindowsOllama( - { - watcherPath: snapshot.watcherPath || undefined, - installedPath: opts.installedPath, - }, - operations.launchOperations, - ); - if (launched) { - rollbackRequired = false; - return true; - } - rollbackRequired = false; - rollbackWindowsOllamaSetup(snapshot, operations); - return false; + mutation.commit(); + return true; } catch (error) { - const shouldRollback = rollbackRequired; - rollbackRequired = false; - if (shouldRollback) rollbackWindowsOllamaSetup(snapshot, operations); + mutation.rollback(); throw error; - } finally { - removeInterruptHandler(); } } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 8279155474d..adadc1edfc1 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -229,7 +229,6 @@ const { } = require("./inference/ollama/proxy"); const { installOllamaOnWindowsHost, - awaitWindowsOllamaReady, setupWindowsOllamaWith0000Binding, switchToWindowsOllamaHost, printWindowsOllamaTimeoutDiagnostics, @@ -428,11 +427,7 @@ const promptValidatedSandboxName = sandboxAgent.createPromptValidatedSandboxName exit: process.exit, }); const modelRouter: typeof import("./onboard/model-router") = require("./onboard/model-router"); -const { - isRoutedInferenceProvider, - loadBlueprintProfile, - reconcileModelRouter, -} = modelRouter; +const { isRoutedInferenceProvider, loadBlueprintProfile, reconcileModelRouter } = modelRouter; const routedInference: typeof import("./onboard/routed-inference") = require("./onboard/routed-inference"); const { OnboardRuntimeBoundary, @@ -999,7 +994,6 @@ const { printOllamaExposureWarning, switchToWindowsOllamaHost, installOllamaOnWindowsHost, - awaitWindowsOllamaReady, setupWindowsOllamaWith0000Binding, printWindowsOllamaTimeoutDiagnostics, resetOllamaHostCache, @@ -1507,17 +1501,13 @@ const gatewayRecovery = createGatewayRecoveryOrchestration({ startGatewayWithOptions: gatewayStart.startGatewayWithOptions, }); -const { - recoverGatewayRuntime, - startDockerDriverGateway, - startGateway, - startGatewayForRecovery, -} = createGatewayLifecycleApplication({ - dockerDriverStart: dockerDriverGatewayStart, - recovery: gatewayRecovery, - registration: gatewayRegistration, - start: gatewayStart, -}); +const { recoverGatewayRuntime, startDockerDriverGateway, startGateway, startGatewayForRecovery } = + createGatewayLifecycleApplication({ + dockerDriverStart: dockerDriverGatewayStart, + recovery: gatewayRecovery, + registration: gatewayRegistration, + start: gatewayStart, + }); const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ diff --git a/src/lib/onboard/setup-nim-ollama.test.ts b/src/lib/onboard/setup-nim-ollama.test.ts index 956ea2a3236..d181c6b7c5e 100644 --- a/src/lib/onboard/setup-nim-ollama.test.ts +++ b/src/lib/onboard/setup-nim-ollama.test.ts @@ -49,8 +49,12 @@ function makeDeps(overrides: Partial = {}): Deps { }), printOllamaExposureWarning: () => {}, switchToWindowsOllamaHost: () => {}, - installOllamaOnWindowsHost: async () => ({ ok: true, path: "C:/Ollama/ollama.exe" }), - awaitWindowsOllamaReady: () => true, + installOllamaOnWindowsHost: async () => ({ + ok: true, + path: "C:/Ollama/ollama.exe", + commit: () => {}, + rollback: () => {}, + }), setupWindowsOllamaWith0000Binding: () => true, printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, @@ -264,7 +268,12 @@ describe("createSetupNimOllamaHandlers", () => { throw new Error("route conflict"); }; const switchHost = vi.fn(); - const install = vi.fn(async () => ({ ok: true })); + const install = vi.fn(async () => ({ + ok: true as const, + path: "C:/Ollama/ollama.exe", + commit: () => {}, + rollback: () => {}, + })); const restart = vi.fn(() => true); const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( makeDeps({ @@ -295,7 +304,12 @@ describe("createSetupNimOllamaHandlers", () => { selection.revalidateSandboxIdentity = () => { throw new Error("Sandbox identity changed before local inference"); }; - const install = vi.fn(async () => ({ ok: true, path: "C:/Ollama/ollama.exe" })); + const install = vi.fn(async () => ({ + ok: true as const, + path: "C:/Ollama/ollama.exe", + commit: () => {}, + rollback: () => {}, + })); const start = vi.fn(() => true); const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( makeDeps({ @@ -320,6 +334,82 @@ describe("createSetupNimOllamaHandlers", () => { expect(start).not.toHaveBeenCalled(); }); + it("commits a new Windows Ollama install after model selection", async () => { + const commit = vi.fn(); + const rollback = vi.fn(); + const revalidate = vi.fn(); + const install = vi.fn(async (args: { beforeRestart: () => void }) => { + args.beforeRestart(); + return { + ok: true as const, + path: "C:/Ollama/ollama.exe", + commit, + rollback, + }; + }); + const state = makeState(); + state.revalidateSandboxIdentity = revalidate; + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ installOllamaOnWindowsHost: install }), + ); + + await expect( + handleWindowsHostOllamaSelection( + null, + "install-windows-ollama", + "qwen3:8b", + false, + false, + null, + state, + ), + ).resolves.toBe("selected"); + + expect(revalidate.mock.calls.map(([operation]) => operation)).toEqual([ + "install the Windows Ollama runtime", + "start the Windows Ollama runtime", + ]); + expect(commit).toHaveBeenCalledOnce(); + expect(rollback).not.toHaveBeenCalled(); + }); + + it("rolls back a new Windows Ollama install when model selection returns", async () => { + const commit = vi.fn(); + const rollback = vi.fn(); + const install = vi.fn(async () => ({ + ok: true as const, + path: "C:/Ollama/ollama.exe", + commit, + rollback, + })); + const resetHost = vi.fn(); + const state = makeState(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + isNonInteractive: () => false, + installOllamaOnWindowsHost: install, + resetOllamaHostCache: resetHost, + selectAndValidateOllamaModel: async () => ({ outcome: "back-to-selection" }), + }), + ); + + await expect( + handleWindowsHostOllamaSelection( + null, + "install-windows-ollama", + "qwen3:8b", + false, + false, + null, + state, + ), + ).resolves.toBe("retry-selection"); + + expect(commit).not.toHaveBeenCalled(); + expect(rollback).toHaveBeenCalledOnce(); + expect(resetHost).toHaveBeenCalledOnce(); + }); + it("preserves accepted tools-incompatible state for running Ollama", async () => { const state = makeState(); const { handleRunningOllamaSelection } = createSetupNimOllamaHandlers(makeDeps()); diff --git a/src/lib/onboard/setup-nim-ollama.ts b/src/lib/onboard/setup-nim-ollama.ts index e27aca8ff53..2a897497f4a 100644 --- a/src/lib/onboard/setup-nim-ollama.ts +++ b/src/lib/onboard/setup-nim-ollama.ts @@ -12,6 +12,10 @@ const { type SetupNimSelectionResult = "selected" | "retry-selection"; +type WindowsOllamaInstallResult = + | { ok: false; path: string; reason: "install" | "readiness" } + | { ok: true; path: string; commit: () => void; rollback: () => void }; + type SetupNimOllamaDeps = { OLLAMA_PORT: number; OLLAMA_PROXY_PORT: number; @@ -49,8 +53,9 @@ type SetupNimOllamaDeps = { >; printOllamaExposureWarning: () => void; switchToWindowsOllamaHost: () => void; - installOllamaOnWindowsHost: () => Promise<{ ok: boolean; path?: string | null }>; - awaitWindowsOllamaReady: () => boolean; + installOllamaOnWindowsHost: (args: { + beforeRestart: () => void; + }) => Promise; setupWindowsOllamaWith0000Binding: (args: { announceStop?: boolean; installedPath?: string | null; @@ -216,46 +221,57 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { : !(await deps.prompt(promptMsg)).trim().toLowerCase().startsWith("n"); if (!proceed) return "retry-selection"; - if (isSwitch) { - state.revalidateSandboxIdentity?.("switch to the Windows Ollama runtime"); - deps.switchToWindowsOllamaHost(); - } else if (isInstall) { - state.revalidateSandboxIdentity?.("install the Windows Ollama runtime"); - const installResult = await deps.installOllamaOnWindowsHost(); - if (!installResult.ok) { - console.error( - " Install did not produce ollama.exe on PATH. Check the installer output above.", - ); - if (deps.isNonInteractive()) deps.process.exit(1); - return "retry-selection"; - } - if (!deps.awaitWindowsOllamaReady()) { - console.log(" Installer did not leave a reachable Ollama daemon; restarting it..."); + let installSession: Extract | null = null; + try { + if (isSwitch) { + state.revalidateSandboxIdentity?.("switch to the Windows Ollama runtime"); + deps.switchToWindowsOllamaHost(); + } else if (isInstall) { + state.revalidateSandboxIdentity?.("install the Windows Ollama runtime"); + const installResult = await deps.installOllamaOnWindowsHost({ + beforeRestart: () => + state.revalidateSandboxIdentity?.("start the Windows Ollama runtime"), + }); + if (!installResult.ok) { + if (installResult.reason === "readiness") { + deps.printWindowsOllamaTimeoutDiagnostics(); + } else { + console.error( + " Install did not produce ollama.exe on PATH. Check the installer output above.", + ); + } + if (deps.isNonInteractive()) deps.process.exit(1); + return "retry-selection"; + } + installSession = installResult; + console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); + } else { state.revalidateSandboxIdentity?.("start the Windows Ollama runtime"); - if (!deps.setupWindowsOllamaWith0000Binding({ installedPath: installResult.path })) { + if ( + !deps.setupWindowsOllamaWith0000Binding({ + announceStop: isRestart, + installedPath: winOllamaInstalledPath || undefined, + }) + ) { deps.printWindowsOllamaTimeoutDiagnostics(); if (deps.isNonInteractive()) deps.process.exit(1); return "retry-selection"; } + console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); } - console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); - } else { - state.revalidateSandboxIdentity?.("start the Windows Ollama runtime"); - if ( - !deps.setupWindowsOllamaWith0000Binding({ - announceStop: isRestart, - installedPath: winOllamaInstalledPath || undefined, - }) - ) { - deps.printWindowsOllamaTimeoutDiagnostics(); - if (deps.isNonInteractive()) deps.process.exit(1); - return "retry-selection"; + + const result = await selectModel(gpu, state, requestedModel, null, lockedModel); + if (result === "retry-selection") { + installSession?.rollback(); + deps.resetOllamaHostCache(); + } else { + installSession?.commit(); } - console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); + return result; + } catch (error) { + installSession?.rollback(); + throw error; } - const result = await selectModel(gpu, state, requestedModel, null, lockedModel); - if (result === "retry-selection") deps.resetOllamaHostCache(); - return result; } async function handleRunningOllamaSelection( diff --git a/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts b/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts index e54c3791e56..354769f228a 100644 --- a/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts +++ b/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts @@ -102,8 +102,12 @@ function makeOllamaDeps(overrides: Partial = {}): SetupNimOl }), printOllamaExposureWarning: () => {}, switchToWindowsOllamaHost: () => {}, - installOllamaOnWindowsHost: async () => ({ ok: true }), - awaitWindowsOllamaReady: () => true, + installOllamaOnWindowsHost: async () => ({ + ok: true, + path: "C:/Ollama/ollama.exe", + commit: () => {}, + rollback: () => {}, + }), setupWindowsOllamaWith0000Binding: () => true, printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, diff --git a/test/onboarding/onboard-selection.test.ts b/test/onboarding/onboard-selection.test.ts index 1c319393f6e..fbd4ff93e82 100644 --- a/test/onboarding/onboard-selection.test.ts +++ b/test/onboarding/onboard-selection.test.ts @@ -485,8 +485,12 @@ function makeSetupNimOllamaDeps(overrides: Partial = {}): Se }), printOllamaExposureWarning: () => {}, switchToWindowsOllamaHost: () => {}, - installOllamaOnWindowsHost: async () => ({ ok: true }), - awaitWindowsOllamaReady: () => true, + installOllamaOnWindowsHost: async () => ({ + ok: true, + path: "C:/Ollama/ollama.exe", + commit: () => {}, + rollback: () => {}, + }), setupWindowsOllamaWith0000Binding: () => true, printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, @@ -1186,7 +1190,11 @@ describe("onboard provider selection UX", { timeout: PROVIDER_SELECTION_TEST_TIM }); it("offers Gemini 3.6 Flash instead of 2.5 Flash and supports Other (#9298)", async () => { - const acceptedDefault = await promptRemoteModel("Google Gemini", "gemini", "gemini-3.6-flash", null, + const acceptedDefault = await promptRemoteModel( + "Google Gemini", + "gemini", + "gemini-3.6-flash", + null, { promptFn: async () => "", writeLine: () => {} }, ); assert.equal(acceptedDefault, "gemini-3.6-flash"); @@ -1485,9 +1493,12 @@ reportChildScenario(async () => { ); }); - it("treats an implicit latest Ollama model as installed during systemd repair", { - timeout: PROVIDER_SELECTION_TEST_TIMEOUT_MS, - }, () => { + it( + "treats an implicit latest Ollama model as installed during systemd repair", + { + timeout: PROVIDER_SELECTION_TEST_TIMEOUT_MS, + }, + () => { const workspace = onboardProcessWorkspace("nemoclaw-onboard-ollama-systemd-"); const { root: tmpDir } = workspace; const fakeBin = workspace.binDir; @@ -1579,11 +1590,15 @@ reportChildScenario(async () => { ), "should install and wait for the Ollama systemd drop-in restart", ); - }); + }, + ); - it("preserves existing Ollama systemd override settings while repairing loopback", { - timeout: 10_000, - }, () => { + it( + "preserves existing Ollama systemd override settings while repairing loopback", + { + timeout: 10_000, + }, + () => { const workspace = onboardProcessWorkspace("nemoclaw-onboard-ollama-systemd-merge-"); const { root: tmpDir } = workspace; const fakeBin = workspace.binDir; @@ -1715,11 +1730,15 @@ reportChildScenario(async () => { payload.installedBody.includes('Environment="HTTPS_PROXY=http://proxy.internal:8080"'), "other Environment= settings should be preserved", ); - }); + }, + ); - it("adds Spark CUDA v13 and enables the Ollama systemd service on managed install", { - timeout: 10_000, - }, () => { + it( + "adds Spark CUDA v13 and enables the Ollama systemd service on managed install", + { + timeout: 10_000, + }, + () => { const workspace = onboardProcessWorkspace("nemoclaw-ollama-systemd-spark-"); const { root: tmpDir } = workspace; @@ -1792,11 +1811,15 @@ reportChildScenario(() => { ), "managed Ollama installs should enable the service for reboot survival", ); - }); + }, + ); - it("allows prompt-capable sudo in non-interactive Ollama systemd setup", { - timeout: 10_000, - }, () => { + it( + "allows prompt-capable sudo in non-interactive Ollama systemd setup", + { + timeout: 10_000, + }, + () => { const workspace = onboardProcessWorkspace("nemoclaw-ollama-systemd-sudo-mode-"); const { root: tmpDir } = workspace; @@ -1852,7 +1875,8 @@ reportChildScenario(() => { ), "prompt sudo mode should not use sudo -n", ); - }); + }, + ); it("rejects unsupported non-interactive sudo mode values", () => { const previousMode = process.env.NEMOCLAW_NON_INTERACTIVE_SUDO_MODE; @@ -1885,9 +1909,12 @@ reportChildScenario(() => { } }); - it("repairs already-loopback systemd Ollama without starting a duplicate daemon", { - timeout: 10_000, - }, () => { + it( + "repairs already-loopback systemd Ollama without starting a duplicate daemon", + { + timeout: 10_000, + }, + () => { const workspace = onboardProcessWorkspace("nemoclaw-onboard-ollama-systemd-loopback-"); const { root: tmpDir } = workspace; const fakeBin = workspace.binDir; @@ -1985,11 +2012,15 @@ reportChildScenario(async () => { payload.events.slice(restartIndex + 1).includes("tags"), "should re-probe after the systemd restart instead of trusting a stale loopback cache", ); - }); + }, + ); - it("fails closed instead of starting unmanaged Ollama when systemd restart stays unreachable", { - timeout: 15_000, - }, () => { + it( + "fails closed instead of starting unmanaged Ollama when systemd restart stays unreachable", + { + timeout: 15_000, + }, + () => { const workspace = onboardProcessWorkspace("nemoclaw-onboard-existing-systemd-restart-fail-"); const { root: tmpDir } = workspace; @@ -2050,7 +2081,8 @@ const { setupNim } = require(${onboardPath}); assert.equal(result.status, 1); assert.match(result.stderr, /Ollama systemd restart did not recover/); assert.doesNotMatch(result.stderr, /manual-start/); - }); + }, + ); it("fails closed when an existing Ollama systemd override cannot be applied", () => { const workspace = onboardProcessWorkspace("nemoclaw-onboard-existing-systemd-fail-"); @@ -3542,9 +3574,7 @@ reportChildScenario(async () => { assert.ok(zstdWarningIndex >= 0 && zstdWarningIndex < zstdCommandIndex); assert.ok(installerWarningIndex >= 0 && installerWarningIndex < installerCommandIndex); assert.equal(events[installerCommandIndex]?.stdio, "inherit"); - assert.ok( - commands.some((command) => command.includes("/install.sh'")), - ); + assert.ok(commands.some((command) => command.includes("/install.sh'"))); assert.ok(!commands.some((command) => command.includes("brew install"))); assert.ok( commands.some((command) => command.includes("OLLAMA_HOST=127.0.0.1:11434 ollama serve")), @@ -3852,63 +3882,12 @@ const { setupNim } = require(${onboardPath}); assert.equal(prompt.mock.calls.length, 0); assert.equal(result.provider, "ollama-local"); assert.ok(notes.some((line) => line.includes("[non-interactive] Provider: ollama"))); - assert.ok( - commands.some((command) => command.includes("/install.sh'")), - ); + assert.ok(commands.some((command) => command.includes("/install.sh'"))); } finally { resetOllamaHostCache(); } }); - it("restarts Windows-host Ollama after install when installer auto-start is not reachable", async () => { - const installedPath = "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe"; - const install = vi.fn(async () => ({ ok: true, path: installedPath })); - const awaitReady = vi.fn(() => false); - const setup = vi.fn(() => true); - const lines: string[] = []; - const log = vi.spyOn(console, "log").mockImplementation((...args) => { - lines.push(args.join(" ")); - }); - const state = makeOllamaSelectionState(); - const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( - makeSetupNimOllamaDeps({ - installOllamaOnWindowsHost: install, - awaitWindowsOllamaReady: awaitReady, - setupWindowsOllamaWith0000Binding: setup, - }), - ); - - try { - const result = await handleWindowsHostOllamaSelection( - null, - "install-windows-ollama", - "qwen3:8b", - false, - false, - null, - state, - ); - - assert.equal(result, "selected"); - assert.equal(state.provider, "ollama-local"); - assert.equal(state.model, "qwen3:8b"); - assert.equal(install.mock.calls.length, 1); - assert.equal(awaitReady.mock.calls.length, 1); - assert.deepEqual( - setup.mock.calls.map(([options]) => options), - [{ installedPath }], - ); - assert.ok( - lines.some((line) => - line.includes("Installer did not leave a reachable Ollama daemon; restarting it"), - ), - ); - assert.ok(lines.some((line) => line.includes("Using Ollama on host.docker.internal:11434"))); - } finally { - log.mockRestore(); - } - }); - it("shows Windows-host Ollama in the menu with a Docker Desktop requirement on native Docker WSL", () => { const requirement = getWindowsHostOllamaDockerRequirement("docker"); const { options } = buildWindowsProviderMenu(requirement, { @@ -3927,22 +3906,24 @@ const { setupNim } = require(${onboardPath}); { provider: "start-windows-ollama", installed: true }, { provider: "install-windows-ollama", installed: false }, ] as const)("rejects $provider on native Docker WSL before launching Ollama", (scenario) => { - const boundary = runNativeDockerWindowsProviderBoundary({ - ...scenario, - reachable: false, - timeoutMs: PROVIDER_SELECTION_TEST_TIMEOUT_MS, - }); - assert.equal(boundary.status, 1, `${scenario.provider} unexpectedly passed`); - assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); - assert.match(boundary.stderr, new RegExp(scenario.provider + " requires Docker Desktop")); - assert.match(boundary.stderr, /Choose WSL-local Ollama/); - assert.doesNotMatch( - boundary.stderr, - /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, - ); + const boundary = runNativeDockerWindowsProviderBoundary({ + ...scenario, + reachable: false, + timeoutMs: PROVIDER_SELECTION_TEST_TIMEOUT_MS, + }); + assert.equal(boundary.status, 1, `${scenario.provider} unexpectedly passed`); + assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); + assert.match(boundary.stderr, new RegExp(scenario.provider + " requires Docker Desktop")); + assert.match(boundary.stderr, /Choose WSL-local Ollama/); + assert.doesNotMatch( + boundary.stderr, + /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, + ); }); - it.each(["ollama", "start-windows-ollama", "install-windows-ollama"] as const)("rejects reachable Windows-host Ollama on native Docker WSL through generic and fallback paths [%s]", (provider) => { + it.each(["ollama", "start-windows-ollama", "install-windows-ollama"] as const)( + "rejects reachable Windows-host Ollama on native Docker WSL through generic and fallback paths [%s]", + (provider) => { const boundary = runNativeDockerWindowsProviderBoundary({ provider, installed: true, @@ -3957,7 +3938,8 @@ const { setupNim } = require(${onboardPath}); boundary.stderr, /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, ); - }); + }, + ); it("uses the Windows-host start path when install-windows-ollama is requested but Ollama is already installed", async () => { const requirement = getWindowsHostOllamaDockerRequirement("docker-desktop"); @@ -3970,7 +3952,11 @@ const { setupNim } = require(${onboardPath}); const selectedResolution = requireSelectedProviderResolution(resolution); assert.equal(selectedResolution.selected.key, "start-windows-ollama"); - const install = vi.fn(async () => ({ ok: false, path: "" })); + const install = vi.fn(async () => ({ + ok: false as const, + path: "", + reason: "install" as const, + })); const setup = vi.fn(() => true); const lines: string[] = []; const log = vi.spyOn(console, "log").mockImplementation((...args) => { @@ -4023,7 +4009,11 @@ const { setupNim } = require(${onboardPath}); loopbackOnly: true, }); - const install = vi.fn(async () => ({ ok: false, path: "" })); + const install = vi.fn(async () => ({ + ok: false as const, + path: "", + reason: "install" as const, + })); const setup = vi.fn(() => true); const log = vi.spyOn(console, "log").mockImplementation(() => {}); const state = makeOllamaSelectionState(); @@ -4070,7 +4060,11 @@ const { setupNim } = require(${onboardPath}); loopbackOnly: true, }); - const install = vi.fn(async () => ({ ok: false, path: "" })); + const install = vi.fn(async () => ({ + ok: false as const, + path: "", + reason: "install" as const, + })); const setup = vi.fn(() => true); const log = vi.spyOn(console, "log").mockImplementation(() => {}); const state = makeOllamaSelectionState(); diff --git a/test/support/onboard-selection-test-helpers.ts b/test/support/onboard-selection-test-helpers.ts index b63ef5ecd29..3127d6ab8bf 100644 --- a/test/support/onboard-selection-test-helpers.ts +++ b/test/support/onboard-selection-test-helpers.ts @@ -252,7 +252,12 @@ local.getOllamaModelOptions = () => { }; windows.installOllamaOnWindowsHost = async () => { console.error("WINDOWS_INSTALL_CALLED"); - return { ok: true, path: "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe" }; + return { + ok: true, + path: "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe", + commit: () => {}, + rollback: () => {}, + }; }; windows.setupWindowsOllamaWith0000Binding = () => { console.error("WINDOWS_SETUP_CALLED"); From 805b0ac227979c92b52f97062b0bb869b17c48ee Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 08:02:14 -0700 Subject: [PATCH 29/48] fix(cli): roll back abandoned Windows restarts Signed-off-by: Prekshi Vyas --- ci/test-file-size-budget.json | 2 +- src/lib/inference/ollama/windows.test.ts | 22 ++++-- src/lib/inference/ollama/windows.ts | 18 +++-- src/lib/onboard/setup-nim-ollama.test.ts | 78 ++++++++++++++++++- src/lib/onboard/setup-nim-ollama.ts | 28 ++++--- ...board-ollama-upgrade-version-floor.test.ts | 6 +- test/onboarding/onboard-selection.test.ts | 13 ++-- .../support/onboard-selection-test-helpers.ts | 2 +- 8 files changed, 131 insertions(+), 38 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 2cf75ba3d4c..d206c651f33 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -9,6 +9,6 @@ "test/installer-integration/install-preflight.test.ts": 3025, "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 4625, "test/onboarding/onboard-messaging.test.ts": 1971, - "test/onboarding/onboard-selection.test.ts": 4170 + "test/onboarding/onboard-selection.test.ts": 4169 } } diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 088f059d062..f29c3a47bee 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -204,7 +204,9 @@ function captureDefaultWindowsRollbackInvocation(userHost: string | null) { ); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toBe(false); + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toEqual({ + ok: false, + }); } finally { restore(); logSpy.mockRestore(); @@ -313,9 +315,13 @@ describe("Windows Ollama helper", () => { const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - expect( - windows.setupWindowsOllamaWith0000Binding({ installedPath }, boundary.operations), - ).toBe(true); + const result = windows.setupWindowsOllamaWith0000Binding( + { installedPath }, + boundary.operations, + ); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error("expected successful Windows Ollama setup"); + result.commit(); expect(boundary.state.events).toEqual([ "snapshot", "persist", @@ -353,7 +359,7 @@ describe("Windows Ollama helper", () => { { installedPath: daemonPath }, boundary.operations, ), - ).toBe(false); + ).toEqual({ ok: false }); } finally { restore(); logSpy.mockRestore(); @@ -380,7 +386,9 @@ describe("Windows Ollama helper", () => { const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - expect(windows.setupWindowsOllamaWith0000Binding({}, boundary.operations)).toBe(false); + expect(windows.setupWindowsOllamaWith0000Binding({}, boundary.operations)).toEqual({ + ok: false, + }); } finally { restore(); logSpy.mockRestore(); @@ -582,7 +590,7 @@ describe("Windows Ollama helper", () => { { installedPath: daemonPath }, boundary.operations, ), - ).toBe(false); + ).toEqual({ ok: false }); } finally { restore(); logSpy.mockRestore(); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index f2c9dca6bf3..8154d266fa5 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -328,9 +328,16 @@ type WindowsOllamaInstallOperations = WindowsOllamaSetupOperations & { awaitReady: () => boolean; }; +type WindowsOllamaMutationSession = { + commit: () => void; + rollback: () => void; +}; + type WindowsOllamaInstallResult = | { ok: false; path: string; reason: "install" | "readiness" } - | { ok: true; path: string; commit: () => void; rollback: () => void }; + | ({ ok: true; path: string } & WindowsOllamaMutationSession); + +type WindowsOllamaSetupResult = { ok: false } | ({ ok: true } & WindowsOllamaMutationSession); function registerWindowsOllamaInterruptHandler( handler: (signal: WindowsOllamaInterruptSignal) => void, @@ -489,20 +496,19 @@ async function installOllamaOnWindowsHost( function setupWindowsOllamaWith0000Binding( opts: { announceStop?: boolean; installedPath?: string } = {}, operations: WindowsOllamaSetupOperations = WINDOWS_OLLAMA_SETUP_OPERATIONS, -): boolean { +): WindowsOllamaSetupResult { const snapshot = operations.captureSnapshot(); if (!snapshot) { console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); - return false; + return { ok: false }; } const mutation = beginWindowsOllamaMutation(snapshot, operations); try { if (!applyWindowsOllamaBinding(opts, snapshot, operations, mutation.markMutated)) { mutation.rollback(); - return false; + return { ok: false }; } - mutation.commit(); - return true; + return { ok: true, commit: mutation.commit, rollback: mutation.rollback }; } catch (error) { mutation.rollback(); throw error; diff --git a/src/lib/onboard/setup-nim-ollama.test.ts b/src/lib/onboard/setup-nim-ollama.test.ts index d181c6b7c5e..8b8b691dbad 100644 --- a/src/lib/onboard/setup-nim-ollama.test.ts +++ b/src/lib/onboard/setup-nim-ollama.test.ts @@ -55,7 +55,11 @@ function makeDeps(overrides: Partial = {}): Deps { commit: () => {}, rollback: () => {}, }), - setupWindowsOllamaWith0000Binding: () => true, + setupWindowsOllamaWith0000Binding: () => ({ + ok: true, + commit: () => {}, + rollback: () => {}, + }), printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, installOllamaOnMacOS: () => ({ ok: true }), @@ -274,7 +278,11 @@ describe("createSetupNimOllamaHandlers", () => { commit: () => {}, rollback: () => {}, })); - const restart = vi.fn(() => true); + const restart = vi.fn(() => ({ + ok: true as const, + commit: () => {}, + rollback: () => {}, + })); const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( makeDeps({ switchToWindowsOllamaHost: switchHost, @@ -310,7 +318,11 @@ describe("createSetupNimOllamaHandlers", () => { commit: () => {}, rollback: () => {}, })); - const start = vi.fn(() => true); + const start = vi.fn(() => ({ + ok: true as const, + commit: () => {}, + rollback: () => {}, + })); const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( makeDeps({ installOllamaOnWindowsHost: install, @@ -410,6 +422,66 @@ describe("createSetupNimOllamaHandlers", () => { expect(resetHost).toHaveBeenCalledOnce(); }); + it("rolls back a Windows Ollama restart when model selection returns", async () => { + const commit = vi.fn(); + const rollback = vi.fn(); + const restart = vi.fn(() => ({ ok: true as const, commit, rollback })); + const resetHost = vi.fn(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + isNonInteractive: () => false, + resetOllamaHostCache: resetHost, + selectAndValidateOllamaModel: async () => ({ outcome: "back-to-selection" }), + setupWindowsOllamaWith0000Binding: restart, + }), + ); + + await expect( + handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "qwen3:8b", + false, + true, + "C:/Ollama/ollama.exe", + makeState(), + ), + ).resolves.toBe("retry-selection"); + + expect(restart).toHaveBeenCalledOnce(); + expect(commit).not.toHaveBeenCalled(); + expect(rollback).toHaveBeenCalledOnce(); + expect(resetHost).toHaveBeenCalledOnce(); + }); + + it("rolls back a Windows Ollama restart when model selection throws", async () => { + const rollback = vi.fn(); + const restart = vi.fn(() => ({ ok: true as const, commit: vi.fn(), rollback })); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + selectAndValidateOllamaModel: async () => { + throw new Error("model selection failed"); + }, + setupWindowsOllamaWith0000Binding: restart, + }), + ); + + await expect( + handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "qwen3:8b", + false, + true, + "C:/Ollama/ollama.exe", + makeState(), + ), + ).rejects.toThrow("model selection failed"); + + expect(restart).toHaveBeenCalledOnce(); + expect(rollback).toHaveBeenCalledOnce(); + }); + it("preserves accepted tools-incompatible state for running Ollama", async () => { const state = makeState(); const { handleRunningOllamaSelection } = createSetupNimOllamaHandlers(makeDeps()); diff --git a/src/lib/onboard/setup-nim-ollama.ts b/src/lib/onboard/setup-nim-ollama.ts index 2a897497f4a..da72dbd5895 100644 --- a/src/lib/onboard/setup-nim-ollama.ts +++ b/src/lib/onboard/setup-nim-ollama.ts @@ -16,6 +16,10 @@ type WindowsOllamaInstallResult = | { ok: false; path: string; reason: "install" | "readiness" } | { ok: true; path: string; commit: () => void; rollback: () => void }; +type WindowsOllamaSetupResult = + | { ok: false } + | { ok: true; commit: () => void; rollback: () => void }; + type SetupNimOllamaDeps = { OLLAMA_PORT: number; OLLAMA_PROXY_PORT: number; @@ -59,7 +63,7 @@ type SetupNimOllamaDeps = { setupWindowsOllamaWith0000Binding: (args: { announceStop?: boolean; installedPath?: string | null; - }) => boolean; + }) => WindowsOllamaSetupResult; printWindowsOllamaTimeoutDiagnostics: () => void; resetOllamaHostCache: () => void; installOllamaOnMacOS: (args: { @@ -221,7 +225,7 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { : !(await deps.prompt(promptMsg)).trim().toLowerCase().startsWith("n"); if (!proceed) return "retry-selection"; - let installSession: Extract | null = null; + let mutationSession: { commit: () => void; rollback: () => void } | null = null; try { if (isSwitch) { state.revalidateSandboxIdentity?.("switch to the Windows Ollama runtime"); @@ -243,33 +247,33 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { if (deps.isNonInteractive()) deps.process.exit(1); return "retry-selection"; } - installSession = installResult; + mutationSession = installResult; console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); } else { state.revalidateSandboxIdentity?.("start the Windows Ollama runtime"); - if ( - !deps.setupWindowsOllamaWith0000Binding({ - announceStop: isRestart, - installedPath: winOllamaInstalledPath || undefined, - }) - ) { + const setupResult = deps.setupWindowsOllamaWith0000Binding({ + announceStop: isRestart, + installedPath: winOllamaInstalledPath || undefined, + }); + if (!setupResult.ok) { deps.printWindowsOllamaTimeoutDiagnostics(); if (deps.isNonInteractive()) deps.process.exit(1); return "retry-selection"; } + mutationSession = setupResult; console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); } const result = await selectModel(gpu, state, requestedModel, null, lockedModel); if (result === "retry-selection") { - installSession?.rollback(); + mutationSession?.rollback(); deps.resetOllamaHostCache(); } else { - installSession?.commit(); + mutationSession?.commit(); } return result; } catch (error) { - installSession?.rollback(); + mutationSession?.rollback(); throw error; } } diff --git a/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts b/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts index 354769f228a..893b6f2cba4 100644 --- a/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts +++ b/test/onboarding/onboard-ollama-upgrade-version-floor.test.ts @@ -108,7 +108,11 @@ function makeOllamaDeps(overrides: Partial = {}): SetupNimOl commit: () => {}, rollback: () => {}, }), - setupWindowsOllamaWith0000Binding: () => true, + setupWindowsOllamaWith0000Binding: () => ({ + ok: true, + commit: () => {}, + rollback: () => {}, + }), printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, installOllamaOnMacOS: () => ({ ok: true }), diff --git a/test/onboarding/onboard-selection.test.ts b/test/onboarding/onboard-selection.test.ts index fbd4ff93e82..20591f7c1d6 100644 --- a/test/onboarding/onboard-selection.test.ts +++ b/test/onboarding/onboard-selection.test.ts @@ -76,6 +76,7 @@ const CREDENTIAL_RETRY_PROMPT_RE = const OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE = '{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"type":"function","function":{"name":"emit_ok","arguments":"{\\"ok\\":true}"}}]}}]}'; const PROVIDER_SELECTION_TEST_TIMEOUT_MS = testTimeout(60_000); +const WINDOWS_SETUP_SUCCESS = { ok: true as const, commit: () => {}, rollback: () => {} }; const repoRoot = path.join(import.meta.dirname, "../.."); const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); const credentialsPath = JSON.stringify( @@ -486,12 +487,10 @@ function makeSetupNimOllamaDeps(overrides: Partial = {}): Se printOllamaExposureWarning: () => {}, switchToWindowsOllamaHost: () => {}, installOllamaOnWindowsHost: async () => ({ - ok: true, + ...WINDOWS_SETUP_SUCCESS, path: "C:/Ollama/ollama.exe", - commit: () => {}, - rollback: () => {}, }), - setupWindowsOllamaWith0000Binding: () => true, + setupWindowsOllamaWith0000Binding: () => WINDOWS_SETUP_SUCCESS, printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, installOllamaOnMacOS: () => ({ ok: true }), @@ -3957,7 +3956,7 @@ const { setupNim } = require(${onboardPath}); path: "", reason: "install" as const, })); - const setup = vi.fn(() => true); + const setup = vi.fn((_args?: unknown) => WINDOWS_SETUP_SUCCESS); const lines: string[] = []; const log = vi.spyOn(console, "log").mockImplementation((...args) => { lines.push(args.join(" ")); @@ -4014,7 +4013,7 @@ const { setupNim } = require(${onboardPath}); path: "", reason: "install" as const, })); - const setup = vi.fn(() => true); + const setup = vi.fn((_args?: unknown) => WINDOWS_SETUP_SUCCESS); const log = vi.spyOn(console, "log").mockImplementation(() => {}); const state = makeOllamaSelectionState(); const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( @@ -4065,7 +4064,7 @@ const { setupNim } = require(${onboardPath}); path: "", reason: "install" as const, })); - const setup = vi.fn(() => true); + const setup = vi.fn((_args?: unknown) => WINDOWS_SETUP_SUCCESS); const log = vi.spyOn(console, "log").mockImplementation(() => {}); const state = makeOllamaSelectionState(); const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( diff --git a/test/support/onboard-selection-test-helpers.ts b/test/support/onboard-selection-test-helpers.ts index 3127d6ab8bf..d9e4198bd9d 100644 --- a/test/support/onboard-selection-test-helpers.ts +++ b/test/support/onboard-selection-test-helpers.ts @@ -261,7 +261,7 @@ windows.installOllamaOnWindowsHost = async () => { }; windows.setupWindowsOllamaWith0000Binding = () => { console.error("WINDOWS_SETUP_CALLED"); - return true; + return { ok: true, commit: () => {}, rollback: () => {} }; }; windows.switchToWindowsOllamaHost = () => { console.error("WINDOWS_SWITCH_CALLED"); From 087b94076512b22e555148751dd79b121281568a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 08:51:03 -0700 Subject: [PATCH 30/48] test(cli): isolate doctor Ollama inventory Signed-off-by: Prekshi Vyas --- src/lib/actions/sandbox/doctor-flow.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 8c5eb1d27f4..6ba024ee5e3 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -51,6 +51,7 @@ function createDoctorHarness( getSandboxSpy: MockInstance; getNamedGatewayLifecycleStateSpy: MockInstance; healthProbeSpy: MockInstance; + ollamaInventoryProbeSpy: MockInstance; inspectMutableConfigPermsSpy: MockInstance; loadAgentSpy: MockInstance; probeSandboxInferenceGatewayHealthSpy: MockInstance; @@ -192,6 +193,10 @@ function createDoctorHarness( endpoint: "http://127.0.0.1:11434/v1/chat/completions", detail: "healthy", }); + const ollamaInventoryProbeSpy = vi.spyOn(health, "probeOllamaHostInventory").mockReturnValue({ + endpoint: "http://127.0.0.1:11434/api/tags", + inventory: ["m"], + }); const probeSandboxInferenceGatewayHealthSpy = vi .spyOn(inferenceRouteHealth, "probeSandboxInferenceGatewayHealth") .mockResolvedValue({ @@ -274,6 +279,7 @@ function createDoctorHarness( getSandboxSpy, getNamedGatewayLifecycleStateSpy, healthProbeSpy, + ollamaInventoryProbeSpy, inspectMutableConfigPermsSpy, loadAgentSpy, probeSandboxInferenceGatewayHealthSpy, @@ -492,6 +498,7 @@ describe("runSandboxDoctor flow", () => { ]), ); expect(exitSpy).not.toHaveBeenCalled(); + expect(harness.ollamaInventoryProbeSpy).toHaveBeenCalledOnce(); expect(harness.logSpy).not.toHaveBeenCalled(); }, ); From 9423bb8bd89795d77a99a677f3a9372065ea4b6c Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 09:06:13 -0700 Subject: [PATCH 31/48] fix(cli): align Windows Ollama diagnostics Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 18 ++++++++++++++++++ src/lib/inference/ollama/windows.ts | 15 +++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index f29c3a47bee..6c29db324f5 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -254,6 +254,24 @@ function createWindowsInstallBoundary(options: { } describe("Windows Ollama helper", () => { + it("routes timeout recovery through the credential-isolated NemoClaw probe", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + windows.printWindowsOllamaTimeoutDiagnostics(); + } finally { + restore(); + } + + const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + errorSpy.mockRestore(); + expect(diagnostic).toContain("nemoclaw onboard"); + expect(diagnostic).toContain("isolated Docker client configuration"); + expect(diagnostic).toContain("removes that temporary configuration afterward"); + expect(diagnostic).not.toContain("docker run"); + }); + it("continues probing after a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); const localInference = require(LOCAL_INFERENCE_PATH); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 8154d266fa5..94cef03995f 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -9,7 +9,6 @@ const { spawn } = require("child_process"); const { run, runCapture } = require("../../runner"); const { createOllamaApiCapture, - getOllamaApiCommand, isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, setResolvedOllamaHost, @@ -527,18 +526,10 @@ function printWindowsOllamaTimeoutDiagnostics(): void { console.error( ' powershell.exe -Command "Get-NetTCPConnection -LocalPort 11434 -State Listen -ErrorAction SilentlyContinue"', ); + console.error(" After correcting the Windows process or listener, retry:"); + console.error(" nemoclaw onboard"); console.error( - ` ${getOllamaApiCommand( - [ - "-sf", - "--connect-timeout", - "2", - "--max-time", - "5", - `http://${OLLAMA_HOST_DOCKER_INTERNAL}:${OLLAMA_PORT}/api/tags`, - ], - OLLAMA_HOST_DOCKER_INTERNAL, - ).join(" ")}`, + " NemoClaw repeats the reachability check with an isolated Docker client configuration and removes that temporary configuration afterward.", ); } From e4d15708db5cc36f940b33a78c9e1ffa23879fde Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 09:27:44 -0700 Subject: [PATCH 32/48] test(policy): assert missing OpenShell exit Signed-off-by: Prekshi Vyas --- test/runtime/policy/policies.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtime/policy/policies.test.ts b/test/runtime/policy/policies.test.ts index 283d2c912a8..273c722fe93 100644 --- a/test/runtime/policy/policies.test.ts +++ b/test/runtime/policy/policies.test.ts @@ -631,8 +631,8 @@ exit 1 }) as never); try { - expect(policies.applyPreset("my-assistant", "npm")).toBe(false); - expect(exitSpy).not.toHaveBeenCalled(); + expect(() => policies.applyPreset("my-assistant", "npm")).toThrow(/__test_exit__/); + expect(exitSpy).toHaveBeenCalledWith(1); // No `nemoclaw-policy-*` temp dir should have been created before // the resolvability check exited. expect( From 15aa6a8cee42b443e5eb5a8903ea5c73bec51a6a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 10:00:42 -0700 Subject: [PATCH 33/48] fix(cli): close Windows Ollama review gaps Signed-off-by: Prekshi Vyas --- src/lib/inference/context-window.test.ts | 2 +- .../local-windows-ollama-transport.test.ts | 17 +++++-- src/lib/inference/local.ts | 19 ++++--- src/lib/inference/ollama/windows.test.ts | 26 ++++++++-- src/lib/inference/ollama/windows.ts | 51 +++++++++++++------ src/lib/onboard.ts | 2 + src/lib/onboard/setup-nim-ollama.test.ts | 29 +++++++++++ src/lib/onboard/setup-nim-ollama.ts | 25 +++++---- 8 files changed, 130 insertions(+), 41 deletions(-) diff --git a/src/lib/inference/context-window.test.ts b/src/lib/inference/context-window.test.ts index 8a1127dd5e1..d130f085a14 100644 --- a/src/lib/inference/context-window.test.ts +++ b/src/lib/inference/context-window.test.ts @@ -160,7 +160,7 @@ describe("resolveContextWindowForModel default dependencies (#8974)", () => { cleanup(); } }); - vi.mocked(getOllamaProbeCommand).mockReturnValue(["docker", "run", "ollama-probe"]); + vi.mocked(getOllamaProbeCommand).mockReturnValue(["curl", "ollama-probe"]); vi.mocked(resolveOllamaRuntimeContextWindow).mockReturnValue(16384); expect(resolveContextWindowForModel("ollama-local", "qwen3.5:9b")).toBe(16384); diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index b5724cbdfa1..7f8aaf72ba9 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -366,16 +366,18 @@ describe("Windows-host Ollama transport", () => { it("validates a Windows-host model through Docker Desktop (#10553)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const cleanup = vi.fn(() => ({ ok: true as const })); const capture = respondsOnlyThroughDockerDesktop( "/api/show", JSON.stringify({ capabilities: ["tools"] }), ); - const captureEx = vi.fn((command: readonly string[]) => { + const captureEx = vi.fn((command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => { const expected = command[0] === "docker" && command[1] === "run" && command[2] === "--rm" && command[3] === CONTAINER_REACHABILITY_IMAGE && + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" && command.some((argument) => argument === "http://host.docker.internal:11434/api/generate"); return { stdout: expected ? JSON.stringify({ done: true, response: "ready" }) : "", @@ -385,10 +387,17 @@ describe("Windows-host Ollama transport", () => { }; }); - expect(validateOllamaModel("qwen3.5:9b", capture, () => false, captureEx)).toEqual({ - ok: true, - }); + expect( + validateOllamaModel("qwen3.5:9b", capture, () => false, captureEx, { + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + }), + ).toEqual({ ok: true }); expect(captureEx).toHaveBeenCalledOnce(); + expect(cleanup).toHaveBeenCalledOnce(); }); it("validates health and container reachability through Docker Desktop (#10553)", () => { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 0f1a3079a66..8949d67c529 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -2035,8 +2035,9 @@ export function getOllamaProbeCommand( }); const host = getResolvedOllamaHost(); const endpoint = `http://${host}:${OLLAMA_PORT}/api/generate`; - return getOllamaApiCommand( - buildValidatedCurlCommandArgs([ + return [ + "curl", + ...buildValidatedCurlCommandArgs([ "-sS", "--max-time", String(timeoutSeconds), @@ -2046,8 +2047,7 @@ export function getOllamaProbeCommand( payload, endpoint, ]), - host, - ); + ]; } export function validateOllamaModel( @@ -2055,10 +2055,17 @@ export function validateOllamaModel( runCaptureImpl?: RunCaptureFn, isSparkImpl?: () => boolean, runCaptureExImpl?: RunCaptureExFn, - options: { allowToolsIncompatible?: boolean } = {}, + options: { + allowToolsIncompatible?: boolean; + prepareDockerEnvironment?: PrepareDockerEnvironmentFn; + } = {}, ): ValidationResult { const capture = runCaptureImpl ?? runCapture; - const captureEx = createOllamaApiCaptureEx(runCaptureExImpl ?? runCaptureEx); + const captureEx = createOllamaApiCaptureEx( + runCaptureExImpl ?? runCaptureEx, + getResolvedOllamaHost(), + options.prepareDockerEnvironment, + ); const isSpark = isSparkImpl ?? (() => detectNvidiaPlatform() === "spark"); const sparkHost = isSpark(); const probeCmd = getOllamaProbeCommand(model); diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 6c29db324f5..ece44486e7c 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -206,6 +206,7 @@ function captureDefaultWindowsRollbackInvocation(userHost: string | null) { try { expect(windows.setupWindowsOllamaWith0000Binding({ installedPath: daemonPath })).toEqual({ ok: false, + reason: "readiness", }); } finally { restore(); @@ -254,7 +255,7 @@ function createWindowsInstallBoundary(options: { } describe("Windows Ollama helper", () => { - it("routes timeout recovery through the credential-isolated NemoClaw probe", () => { + it("describes Docker-client isolation in Windows Ollama timeout diagnostics", () => { const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); @@ -272,6 +273,24 @@ describe("Windows Ollama helper", () => { expect(diagnostic).not.toContain("docker run"); }); + it("describes a snapshot inspection failure without claiming a startup timeout", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + windows.printWindowsOllamaSnapshotDiagnostics(); + } finally { + restore(); + } + + const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + errorSpy.mockRestore(); + expect(diagnostic).toContain("User-scope OLLAMA_HOST"); + expect(diagnostic).toContain("existing Ollama processes"); + expect(diagnostic).toContain("nemoclaw onboard"); + expect(diagnostic).not.toContain("Timed out waiting for Ollama to start"); + }); + it("continues probing after a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); const localInference = require(LOCAL_INFERENCE_PATH); @@ -377,7 +396,7 @@ describe("Windows Ollama helper", () => { { installedPath: daemonPath }, boundary.operations, ), - ).toEqual({ ok: false }); + ).toEqual({ ok: false, reason: "readiness" }); } finally { restore(); logSpy.mockRestore(); @@ -406,6 +425,7 @@ describe("Windows Ollama helper", () => { try { expect(windows.setupWindowsOllamaWith0000Binding({}, boundary.operations)).toEqual({ ok: false, + reason: "readiness", }); } finally { restore(); @@ -608,7 +628,7 @@ describe("Windows Ollama helper", () => { { installedPath: daemonPath }, boundary.operations, ), - ).toEqual({ ok: false }); + ).toEqual({ ok: false, reason: "readiness" }); } finally { restore(); logSpy.mockRestore(); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 94cef03995f..b9af285d5a0 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -1,4 +1,3 @@ -// @ts-nocheck // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // @@ -327,16 +326,20 @@ type WindowsOllamaInstallOperations = WindowsOllamaSetupOperations & { awaitReady: () => boolean; }; -type WindowsOllamaMutationSession = { +export type WindowsOllamaMutationSession = { commit: () => void; rollback: () => void; }; -type WindowsOllamaInstallResult = - | { ok: false; path: string; reason: "install" | "readiness" } +export type WindowsOllamaFailureReason = "binding" | "install" | "readiness" | "snapshot"; + +export type WindowsOllamaInstallResult = + | { ok: false; path: string; reason: WindowsOllamaFailureReason } | ({ ok: true; path: string } & WindowsOllamaMutationSession); -type WindowsOllamaSetupResult = { ok: false } | ({ ok: true } & WindowsOllamaMutationSession); +export type WindowsOllamaSetupResult = + | { ok: false; reason: Exclude } + | ({ ok: true } & WindowsOllamaMutationSession); function registerWindowsOllamaInterruptHandler( handler: (signal: WindowsOllamaInterruptSignal) => void, @@ -428,10 +431,10 @@ function applyWindowsOllamaBinding( snapshot: WindowsOllamaHostSnapshot, operations: WindowsOllamaSetupOperations, markMutated: () => void, -): boolean { +): { ok: true } | { ok: false; reason: "binding" | "readiness" } { if (!operations.persistBinding()) { console.error(" Could not persist the Windows Ollama host binding."); - return false; + return { ok: false, reason: "binding" }; } markMutated(); if (opts.announceStop) { @@ -445,7 +448,9 @@ function applyWindowsOllamaBinding( installedPath: opts.installedPath, }, operations.launchOperations, - ); + ) + ? { ok: true } + : { ok: false, reason: "readiness" }; } async function installOllamaOnWindowsHost( @@ -455,7 +460,7 @@ async function installOllamaOnWindowsHost( const snapshot = operations.captureSnapshot(); if (!snapshot) { console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); - return { ok: false, path: "", reason: "install" }; + return { ok: false, path: "", reason: "snapshot" }; } const mutation = beginWindowsOllamaMutation(snapshot, operations); mutation.markMutated(); @@ -475,11 +480,15 @@ async function installOllamaOnWindowsHost( if (!operations.awaitReady()) { console.log(" Installer did not leave a reachable Ollama daemon; restarting it..."); opts.beforeRestart?.(); - if ( - !applyWindowsOllamaBinding({ installedPath }, snapshot, operations, mutation.markMutated) - ) { + const setupResult = applyWindowsOllamaBinding( + { installedPath }, + snapshot, + operations, + mutation.markMutated, + ); + if (!setupResult.ok) { mutation.rollback(); - return { ok: false, path: installedPath, reason: "readiness" }; + return { ok: false, path: installedPath, reason: setupResult.reason }; } } return { ok: true, path: installedPath, commit: mutation.commit, rollback: mutation.rollback }; @@ -499,13 +508,14 @@ function setupWindowsOllamaWith0000Binding( const snapshot = operations.captureSnapshot(); if (!snapshot) { console.error(" Could not capture the existing Windows Ollama state; leaving it unchanged."); - return { ok: false }; + return { ok: false, reason: "snapshot" }; } const mutation = beginWindowsOllamaMutation(snapshot, operations); try { - if (!applyWindowsOllamaBinding(opts, snapshot, operations, mutation.markMutated)) { + const setupResult = applyWindowsOllamaBinding(opts, snapshot, operations, mutation.markMutated); + if (!setupResult.ok) { mutation.rollback(); - return { ok: false }; + return setupResult; } return { ok: true, commit: mutation.commit, rollback: mutation.rollback }; } catch (error) { @@ -533,11 +543,20 @@ function printWindowsOllamaTimeoutDiagnostics(): void { ); } +function printWindowsOllamaSnapshotDiagnostics(): void { + console.error(" NemoClaw could not inspect the existing Windows Ollama state."); + console.error( + " In Windows PowerShell, verify that the current user can query the User-scope OLLAMA_HOST value and existing Ollama processes, then retry:", + ); + console.error(" nemoclaw onboard"); +} + module.exports = { installOllamaOnWindowsHost, awaitWindowsOllamaReady, setupWindowsOllamaWith0000Binding, sleep: sleepSeconds, switchToWindowsOllamaHost, + printWindowsOllamaSnapshotDiagnostics, printWindowsOllamaTimeoutDiagnostics, }; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index adadc1edfc1..dd943545dcb 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -231,6 +231,7 @@ const { installOllamaOnWindowsHost, setupWindowsOllamaWith0000Binding, switchToWindowsOllamaHost, + printWindowsOllamaSnapshotDiagnostics, printWindowsOllamaTimeoutDiagnostics, } = require("./inference/ollama/windows"); const vllmInference = require("./inference/vllm"); @@ -995,6 +996,7 @@ const { switchToWindowsOllamaHost, installOllamaOnWindowsHost, setupWindowsOllamaWith0000Binding, + printWindowsOllamaSnapshotDiagnostics, printWindowsOllamaTimeoutDiagnostics, resetOllamaHostCache, installOllamaOnMacOS, diff --git a/src/lib/onboard/setup-nim-ollama.test.ts b/src/lib/onboard/setup-nim-ollama.test.ts index 8b8b691dbad..d52762310c2 100644 --- a/src/lib/onboard/setup-nim-ollama.test.ts +++ b/src/lib/onboard/setup-nim-ollama.test.ts @@ -60,6 +60,7 @@ function makeDeps(overrides: Partial = {}): Deps { commit: () => {}, rollback: () => {}, }), + printWindowsOllamaSnapshotDiagnostics: () => {}, printWindowsOllamaTimeoutDiagnostics: () => {}, resetOllamaHostCache: () => {}, installOllamaOnMacOS: () => ({ ok: true }), @@ -346,6 +347,34 @@ describe("createSetupNimOllamaHandlers", () => { expect(start).not.toHaveBeenCalled(); }); + it("reports a Windows snapshot failure without claiming a startup timeout", async () => { + const snapshotDiagnostic = vi.fn(); + const timeoutDiagnostic = vi.fn(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + isNonInteractive: () => false, + printWindowsOllamaSnapshotDiagnostics: snapshotDiagnostic, + printWindowsOllamaTimeoutDiagnostics: timeoutDiagnostic, + setupWindowsOllamaWith0000Binding: () => ({ ok: false, reason: "snapshot" }), + }), + ); + + await expect( + handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "qwen3:8b", + false, + true, + "C:/Ollama/ollama.exe", + makeState(), + ), + ).resolves.toBe("retry-selection"); + + expect(snapshotDiagnostic).toHaveBeenCalledOnce(); + expect(timeoutDiagnostic).not.toHaveBeenCalled(); + }); + it("commits a new Windows Ollama install after model selection", async () => { const commit = vi.fn(); const rollback = vi.fn(); diff --git a/src/lib/onboard/setup-nim-ollama.ts b/src/lib/onboard/setup-nim-ollama.ts index da72dbd5895..9e8a166bc03 100644 --- a/src/lib/onboard/setup-nim-ollama.ts +++ b/src/lib/onboard/setup-nim-ollama.ts @@ -3,6 +3,10 @@ import type { OllamaStartupOutcome } from "./ollama-startup"; import type { SetupNimSelectionState } from "./setup-nim-selection"; +import type { + WindowsOllamaInstallResult, + WindowsOllamaSetupResult, +} from "../inference/ollama/windows"; const { getRequestedModelFromEnv, @@ -12,14 +16,6 @@ const { type SetupNimSelectionResult = "selected" | "retry-selection"; -type WindowsOllamaInstallResult = - | { ok: false; path: string; reason: "install" | "readiness" } - | { ok: true; path: string; commit: () => void; rollback: () => void }; - -type WindowsOllamaSetupResult = - | { ok: false } - | { ok: true; commit: () => void; rollback: () => void }; - type SetupNimOllamaDeps = { OLLAMA_PORT: number; OLLAMA_PROXY_PORT: number; @@ -64,6 +60,7 @@ type SetupNimOllamaDeps = { announceStop?: boolean; installedPath?: string | null; }) => WindowsOllamaSetupResult; + printWindowsOllamaSnapshotDiagnostics?: () => void; printWindowsOllamaTimeoutDiagnostics: () => void; resetOllamaHostCache: () => void; installOllamaOnMacOS: (args: { @@ -237,9 +234,11 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { state.revalidateSandboxIdentity?.("start the Windows Ollama runtime"), }); if (!installResult.ok) { - if (installResult.reason === "readiness") { + if (installResult.reason === "snapshot") { + deps.printWindowsOllamaSnapshotDiagnostics?.(); + } else if (installResult.reason === "readiness") { deps.printWindowsOllamaTimeoutDiagnostics(); - } else { + } else if (installResult.reason === "install") { console.error( " Install did not produce ollama.exe on PATH. Check the installer output above.", ); @@ -256,7 +255,11 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { installedPath: winOllamaInstalledPath || undefined, }); if (!setupResult.ok) { - deps.printWindowsOllamaTimeoutDiagnostics(); + if (setupResult.reason === "snapshot") { + deps.printWindowsOllamaSnapshotDiagnostics?.(); + } else if (setupResult.reason === "readiness") { + deps.printWindowsOllamaTimeoutDiagnostics(); + } if (deps.isNonInteractive()) deps.process.exit(1); return "retry-selection"; } From 3309602b449f0749bf29d8e7f3a565db33afd1a7 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 11:04:22 -0700 Subject: [PATCH 34/48] fix(deps): update transitive fast-uri security patch Signed-off-by: Prekshi Vyas --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6dc8be86bfe..1922d8d6382 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6281,9 +6281,9 @@ "peer": true }, "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "funding": [ { "type": "github", From ad04a917ad8aaf20ad8112e147bc0ca7eccb107b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 17:52:50 -0700 Subject: [PATCH 35/48] test(onboard): use exact Ollama URL assertion --- src/lib/onboard/provider-host-state.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index b668b039a85..302b93cf7b0 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -324,9 +324,9 @@ describe("detectInferenceProviderHostState", () => { const state = detectWithDeps(deps); expect(state.windowsOllamaReachable).toBe(false); - expect( - runCapture.mock.calls.some(([command]) => command.includes(WINDOWS_OLLAMA_TAGS_URL)), - ).toBe(false); + expect(runCapture.mock.calls.flatMap(([command]) => command)).not.toContain( + WINDOWS_OLLAMA_TAGS_URL, + ); }); it("passes injected platform and env through WSL detection", () => { From e5238253e07a1ed9f26e2545beacb9167c9905ed Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 2 Sep 2026 18:33:47 -0700 Subject: [PATCH 36/48] fix(agent): resolve ambiguous Ollama recovery host --- .../agent/ollama-restart-recovery.test.ts | 45 +++++++++++++++++-- .../sandbox/agent/ollama-restart-recovery.ts | 2 +- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index aee5388a149..91227e0e62e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -110,7 +110,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); }); - it("uses the persisted direct bridge route for both the default probe and warm-up", async () => { + it("uses the resolved Windows host for an ambiguous direct bridge route", async () => { const runRecoveryCaptureImpl = scriptedRecoveryCapture( unloadedProbeResult(), successfulWarmResult(), @@ -123,7 +123,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { runRecoveryCaptureImpl }, + { getOllamaHost: () => "host.docker.internal", runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "warmed", ok: true }); @@ -144,6 +144,39 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ]); }); + it("uses the resolved WSL-local host for an ambiguous direct bridge route", async () => { + const prepareDockerEnvironment = vi.fn(() => { + throw new Error("unexpected Docker transport"); + }); + const spawnRecoveryChild = completingRecoverySpawner([ + unloadedProbeResult().stdout, + successfulWarmResult().stdout, + ]); + + await expect( + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, + }, + { + getOllamaHost: () => "127.0.0.1", + prepareDockerEnvironment, + spawnRecoveryChild, + }, + ), + ).resolves.toEqual({ kind: "warmed", ok: true }); + + const commands = spawnRecoveryChild.mock.calls.map(([binary, args]) => [binary, ...args]); + expect(commands.map(getCommandUrl)).toEqual([ + `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, + `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, + ]); + expect(commands.map(([binary]) => binary)).toEqual(["curl", "curl"]); + expect(prepareDockerEnvironment).not.toHaveBeenCalled(); + }); + it("runs the production status probe and warm-up through the async capture boundary", async () => { const runRecoveryCaptureImpl = vi .fn() @@ -212,7 +245,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { spawnRecoveryChild, prepareDockerEnvironment }, + { + getOllamaHost: () => "host.docker.internal", + spawnRecoveryChild, + prepareDockerEnvironment, + }, ), ).resolves.toMatchObject({ kind: "skipped", reason: "model-absent" }); @@ -560,7 +597,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "gemma4:26b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { runRecoveryCaptureImpl }, + { getOllamaHost: () => "host.docker.internal", runRecoveryCaptureImpl }, ), ).resolves.toEqual({ kind: "skipped", diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index b0aed6a66b7..58ab19448b7 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -133,7 +133,7 @@ function resolveRawOllamaHost( hostname === OPENSHELL_HOST_BRIDGE && port === OLLAMA_PORT ) { - return OLLAMA_HOST_DOCKER_INTERNAL; + return getAllowedFallbackHost(getOllamaHost); } if ( endpoint.protocol === "http:" && From d3286d28e213c44c7f7cc00de12b2009ddb75285 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 3 Sep 2026 01:03:43 -0700 Subject: [PATCH 37/48] fix(ollama): make recovery rollback transactional --- .../agent/ollama-restart-recovery.test.ts | 23 +++ .../local-windows-ollama-transport.test.ts | 76 ++++++++- src/lib/inference/local.ts | 34 +++- src/lib/inference/ollama/windows.test.ts | 54 +++++- src/lib/inference/ollama/windows.ts | 159 ++++++++++++++---- src/lib/onboard.ts | 22 +-- src/lib/onboard/ollama-probe-failure.test.ts | 90 +++++----- src/lib/onboard/ollama-probe-failure.ts | 46 ++++- src/lib/onboard/setup-nim-ollama.test.ts | 40 +++++ src/lib/onboard/setup-nim-ollama.ts | 31 +++- 10 files changed, 469 insertions(+), 106 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 91227e0e62e..5e58e68872a 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -6,6 +6,7 @@ import type { StdioOptions } from "node:child_process"; import { describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { CONTAINER_REACHABILITY_IMAGE } from "../../../inference/local"; import type { SandboxExecSignalSource } from "../exec"; import type { AgentDispatchChild } from "./passthrough-dispatch"; import { @@ -260,6 +261,23 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ]); expect(commands.every((command) => command[0] === "docker")).toBe(true); + const proxyGuard = [ + "HTTP_PROXY=", + "http_proxy=", + "HTTPS_PROXY=", + "https_proxy=", + "ALL_PROXY=", + "all_proxy=", + "FTP_PROXY=", + "ftp_proxy=", + "NO_PROXY=host.docker.internal", + "no_proxy=host.docker.internal", + ]; + expect(commands).toEqual([ + expect.arrayContaining([...proxyGuard, CONTAINER_REACHABILITY_IMAGE]), + expect.arrayContaining([...proxyGuard, CONTAINER_REACHABILITY_IMAGE]), + expect.arrayContaining([...proxyGuard, CONTAINER_REACHABILITY_IMAGE]), + ]); expect(spawnRecoveryChild.mock.calls.map(([, , , env]) => env.DOCKER_CONFIG)).toEqual([ "/tmp/credential-free-docker-1", "/tmp/credential-free-docker-2", @@ -605,6 +623,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, inventoryLabel: "llama3.2:1b", }); + expect(runRecoveryCaptureImpl).toHaveBeenCalledTimes(3); expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[2]?.[0] ?? [])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/tags`, ); @@ -629,6 +648,10 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: "http://127.0.0.1:11434", detail: expect.stringContaining("runner stopped unexpectedly"), }); + expect(runRecoveryCaptureImpl).toHaveBeenCalledTimes(3); + expect(getCommandUrl(runRecoveryCaptureImpl.mock.calls[2]?.[0] ?? [])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/tags`, + ); }); it("accepts a completed thinking-only response from a thinking model", async () => { diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 7f8aaf72ba9..b15da2af814 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -18,6 +18,7 @@ import { OLLAMA_HOST_DOCKER_INTERNAL, loadPersistedOllamaHost, persistResolvedOllamaHost, + prepareOllamaApiExecution, probeLocalProviderHealth, probeOllamaModelCapabilities, resetOllamaHostCache, @@ -65,6 +66,26 @@ describe("Windows-host Ollama transport", () => { "docker", "run", "--rm", + "--env", + "HTTP_PROXY=", + "--env", + "http_proxy=", + "--env", + "HTTPS_PROXY=", + "--env", + "https_proxy=", + "--env", + "ALL_PROXY=", + "--env", + "all_proxy=", + "--env", + "FTP_PROXY=", + "--env", + "ftp_proxy=", + "--env", + "NO_PROXY=host.docker.internal", + "--env", + "no_proxy=host.docker.internal", CONTAINER_REACHABILITY_IMAGE, "-sf", "--connect-timeout", @@ -80,6 +101,51 @@ describe("Windows-host Ollama transport", () => { ]); }); + it("overrides healthy Docker-config proxies without changing Docker client authority", () => { + const cleanup = vi.fn(() => ({ ok: true as const })); + const dockerEnv = { + DOCKER_CONFIG: "/tmp/healthy-docker-config", + DOCKER_CONTEXT: "desktop-linux", + DOCKER_HOST: "npipe:////./pipe/dockerDesktopLinuxEngine", + HTTPS_PROXY: "https://operator:private-token@proxy.example", + }; + const execution = prepareOllamaApiExecution( + ["curl", "-sf", "http://host.docker.internal:11434/api/tags"], + OLLAMA_HOST_DOCKER_INTERNAL, + { + env: dockerEnv, + prepareDockerEnvironment: () => ({ + env: dockerEnv, + isolatedCredentialConfig: false, + cleanup, + }), + }, + ); + + expect(execution.env).toEqual(dockerEnv); + expect(execution.command).toEqual( + expect.arrayContaining([ + "HTTP_PROXY=", + "http_proxy=", + "HTTPS_PROXY=", + "https_proxy=", + "ALL_PROXY=", + "all_proxy=", + "FTP_PROXY=", + "ftp_proxy=", + "NO_PROXY=host.docker.internal", + "no_proxy=host.docker.internal", + ]), + ); + expect(execution.command.join(" ")).not.toContain("private-token"); + expect(execution.command.indexOf("HTTPS_PROXY=")).toBeLessThan( + execution.command.indexOf(CONTAINER_REACHABILITY_IMAGE), + ); + + execution.cleanup(); + expect(cleanup).toHaveBeenCalledOnce(); + }); + it("restores the accepted route receipt in a fresh process", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-receipt-")); try { @@ -248,7 +314,7 @@ describe("Windows-host Ollama transport", () => { const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-discovery-")); const commands: (readonly string[])[] = []; const endpoints: string[] = []; - const capture = vi.fn((command: readonly string[]) => { + const capture = vi.fn((command: readonly string[], _options?: unknown) => { commands.push(command); const endpoint = command.at(-1) ?? ""; endpoints.push(endpoint); @@ -276,6 +342,10 @@ describe("Windows-host Ollama transport", () => { "5", ]), ); + expect(capture.mock.calls[1]?.[1]).toMatchObject({ + ignoreError: true, + timeout: 5_000, + }); } finally { rmSync(stateRoot, { recursive: true, force: true }); } @@ -328,7 +398,7 @@ describe("Windows-host Ollama transport", () => { command[0] === "docker" && command[1] === "run" && command[2] === "--rm" && - command[3] === CONTAINER_REACHABILITY_IMAGE && + command.includes(CONTAINER_REACHABILITY_IMAGE) && command.some((argument) => argument === "http://host.docker.internal:11434/api/tags") ? JSON.stringify({ models: [] }) : "", @@ -376,7 +446,7 @@ describe("Windows-host Ollama transport", () => { command[0] === "docker" && command[1] === "run" && command[2] === "--rm" && - command[3] === CONTAINER_REACHABILITY_IMAGE && + command.includes(CONTAINER_REACHABILITY_IMAGE) && options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" && command.some((argument) => argument === "http://host.docker.internal:11434/api/generate"); return { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 8949d67c529..7d1b0e91e5f 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -204,7 +204,7 @@ export function findReachableOllamaHost( "5", `http://${host}:${OLLAMA_PORT}/api/tags`, ], - { ignoreError: true }, + { ignoreError: true, timeout: 5_000 }, ); if (isValidOllamaTagsResponseBody(result)) { _resolvedOllamaHost = host; @@ -278,12 +278,42 @@ export function clearPersistedOllamaHostIfUnused( } /** Keep Windows-host Ollama requests in Docker Desktop's verified network context. */ +const OLLAMA_DOCKER_PROXY_GUARD_ARGS = [ + "--env", + "HTTP_PROXY=", + "--env", + "http_proxy=", + "--env", + "HTTPS_PROXY=", + "--env", + "https_proxy=", + "--env", + "ALL_PROXY=", + "--env", + "all_proxy=", + "--env", + "FTP_PROXY=", + "--env", + "ftp_proxy=", + "--env", + `NO_PROXY=${OLLAMA_HOST_DOCKER_INTERNAL}`, + "--env", + `no_proxy=${OLLAMA_HOST_DOCKER_INTERNAL}`, +] as const; + export function getOllamaApiCommand( curlArgs: readonly string[], host: string = getResolvedOllamaHost(), ): string[] { return host === OLLAMA_HOST_DOCKER_INTERNAL - ? ["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE, ...curlArgs] + ? [ + "docker", + "run", + "--rm", + ...OLLAMA_DOCKER_PROXY_GUARD_ARGS, + CONTAINER_REACHABILITY_IMAGE, + ...curlArgs, + ] : ["curl", ...curlArgs]; } diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index ece44486e7c..9a204e4a9b1 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -227,9 +227,13 @@ function createWindowsInstallBoundary(options: { readinessResults?: boolean[]; rollbackStatus?: number; installerInterrupt?: "SIGINT" | "SIGTERM"; + cancelInstaller?: () => Promise; }) { const boundary = createWindowsSetupBoundary(options); - const cancelInstaller = vi.fn(() => boundary.state.events.push("cancel-installer")); + const cancelInstaller = vi.fn(async () => { + boundary.state.events.push("cancel-installer"); + await options.cancelInstaller?.(); + }); const completion = options.installerInterrupt ? Promise.resolve().then(() => boundary.triggerInterrupt(options.installerInterrupt!)) : Promise.resolve(); @@ -240,7 +244,7 @@ function createWindowsInstallBoundary(options: { boundary.state.userHost = "0.0.0.0:11434"; boundary.state.watcherRunning = true; boundary.state.daemonRunning = true; - return { completion, cancel: cancelInstaller }; + return { completion, cancelAndWait: cancelInstaller }; }), resolveInstalledPath: vi.fn(() => { boundary.state.events.push("resolve-path"); @@ -549,20 +553,30 @@ describe("Windows Ollama helper", () => { it("cancels install and restores prior Windows state before preserving SIGTERM", async () => { const priorHost = "127.0.0.1:11434"; const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + let finishProcessTreeCancellation = () => {}; + const processTreeDrained = new Promise((resolve) => { + finishProcessTreeCancellation = resolve; + }); const boundary = createWindowsInstallBoundary({ userHost: priorHost, watcherPath, daemonPath: null, installerInterrupt: "SIGTERM", + cancelInstaller: () => processTreeDrained, }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); try { - await expect(windows.installOllamaOnWindowsHost({}, boundary.operations)).rejects.toThrow( - PreservedWindowsInterrupt, - ); + const installation = windows.installOllamaOnWindowsHost({}, boundary.operations); + await vi.waitFor(() => expect(boundary.cancelInstaller).toHaveBeenCalledOnce()); + expect(boundary.state.events).toEqual(["snapshot", "install", "cancel-installer"]); + expect(boundary.operations.rollbackSnapshot).not.toHaveBeenCalled(); + expect(boundary.operations.preserveInterrupt).not.toHaveBeenCalled(); + + finishProcessTreeCancellation(); + await expect(installation).rejects.toThrow(PreservedWindowsInterrupt); } finally { restore(); logSpy.mockRestore(); @@ -582,6 +596,36 @@ describe("Windows Ollama helper", () => { expect(boundary.cancelInstaller).toHaveBeenCalledOnce(); }); + it("does not restore mutable Windows state when installer tree cancellation fails", async () => { + const boundary = createWindowsInstallBoundary({ + userHost: "127.0.0.1:11434", + watcherPath: "C:\\Users\\tester\\Ollama\\ollama app.exe", + daemonPath: null, + installerInterrupt: "SIGINT", + cancelInstaller: async () => { + throw new Error("taskkill tree termination failed"); + }, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + await expect(windows.installOllamaOnWindowsHost({}, boundary.operations)).rejects.toThrow( + "taskkill tree termination failed", + ); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(boundary.cancelInstaller).toHaveBeenCalledOnce(); + expect(boundary.operations.rollbackSnapshot).not.toHaveBeenCalled(); + expect(boundary.operations.preserveInterrupt).not.toHaveBeenCalled(); + expect(boundary.state.events).toEqual(["snapshot", "install", "cancel-installer"]); + }); + it("reports manual recovery when install rollback fails", async () => { const boundary = createWindowsInstallBoundary({ userHost: "127.0.0.1:11434", diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index b9af285d5a0..fb2cce10a58 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -21,9 +21,39 @@ function psSingleQuote(value: string): string { type WindowsOllamaInstallerProcess = { completion: Promise; - cancel: () => void; + cancelAndWait: () => Promise; }; +const WINDOWS_INSTALLER_PID_SENTINEL = "__NEMOCLAW_WINDOWS_INSTALLER_PID__:"; + +function parseWindowsProcessId(value: string): number | null { + if (!/^[1-9][0-9]*$/.test(value)) return null; + const pid = Number(value); + return Number.isSafeInteger(pid) && pid <= 0xffff_ffff ? pid : null; +} + +function terminateWindowsProcessTree(pid: number): Promise { + return new Promise((resolve, reject) => { + const taskkill = spawn("taskkill.exe", ["/PID", String(pid), "/T", "/F"], { + stdio: "ignore", + }); + taskkill.once("error", (error: NodeJS.ErrnoException) => { + reject( + new Error( + `Failed to start taskkill.exe for Windows installer PID ${pid}: ${error.message}`, + ), + ); + }); + taskkill.once("close", (code: number | null) => { + if (code === 0) resolve(); + else + reject( + new Error(`taskkill.exe failed for Windows installer PID ${pid} (exit ${String(code)})`), + ); + }); + }); +} + // Pre-set OLLAMA_HOST in both User scope (persists across logins) and the // current PowerShell session (inherited by the installer's auto-spawned // ollama_app + daemon) so the new daemon binds 0.0.0.0 from the start. @@ -37,27 +67,71 @@ function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { "powershell.exe", [ "-Command", - "[Environment]::SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User'); $env:OLLAMA_HOST='0.0.0.0:11434'; irm https://ollama.com/install.ps1 | iex", + `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; [Console]::Out.WriteLine('${WINDOWS_INSTALLER_PID_SENTINEL}' + $PID); [Console]::Out.Flush(); ` + + "[Environment]::SetEnvironmentVariable('OLLAMA_HOST','0.0.0.0:11434','User'); $env:OLLAMA_HOST='0.0.0.0:11434'; irm https://ollama.com/install.ps1 | iex", ], { stdio: ["ignore", "pipe", "pipe"] }, ); + let windowsPid: number | null = null; + let stdoutBuffer = ""; + let resolveWindowsPid: (pid: number | null) => void = () => {}; + const windowsPidReady = new Promise((resolve) => { + resolveWindowsPid = resolve; + }); + const flushStdoutLines = (final: boolean) => { + while (true) { + const newlineIndex = stdoutBuffer.indexOf("\n"); + if (newlineIndex < 0 && !final) return; + const line = newlineIndex < 0 ? stdoutBuffer : stdoutBuffer.slice(0, newlineIndex + 1); + stdoutBuffer = newlineIndex < 0 ? "" : stdoutBuffer.slice(newlineIndex + 1); + if (!line) return; + const candidate = line.replace(/\r?\n$/, "").slice(WINDOWS_INSTALLER_PID_SENTINEL.length); + const parsedPid = line.startsWith(WINDOWS_INSTALLER_PID_SENTINEL) + ? parseWindowsProcessId(candidate) + : null; + if (parsedPid !== null && windowsPid === null) { + windowsPid = parsedPid; + resolveWindowsPid(parsedPid); + } else { + process.stdout.write(line); + } + } + }; const completion = new Promise((resolve) => { - child.stdout?.on("data", (chunk: Buffer) => process.stdout.write(chunk)); + child.stdout?.on("data", (chunk: Buffer) => { + stdoutBuffer += chunk.toString("utf8"); + flushStdoutLines(false); + }); child.stderr?.on("data", (chunk: Buffer) => process.stderr.write(chunk)); - child.on("close", () => resolve()); + child.on("close", () => { + flushStdoutLines(true); + resolveWindowsPid(windowsPid); + resolve(); + }); child.on("error", (err: NodeJS.ErrnoException) => { console.error(` Failed to spawn powershell.exe: ${err.message}`); + resolveWindowsPid(null); resolve(); }); }); + let cancellation: Promise | null = null; return { completion, - cancel: () => { - try { - child.kill("SIGTERM"); - } catch { - // Rollback below still stops any installer-created Ollama processes. - } + cancelAndWait: () => { + cancellation ??= (async () => { + const pid = await windowsPidReady; + if (pid !== null) { + await terminateWindowsProcessTree(pid); + } else { + try { + child.kill("SIGTERM"); + } catch { + // The wrapper has already exited, so completion below is authoritative. + } + } + await completion; + })(); + return cancellation; }, }; } @@ -316,7 +390,9 @@ type WindowsOllamaSetupOperations = { wait: (seconds: number) => void; launchOperations: WindowsOllamaLaunchOperations; rollbackSnapshot: (snapshot: WindowsOllamaHostSnapshot) => boolean; - registerInterruptHandler: (handler: (signal: WindowsOllamaInterruptSignal) => void) => () => void; + registerInterruptHandler: ( + handler: (signal: WindowsOllamaInterruptSignal) => void | Promise, + ) => () => void; preserveInterrupt: (signal: WindowsOllamaInterruptSignal) => void; }; @@ -328,7 +404,7 @@ type WindowsOllamaInstallOperations = WindowsOllamaSetupOperations & { export type WindowsOllamaMutationSession = { commit: () => void; - rollback: () => void; + rollback: () => void | Promise; }; export type WindowsOllamaFailureReason = "binding" | "install" | "readiness" | "snapshot"; @@ -342,10 +418,21 @@ export type WindowsOllamaSetupResult = | ({ ok: true } & WindowsOllamaMutationSession); function registerWindowsOllamaInterruptHandler( - handler: (signal: WindowsOllamaInterruptSignal) => void, + handler: (signal: WindowsOllamaInterruptSignal) => void | Promise, ): () => void { - const onSigint = () => handler("SIGINT"); - const onSigterm = () => handler("SIGTERM"); + const reportFailure = (error: unknown) => { + console.error(` Windows Ollama interrupt rollback failed: ${String(error)}`); + process.exitCode = 1; + }; + const runHandler = (signal: WindowsOllamaInterruptSignal) => { + try { + Promise.resolve(handler(signal)).catch(reportFailure); + } catch (error) { + reportFailure(error); + } + }; + const onSigint = () => runHandler("SIGINT"); + const onSigterm = () => runHandler("SIGTERM"); process.once("SIGINT", onSigint); process.once("SIGTERM", onSigterm); return () => { @@ -387,32 +474,40 @@ function beginWindowsOllamaMutation( ) { let active = true; let mutated = false; - let cancelActiveOperation = () => {}; + let cancelActiveOperation: () => void | Promise = () => {}; let removeInterruptHandler = () => {}; + let rollbackPromise: Promise | null = null; + let interruptPromise: Promise | null = null; const commit = () => { if (!active) return; active = false; cancelActiveOperation = () => {}; removeInterruptHandler(); }; - const rollback = () => { + const restore = () => { + if (mutated) rollbackWindowsOllamaSetup(snapshot, operations); + }; + const rollback = (): void | Promise => { + if (rollbackPromise) return rollbackPromise; if (!active) return; active = false; removeInterruptHandler(); const cancel = cancelActiveOperation; cancelActiveOperation = () => {}; - try { - cancel(); - } finally { - if (mutated) rollbackWindowsOllamaSetup(snapshot, operations); + const cancellation = cancel(); + if (cancellation && typeof cancellation.then === "function") { + rollbackPromise = Promise.resolve(cancellation).then(restore); + return rollbackPromise; } + restore(); }; removeInterruptHandler = operations.registerInterruptHandler((signal) => { - try { - rollback(); - } finally { - operations.preserveInterrupt(signal); + const result = rollback(); + if (result && typeof result.then === "function") { + interruptPromise = result.then(() => operations.preserveInterrupt(signal)); + return interruptPromise; } + operations.preserveInterrupt(signal); }); return { commit, @@ -420,9 +515,12 @@ function beginWindowsOllamaMutation( mutated = true; }, rollback, - setInterruptCancellation: (cancel: () => void) => { + setInterruptCancellation: (cancel: () => void | Promise) => { if (active) cancelActiveOperation = cancel; }, + waitForInterrupt: async () => { + if (interruptPromise) await interruptPromise; + }, }; } @@ -468,12 +566,13 @@ async function installOllamaOnWindowsHost( console.log(" This can take several minutes. Output may pause silently"); try { const installer = operations.startInstaller(); - mutation.setInterruptCancellation(installer.cancel); + mutation.setInterruptCancellation(installer.cancelAndWait); await installer.completion; mutation.setInterruptCancellation(() => {}); + await mutation.waitForInterrupt(); const installedPath = operations.resolveInstalledPath(); if (!installedPath) { - mutation.rollback(); + await mutation.rollback(); return { ok: false, path: "", reason: "install" }; } console.log(` ✓ Installed: ${installedPath}`); @@ -487,13 +586,13 @@ async function installOllamaOnWindowsHost( mutation.markMutated, ); if (!setupResult.ok) { - mutation.rollback(); + await mutation.rollback(); return { ok: false, path: installedPath, reason: setupResult.reason }; } } return { ok: true, path: installedPath, commit: mutation.commit, rollback: mutation.rollback }; } catch (error) { - mutation.rollback(); + await mutation.rollback(); throw error; } } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index bf0a29c9448..690db335a18 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -889,14 +889,14 @@ const verifyDirectSandboxGpu = sandboxGpuPreflight.createDirectSandboxGpuVerifie }); const registration = credentialProviderRegistration.createCredentialProviderRegistration({ - root: ROOT, - runOpenshell, - getGatewayName: () => GATEWAY_NAME, - getCredential, - updateSession: onboardSession.updateSession, - stagedLegacyValues, - migratedLegacyKeys, - persistMigratedLegacyKeys, + root: ROOT, + runOpenshell, + getGatewayName: () => GATEWAY_NAME, + getCredential, + updateSession: onboardSession.updateSession, + stagedLegacyValues, + migratedLegacyKeys, + persistMigratedLegacyKeys, }); const { upsertProvider, upsertMessagingProviders, providerMatchesGatewayCredential } = registration; const providerExistsInGateway = (name: string, gatewayName: string = GATEWAY_NAME) => @@ -1712,7 +1712,7 @@ async function selectAndValidateOllamaModel( "non-interactive mode cannot prompt for confirmation. " + "Re-run with --yes / -y (or NEMOCLAW_YES=1) to authorise the download.", ); - process.exit(1); + ollamaFlow.deferOllamaProcessExit(); } else { const proceed = await promptYesNoOrDefault( ` Download Ollama model '${selectedModel}' (${sizeLabel})?`, @@ -1743,7 +1743,7 @@ async function selectAndValidateOllamaModel( const allowToolsIncompatible = probe.allowToolsIncompatible === true; const validationBaseUrl = getLocalProviderValidationBaseUrl(provider); if (!validationBaseUrl) - abortNonInteractive("Local Ollama validation URL could not be determined."); + ollamaFlow.deferAbort("Local Ollama validation URL could not be determined."); const validation = await validateOpenAiLikeSelection( "Local Ollama", validationBaseUrl!, @@ -1755,7 +1755,7 @@ async function selectAndValidateOllamaModel( ); if (validation.retry === "selection") return { outcome: "back-to-selection" }; if (!validation.ok) { - if (isNonInteractive()) abortNonInteractive(`model '${selectedModel}' failed validation.`); + if (isNonInteractive()) ollamaFlow.deferAbort(`model '${selectedModel}' failed validation.`); continue; } // Ollama's /v1/responses endpoint does not produce correctly formatted diff --git a/src/lib/onboard/ollama-probe-failure.test.ts b/src/lib/onboard/ollama-probe-failure.test.ts index e4c3225da82..a5eef14e844 100644 --- a/src/lib/onboard/ollama-probe-failure.test.ts +++ b/src/lib/onboard/ollama-probe-failure.test.ts @@ -11,8 +11,19 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { completeOllamaRuntimeContextSelection, handleOllamaProbeFailure, + OllamaSelectionFatalError, } from "./ollama-probe-failure"; +function captureFatalSelection(run: () => unknown): OllamaSelectionFatalError { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(OllamaSelectionFatalError); + return error as OllamaSelectionFatalError; + } + throw new Error("expected a fatal Ollama selection outcome"); +} + describe("handleOllamaProbeFailure (#4365)", () => { let originalProvider: string | undefined; let originalNonInteractive: string | undefined; @@ -29,22 +40,23 @@ describe("handleOllamaProbeFailure (#4365)", () => { else process.env.NEMOCLAW_NON_INTERACTIVE = originalNonInteractive; } - it("exits when a pinned Ollama provider hits a daemon failure", () => { + it("defers pinned-provider termination when Ollama hits a daemon failure", () => { process.env.NEMOCLAW_PROVIDER = "ollama"; const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`process.exit:${code ?? 0}`); - }) as never); try { - expect(() => + const fatal = captureFatalSelection(() => handleOllamaProbeFailure( { ok: false, message: "runner crashed", daemonFailure: true }, "nemotron-3-nano:30b", () => false, ), - ).toThrow(/process\.exit:1/); + ); + expect(fatal).toMatchObject({ + termination: "process", + message: "Ollama daemon is unhealthy for model 'nemotron-3-nano:30b'.", + }); const errLines = errSpy.mock.calls.map((c) => String(c[0])); expect( errLines.some((l) => @@ -56,33 +68,31 @@ describe("handleOllamaProbeFailure (#4365)", () => { } finally { errSpy.mockRestore(); logSpy.mockRestore(); - exitSpy.mockRestore(); restore(); } }); - it("aborts non-interactive runs on a daemon failure", () => { + it("defers non-interactive aborts on a daemon failure", () => { delete process.env.NEMOCLAW_PROVIDER; const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`process.exit:${code ?? 0}`); - }) as never); try { - expect(() => + const fatal = captureFatalSelection(() => handleOllamaProbeFailure( { ok: false, message: "runner died", daemonFailure: true }, "nemotron-3-nano:30b", () => true, ), - ).toThrow(/process\.exit:1/); - const errLines = errSpy.mock.calls.map((c) => String(c[0])); - expect(errLines.some((l) => l.includes("Aborting: Ollama daemon is unhealthy"))).toBe(true); + ); + expect(fatal).toMatchObject({ + termination: "non-interactive", + message: "Ollama daemon is unhealthy for model 'nemotron-3-nano:30b'.", + hint: expect.stringContaining("Pick a non-Ollama provider"), + }); } finally { errSpy.mockRestore(); logSpy.mockRestore(); - exitSpy.mockRestore(); restore(); } }); @@ -136,30 +146,26 @@ describe("handleOllamaProbeFailure (#4365)", () => { } }); - it("aborts non-interactive model-level failures via the legacy message", () => { + it("defers non-interactive model-level termination with the legacy message", () => { delete process.env.NEMOCLAW_PROVIDER; const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`process.exit:${code ?? 0}`); - }) as never); try { - expect(() => + const fatal = captureFatalSelection(() => handleOllamaProbeFailure( { ok: false, message: "model requires more system memory" }, "qwen3.5:9b", () => true, ), - ).toThrow(/process\.exit:1/); - const errLines = errSpy.mock.calls.map((c) => String(c[0])); - expect( - errLines.some((l) => l.includes("Aborting: Ollama model 'qwen3.5:9b' unavailable")), - ).toBe(true); + ); + expect(fatal).toMatchObject({ + termination: "non-interactive", + message: "Ollama model 'qwen3.5:9b' unavailable.", + }); } finally { errSpy.mockRestore(); logSpy.mockRestore(); - exitSpy.mockRestore(); restore(); } }); @@ -198,51 +204,49 @@ describe("completeOllamaRuntimeContextSelection (#6760)", () => { } }); - it("aborts non-interactive runs after a runtime context failure", () => { + it("defers non-interactive termination after a runtime context failure", () => { const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`process.exit:${code ?? 0}`); - }) as never); try { - expect(() => + const fatal = captureFatalSelection(() => completeOllamaRuntimeContextSelection( { ok: false, message: "restart Ollama with OLLAMA_CONTEXT_LENGTH=64000" }, selected, () => true, ), - ).toThrow(/process\.exit:1/); - expect(errSpy).toHaveBeenCalledWith( - " [non-interactive] Aborting: restart Ollama with OLLAMA_CONTEXT_LENGTH=64000", ); + expect(fatal).toMatchObject({ + termination: "non-interactive", + message: "restart Ollama with OLLAMA_CONTEXT_LENGTH=64000", + }); + expect(errSpy).not.toHaveBeenCalled(); } finally { errSpy.mockRestore(); - exitSpy.mockRestore(); } }); - it("exits pinned interactive runs after a runtime context failure", () => { + it("defers pinned interactive termination after a runtime context failure", () => { vi.stubEnv("NEMOCLAW_PROVIDER", "ollama"); const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`process.exit:${code ?? 0}`); - }) as never); try { - expect(() => + const fatal = captureFatalSelection(() => completeOllamaRuntimeContextSelection( { ok: false, message: "restart Ollama with OLLAMA_CONTEXT_LENGTH=64000" }, selected, () => false, ), - ).toThrow(/process\.exit:1/); + ); + expect(fatal).toMatchObject({ + termination: "process", + message: "restart Ollama with OLLAMA_CONTEXT_LENGTH=64000", + }); expect(errSpy).toHaveBeenCalledWith(" restart Ollama with OLLAMA_CONTEXT_LENGTH=64000"); expect(logSpy).not.toHaveBeenCalledWith(" Returning to provider selection."); } finally { errSpy.mockRestore(); logSpy.mockRestore(); - exitSpy.mockRestore(); vi.unstubAllEnvs(); } }); diff --git a/src/lib/onboard/ollama-probe-failure.ts b/src/lib/onboard/ollama-probe-failure.ts index 1c7b9d5d3a5..377d1a20b18 100644 --- a/src/lib/onboard/ollama-probe-failure.ts +++ b/src/lib/onboard/ollama-probe-failure.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import type { ApplyOllamaRuntimeContextWindowResult } from "../inference/ollama-runtime-context"; -import { abortNonInteractive } from "./non-interactive-abort"; import { isOllamaProviderPinned } from "./ollama-startup"; export interface OllamaProbeFailureInput { @@ -21,6 +20,28 @@ type SelectedOllamaModel = { export type OllamaModelSelectionOutcome = SelectedOllamaModel | { outcome: "back-to-selection" }; +/** A fatal selection outcome whose process termination must happen after transactional rollback. */ +export class OllamaSelectionFatalError extends Error { + constructor( + readonly termination: "process" | "non-interactive", + message: string, + readonly hint?: string, + ) { + super(message); + this.name = "OllamaSelectionFatalError"; + } +} + +/** Defer a fatal process exit to the owner of any active Ollama mutation. */ +export function deferOllamaProcessExit(): never { + throw new OllamaSelectionFatalError("process", "Fatal Ollama model selection failure."); +} + +/** Defer a non-interactive abort to the owner of any active Ollama mutation. */ +export function deferAbort(message: string, hint?: string): never { + throw new OllamaSelectionFatalError("non-interactive", message, hint); +} + /** Finish Ollama selection, returning to provider selection on an interactive context failure. */ export function completeOllamaRuntimeContextSelection( result: ApplyOllamaRuntimeContextWindowResult, @@ -28,9 +49,13 @@ export function completeOllamaRuntimeContextSelection( isNonInteractive: () => boolean, ): OllamaModelSelectionOutcome { if (result.ok) return selected; - if (isNonInteractive()) abortNonInteractive(result.message); + if (isNonInteractive()) { + throw new OllamaSelectionFatalError("non-interactive", result.message); + } console.error(` ${result.message}`); - if (isOllamaProviderPinned()) process.exit(1); + if (isOllamaProviderPinned()) { + throw new OllamaSelectionFatalError("process", result.message); + } console.log(" Returning to provider selection."); console.log(""); return { outcome: "back-to-selection" }; @@ -60,10 +85,14 @@ export function handleOllamaProbeFailure( console.error( " NEMOCLAW_PROVIDER pins onboarding to Ollama but the Ollama model runner is unhealthy; refusing to loop on Ollama model selection.", ); - process.exit(1); + throw new OllamaSelectionFatalError( + "process", + `Ollama daemon is unhealthy for model '${selectedModel}'.`, + ); } if (isNonInteractive()) { - abortNonInteractive( + throw new OllamaSelectionFatalError( + "non-interactive", `Ollama daemon is unhealthy for model '${selectedModel}'.`, "Pick a non-Ollama provider, restart Ollama, or rerun with NEMOCLAW_PROVIDER set explicitly.", ); @@ -77,7 +106,12 @@ export function handleOllamaProbeFailure( console.log(""); return "back-to-selection"; } - if (isNonInteractive()) abortNonInteractive(`Ollama model '${selectedModel}' unavailable.`); + if (isNonInteractive()) { + throw new OllamaSelectionFatalError( + "non-interactive", + `Ollama model '${selectedModel}' unavailable.`, + ); + } console.log(" Choose a different Ollama model or select Other."); console.log(""); return "continue"; diff --git a/src/lib/onboard/setup-nim-ollama.test.ts b/src/lib/onboard/setup-nim-ollama.test.ts index d52762310c2..0b3e256a561 100644 --- a/src/lib/onboard/setup-nim-ollama.test.ts +++ b/src/lib/onboard/setup-nim-ollama.test.ts @@ -6,6 +6,7 @@ import assert from "node:assert/strict"; import { afterEach, describe, expect, it, vi } from "vitest"; import { MIN_HERMES_OLLAMA_CONTEXT_WINDOW } from "../inference/ollama-runtime-context"; +import { OllamaSelectionFatalError } from "./ollama-probe-failure"; import { createSetupNimOllamaHandlers } from "./setup-nim-ollama"; import type { SetupNimSelectionState } from "./setup-nim-selection"; @@ -511,6 +512,45 @@ describe("createSetupNimOllamaHandlers", () => { expect(rollback).toHaveBeenCalledOnce(); }); + it("awaits Windows rollback before terminating a fatal model selection", async () => { + const events: string[] = []; + const rollback = vi.fn(async () => { + events.push("rollback:start"); + await Promise.resolve(); + events.push("rollback:done"); + }); + const exit = vi.fn((code?: number): never => { + events.push(`exit:${String(code)}`); + throw new Error(`process.exit:${String(code)}`); + }); + const restart = vi.fn(() => ({ ok: true as const, commit: vi.fn(), rollback })); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + process: { ...process, exit } as unknown as NodeJS.Process, + selectAndValidateOllamaModel: async () => { + throw new OllamaSelectionFatalError("process", "fatal model selection"); + }, + setupWindowsOllamaWith0000Binding: restart, + }), + ); + + await expect( + handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "qwen3:8b", + false, + true, + "C:/Ollama/ollama.exe", + makeState(), + ), + ).rejects.toThrow("process.exit:1"); + + expect(events).toEqual(["rollback:start", "rollback:done", "exit:1"]); + expect(rollback).toHaveBeenCalledOnce(); + expect(exit).toHaveBeenCalledWith(1); + }); + it("preserves accepted tools-incompatible state for running Ollama", async () => { const state = makeState(); const { handleRunningOllamaSelection } = createSetupNimOllamaHandlers(makeDeps()); diff --git a/src/lib/onboard/setup-nim-ollama.ts b/src/lib/onboard/setup-nim-ollama.ts index 9e8a166bc03..d6ea7580a32 100644 --- a/src/lib/onboard/setup-nim-ollama.ts +++ b/src/lib/onboard/setup-nim-ollama.ts @@ -7,6 +7,7 @@ import type { WindowsOllamaInstallResult, WindowsOllamaSetupResult, } from "../inference/ollama/windows"; +import { OllamaSelectionFatalError } from "./ollama-probe-failure"; const { getRequestedModelFromEnv, @@ -74,7 +75,7 @@ type SetupNimOllamaDeps = { restartOnly?: boolean; contextWindowFloor?: number; }) => { ok: boolean }; - abortNonInteractive: (message: string) => never; + abortNonInteractive: (message: string, hint?: string) => never; assertOllamaUpgradeApplied: (menu: { hasUpgradableOllama: boolean; }) => { ok: true } | { ok: false; message: string }; @@ -150,6 +151,13 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { } } + function terminateFatalSelection(error: OllamaSelectionFatalError): never { + if (error.termination === "non-interactive") { + deps.abortNonInteractive(error.message, error.hint); + } + deps.process.exit(1); + } + function configureOllamaState(state: SetupNimSelectionState): void { state.provider = "ollama-local"; state.credentialEnv = null; @@ -222,7 +230,7 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { : !(await deps.prompt(promptMsg)).trim().toLowerCase().startsWith("n"); if (!proceed) return "retry-selection"; - let mutationSession: { commit: () => void; rollback: () => void } | null = null; + let mutationSession: { commit: () => void; rollback: () => void | Promise } | null = null; try { if (isSwitch) { state.revalidateSandboxIdentity?.("switch to the Windows Ollama runtime"); @@ -269,14 +277,15 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { const result = await selectModel(gpu, state, requestedModel, null, lockedModel); if (result === "retry-selection") { - mutationSession?.rollback(); + await mutationSession?.rollback(); deps.resetOllamaHostCache(); } else { mutationSession?.commit(); } return result; } catch (error) { - mutationSession?.rollback(); + await mutationSession?.rollback(); + if (error instanceof OllamaSelectionFatalError) terminateFatalSelection(error); throw error; } } @@ -335,7 +344,12 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { return "selected"; case "ready": announceOllamaRoute(); - return selectModel(gpu, state, requestedModel, recoveredModel, lockedModel); + try { + return await selectModel(gpu, state, requestedModel, recoveredModel, lockedModel); + } catch (error) { + if (error instanceof OllamaSelectionFatalError) terminateFatalSelection(error); + throw error; + } default: { const kind = (startup as { kind?: unknown }).kind; Object.assign(state, initialState); @@ -383,7 +397,12 @@ export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { return "retry-selection"; } announceOllamaRoute(); - return selectModel(gpu, state, requestedModel, recoveredModel, lockedModel); + try { + return await selectModel(gpu, state, requestedModel, recoveredModel, lockedModel); + } catch (error) { + if (error instanceof OllamaSelectionFatalError) terminateFatalSelection(error); + throw error; + } } return { From a6e5be491fcc1c7b4c413899ffce4a14f0fe1ef3 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 3 Sep 2026 01:33:56 -0700 Subject: [PATCH 38/48] test(ollama): cover guarded provider host probe --- src/lib/onboard/provider-host-state.test.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 302b93cf7b0..f70740ddddd 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -244,6 +244,26 @@ describe("detectInferenceProviderHostState", () => { "docker", "run", "--rm", + "--env", + "HTTP_PROXY=", + "--env", + "http_proxy=", + "--env", + "HTTPS_PROXY=", + "--env", + "https_proxy=", + "--env", + "ALL_PROXY=", + "--env", + "all_proxy=", + "--env", + "FTP_PROXY=", + "--env", + "ftp_proxy=", + "--env", + "NO_PROXY=host.docker.internal", + "--env", + "no_proxy=host.docker.internal", "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", From 4b8304306a8148838d47ba21954e184c713f52b6 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 3 Sep 2026 02:01:49 -0700 Subject: [PATCH 39/48] fix(ollama): terminate pre-sentinel installer --- src/lib/inference/ollama/windows.test.ts | 71 ++++++++++++++++++++++++ src/lib/inference/ollama/windows.ts | 12 ++-- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 9a204e4a9b1..6df47697ac8 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { EventEmitter } from "node:events"; import { createRequire } from "node:module"; import { describe, expect, it, vi } from "vitest"; @@ -158,14 +159,18 @@ function createWindowsSetupBoundary(options: { function loadWindowsOllamaWithMocks( run: ReturnType, runCapture: ReturnType, + spawnProcess?: ReturnType, ) { + const childProcess = require("node:child_process"); const runner = require(RUNNER_PATH); + const originalSpawn = childProcess.spawn; const originalRun = runner.run; const originalRunCapture = runner.runCapture; // Stub the blocking wait so this test does not spend time on retry delays. const atomicsWaitStub = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); delete require.cache[WINDOWS_DIST_PATH]; + if (spawnProcess) childProcess.spawn = spawnProcess; runner.run = run; runner.runCapture = runCapture; @@ -173,6 +178,7 @@ function loadWindowsOllamaWithMocks( windows: require(WINDOWS_DIST_PATH), restore() { delete require.cache[WINDOWS_DIST_PATH]; + childProcess.spawn = originalSpawn; runner.run = originalRun; runner.runCapture = originalRunCapture; atomicsWaitStub.mockRestore(); @@ -180,6 +186,14 @@ function loadWindowsOllamaWithMocks( }; } +function createMockChildProcess() { + return Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + kill: vi.fn(() => true), + }); +} + function captureDefaultWindowsRollbackInvocation(userHost: string | null) { const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; @@ -259,6 +273,63 @@ function createWindowsInstallBoundary(options: { } describe("Windows Ollama helper", () => { + it("terminates the PowerShell wrapper when cancellation precedes the PID sentinel", async () => { + const child = createMockChildProcess(); + const spawnProcess = vi.fn(() => child); + const { windows, restore } = loadWindowsOllamaWithMocks( + vi.fn(), + vi.fn(), + spawnProcess, + ); + + try { + const installer = windows.startWindowsOllamaInstaller(); + const cancellation = installer.cancelAndWait(); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + + child.emit("close", null); + await expect(cancellation).resolves.toBeUndefined(); + } finally { + restore(); + } + }); + + it("terminates the reported Windows installer process tree", async () => { + const installerChild = createMockChildProcess(); + const taskkillChild = createMockChildProcess(); + const spawnProcess = vi.fn((command: string) => + command === "taskkill.exe" ? taskkillChild : installerChild, + ); + const { windows, restore } = loadWindowsOllamaWithMocks( + vi.fn(), + vi.fn(), + spawnProcess, + ); + + try { + const installer = windows.startWindowsOllamaInstaller(); + installerChild.stdout.emit( + "data", + Buffer.from("__NEMOCLAW_WINDOWS_INSTALLER_PID__:4321\n"), + ); + const cancellation = installer.cancelAndWait(); + expect(installerChild.kill).not.toHaveBeenCalled(); + await vi.waitFor(() => + expect(spawnProcess).toHaveBeenLastCalledWith( + "taskkill.exe", + ["/PID", "4321", "/T", "/F"], + { stdio: "ignore" }, + ), + ); + + taskkillChild.emit("close", 0); + installerChild.emit("close", null); + await expect(cancellation).resolves.toBeUndefined(); + } finally { + restore(); + } + }); + it("describes Docker-client isolation in Windows Ollama timeout diagnostics", () => { const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index fb2cce10a58..81dd29878c6 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -119,16 +119,17 @@ function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { completion, cancelAndWait: () => { cancellation ??= (async () => { - const pid = await windowsPidReady; - if (pid !== null) { - await terminateWindowsProcessTree(pid); - } else { + if (windowsPid === null) { try { child.kill("SIGTERM"); } catch { - // The wrapper has already exited, so completion below is authoritative. + // The wrapper has already exited, so its close event remains authoritative. } } + const pid = await windowsPidReady; + if (pid !== null) { + await terminateWindowsProcessTree(pid); + } await completion; })(); return cancellation; @@ -653,6 +654,7 @@ function printWindowsOllamaSnapshotDiagnostics(): void { module.exports = { installOllamaOnWindowsHost, awaitWindowsOllamaReady, + startWindowsOllamaInstaller, setupWindowsOllamaWith0000Binding, sleep: sleepSeconds, switchToWindowsOllamaHost, From 561e44fd4e3689f7741d9e7136adb05fb3268605 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 02:02:24 -0700 Subject: [PATCH 40/48] test(inference): align protected Windows route fixtures Signed-off-by: Prekshi Vyas --- .../sandbox/doctor-system-checks.test.ts | 39 +++++++++++++++---- .../local-windows-ollama-transport.test.ts | 2 +- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts index 14bc626bae9..e6fb00c5cad 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.test.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -4,12 +4,18 @@ import { createRequire } from "node:module"; import { afterEach, describe, expect, it, vi } from "vitest"; +vi.mock("../../adapters/docker/runtime", async (importOriginal) => ({ + ...(await importOriginal()), + detectContainerRuntimeFromDockerInfo: () => "docker-desktop", +})); + const requireDist = createRequire(import.meta.url); const modulePath = "./doctor-system-checks.js"; describe("doctor system checks", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); delete requireDist.cache[requireDist.resolve(modulePath)]; }); @@ -55,6 +61,10 @@ describe("doctor system checks", () => { }); it("probes a persisted Windows Ollama route through credential-free Docker", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.stubEnv("DOCKER_CONTEXT", "default"); + vi.stubEnv("DOCKER_HOST", ""); + vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); const cleanup = vi.fn(() => ({ ok: true as const })); const prepareDockerEnvironment = vi.fn(() => ({ env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, @@ -63,8 +73,17 @@ describe("doctor system checks", () => { })); const runCaptureImpl = vi.fn( (command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => - command[0] === "docker" && options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" - ? JSON.stringify({ models: [] }) + command.join(" ").includes("Get-NetTCPConnection") + ? "127.0.0.1" + : command[0] === "docker" && + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? command.includes("Host: rebinding.invalid") + ? "403" + : command.some( + (argument) => argument === "http://host.docker.internal:11434/api/tags", + ) + ? JSON.stringify({ models: [] }) + : "" : "", ); const { ollamaDoctorCheck } = requireDist(modulePath); @@ -79,12 +98,18 @@ describe("doctor system checks", () => { status: "ok", detail: "reachable at http://host.docker.internal:11434/api/tags (0 model(s))", }); - expect(runCaptureImpl.mock.calls[0]?.[0]?.[0]).toBe("docker"); - expect(runCaptureImpl.mock.calls[0]?.[0]).toContain( - "http://host.docker.internal:11434/api/tags", + expect(runCaptureImpl.mock.calls).toHaveLength(4); + expect(runCaptureImpl.mock.calls.slice(1).every(([command]) => command[0] === "docker")).toBe( + true, + ); + expect(runCaptureImpl.mock.calls[1]?.[0]).toEqual( + expect.arrayContaining(["http://host.docker.internal:11434/api/tags"]), + ); + expect(runCaptureImpl.mock.calls[2]?.[0]).toEqual( + expect.arrayContaining(["Host: rebinding.invalid"]), ); - expect(prepareDockerEnvironment).toHaveBeenCalledOnce(); - expect(cleanup).toHaveBeenCalledOnce(); + expect(prepareDockerEnvironment).toHaveBeenCalledTimes(3); + expect(cleanup).toHaveBeenCalledTimes(3); }); it.each([ diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index 2f571fba63f..b32eefa4f66 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -168,7 +168,7 @@ describe("Windows-host Ollama transport", () => { ? "127.0.0.1" : command.includes("Host: rebinding.invalid") ? "403" - : command.includes(WINDOWS_OLLAMA_TAGS_URL) + : command.some((argument) => argument === WINDOWS_OLLAMA_TAGS_URL) ? JSON.stringify({ models: [] }) : ""; }, From 680d3a746bccf2a65fd2471354fef450d48cee98 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 02:19:13 -0700 Subject: [PATCH 41/48] test(inference): run doctor fixture across test modes Signed-off-by: Prekshi Vyas --- .../sandbox/doctor-system-checks.test.ts | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts index e6fb00c5cad..d28b74acb8e 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.test.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -1,18 +1,44 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; -import { afterEach, describe, expect, it, vi } from "vitest"; - -vi.mock("../../adapters/docker/runtime", async (importOriginal) => ({ - ...(await importOriginal()), - detectContainerRuntimeFromDockerInfo: () => "docker-desktop", -})); +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; const requireDist = createRequire(import.meta.url); const modulePath = "./doctor-system-checks.js"; describe("doctor system checks", () => { + const originalPath = process.env.PATH; + let fakeDockerDir: string; + + beforeAll(() => { + fakeDockerDir = mkdtempSync(join(tmpdir(), "nemoclaw-doctor-fake-docker-desktop-")); + const fakeDockerPath = join(fakeDockerDir, "docker"); + writeFileSync( + fakeDockerPath, + [ + "#!/bin/sh", + 'if [ "$1" = "info" ]; then', + " printf '%s\\n' 'Operating System: Docker Desktop'", + " exit 0", + "fi", + "exit 1", + "", + ].join("\n"), + ); + chmodSync(fakeDockerPath, 0o755); + process.env.PATH = `${fakeDockerDir}${delimiter}${originalPath ?? ""}`; + }); + + afterAll(() => { + Reflect.deleteProperty(process.env, "PATH"); + Object.assign(process.env, originalPath === undefined ? {} : { PATH: originalPath }); + rmSync(fakeDockerDir, { recursive: true, force: true }); + }); + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); From c948b80212c2138fadb530cbea9f41f907060546 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 07:26:44 -0700 Subject: [PATCH 42/48] fix(inference): bound Windows Ollama cancellation Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 121 +++++++++++++++++++++-- src/lib/inference/ollama/windows.ts | 68 ++++++++++--- 2 files changed, 167 insertions(+), 22 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 71c55c2ad2e..36d2ccfb3ab 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -69,6 +69,7 @@ function createWindowsSetupBoundary(options: { userHost: string | null; watcherPath: string | null; daemonPath: string | null; + persistBindingResult?: boolean; launchStatuses?: number[]; readinessResults?: boolean[]; rollbackStatus?: number; @@ -90,7 +91,7 @@ function createWindowsSetupBoundary(options: { const rollbackStatus = options.rollbackStatus ?? 0; let launchIndex = 0; let readinessIndex = 0; - let interruptHandler = (_signal: "SIGINT" | "SIGTERM") => {}; + let interruptHandler = (_signal: "SIGINT" | "SIGTERM"): void | Promise => {}; const operations = { captureSnapshot: vi.fn(() => { state.events.push("snapshot"); @@ -99,7 +100,7 @@ function createWindowsSetupBoundary(options: { persistBinding: vi.fn(() => { state.events.push("persist"); state.userHost = "127.0.0.1:11434"; - return true; + return options.persistBindingResult ?? true; }), stopProcesses: vi.fn(() => { state.events.push("stop-existing"); @@ -144,10 +145,12 @@ function createWindowsSetupBoundary(options: { : false; return restored; }), - registerInterruptHandler: vi.fn((handler: (signal: "SIGINT" | "SIGTERM") => void) => { - interruptHandler = handler; - return vi.fn(); - }), + registerInterruptHandler: vi.fn( + (handler: (signal: "SIGINT" | "SIGTERM") => void | Promise) => { + interruptHandler = handler; + return vi.fn(); + }, + ), preserveInterrupt: vi.fn((signal: "SIGINT" | "SIGTERM") => { state.events.push(`signal:${signal}`); throw new PreservedWindowsInterrupt(signal); @@ -257,6 +260,7 @@ function createWindowsInstallBoundary(options: { userHost: string | null; watcherPath: string | null; daemonPath: string | null; + persistBindingResult?: boolean; installedPath?: string; initialReady?: boolean; launchStatuses?: number[]; @@ -582,6 +586,33 @@ describe("Windows Ollama helper", () => { expect(boundary.state.events.at(-1)).toBe("restore"); }); + it("restores the prior state when existing-install persistence is uncertain (#10855)", () => { + const priorHost = "127.0.0.1:11434"; + const watcherPath = "C:\\Users\\tester\\Ollama\\ollama app.exe"; + const boundary = createWindowsSetupBoundary({ + userHost: priorHost, + watcherPath, + daemonPath: null, + persistBindingResult: false, + }); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + expect(windows.setupWindowsOllamaLoopbackBinding({}, boundary.operations)).toEqual({ + ok: false, + reason: "binding", + }); + } finally { + restore(); + errorSpy.mockRestore(); + } + + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.watcherRunning).toBe(true); + expect(boundary.state.events).toEqual(["snapshot", "persist", "stop-replacement", "restore"]); + }); + it("redacts the prior binding from a failed Windows rollback diagnostic", () => { const priorHost = "https://operator:private-token@ollama.example:11434"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; @@ -688,6 +719,36 @@ describe("Windows Ollama helper", () => { ]); }); + it("restores the prior state when installer persistence is uncertain (#10855)", async () => { + const priorHost = "127.0.0.1:11434"; + const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; + const boundary = createWindowsInstallBoundary({ + userHost: priorHost, + watcherPath: null, + daemonPath, + persistBindingResult: false, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + await expect(windows.installOllamaOnWindowsHost({}, boundary.operations)).resolves.toEqual({ + ok: false, + path: "", + reason: "binding", + }); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + + expect(boundary.state.userHost).toBe(priorHost); + expect(boundary.state.daemonRunning).toBe(true); + expect(boundary.state.events).toEqual(["snapshot", "persist", "stop-replacement", "restore"]); + }); + it("restores the prior Windows state when installed Ollama remains unreachable", async () => { const priorHost = "127.0.0.1:11434"; const daemonPath = "C:\\Users\\tester\\Ollama\\ollama.exe"; @@ -797,6 +858,54 @@ describe("Windows Ollama helper", () => { expect(boundary.state.events).toEqual(["snapshot", "persist", "install", "cancel-installer"]); }); + it("stops waiting without rollback when cancellation has no PID or child close (#10855)", async () => { + vi.useFakeTimers(); + const child = createMockChildProcess(); + const spawnProcess = vi.fn(() => child); + const boundary = createWindowsInstallBoundary({ + userHost: "127.0.0.1:11434", + watcherPath: "C:\\Users\\tester\\Ollama\\ollama app.exe", + daemonPath: null, + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn(), spawnProcess); + boundary.operations.startInstaller = vi.fn(() => { + boundary.state.events.push("install"); + const installer = windows.startWindowsOllamaInstaller(); + queueMicrotask(() => { + void Promise.resolve(boundary.triggerInterrupt("SIGTERM")).catch(() => {}); + }); + return installer; + }); + + try { + const installation = windows.installOllamaOnWindowsHost({}, boundary.operations); + const rejection = expect(installation).rejects.toThrow( + "Timed out while confirming Windows Ollama installer cancellation", + ); + await Promise.resolve(); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + + await vi.advanceTimersByTimeAsync(5_000); + await rejection; + expect(spawnProcess).toHaveBeenCalledTimes(1); + expect(boundary.operations.rollbackSnapshot).not.toHaveBeenCalled(); + expect(boundary.operations.preserveInterrupt).not.toHaveBeenCalled(); + const diagnostic = errorSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(diagnostic).toContain("Could not confirm that the Windows Ollama installer stopped"); + expect(diagnostic).toContain("did not restore the previous Windows Ollama state"); + expect(diagnostic).toContain("stop the installer and Ollama processes"); + expect(diagnostic).toContain("restore the previous User-scope OLLAMA_HOST value"); + expect(diagnostic).toContain("nemoclaw onboard"); + } finally { + restore(); + logSpy.mockRestore(); + errorSpy.mockRestore(); + vi.useRealTimers(); + } + }); + it("reports manual recovery when install rollback fails", async () => { const boundary = createWindowsInstallBoundary({ userHost: "127.0.0.1:11434", diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 38b900bebbb..8740e503b93 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -37,6 +37,31 @@ type WindowsOllamaInstallerProcess = { }; const WINDOWS_INSTALLER_PID_SENTINEL = "__NEMOCLAW_WINDOWS_INSTALLER_PID__:"; +const WINDOWS_INSTALLER_CANCELLATION_TIMEOUT_MS = 5_000; + +function reportUnconfirmedWindowsInstallerCancellation(): void { + console.error(" Could not confirm that the Windows Ollama installer stopped."); + console.error( + " NemoClaw did not restore the previous Windows Ollama state because the installer may still change it.", + ); + console.error( + " In Windows PowerShell, stop the installer and Ollama processes, restore the previous User-scope OLLAMA_HOST value, then retry:", + ); + console.error(" nemoclaw onboard"); +} + +function waitForWindowsInstallerCancellation(operation: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new Error("Timed out while confirming Windows Ollama installer cancellation")); + }, WINDOWS_INSTALLER_CANCELLATION_TIMEOUT_MS); + timer.unref(); + }); + return Promise.race([operation, deadline]).finally(() => { + if (timer) clearTimeout(timer); + }); +} function parseWindowsProcessId(value: string): number | null { if (!/^[1-9][0-9]*$/.test(value)) return null; @@ -89,6 +114,7 @@ function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { let windowsPid: number | null = null; let stdoutBuffer = ""; let resolveWindowsPid: (pid: number | null) => void = () => {}; + let rejectCompletion: (error: unknown) => void = () => {}; const windowsPidReady = new Promise((resolve) => { resolveWindowsPid = resolve; }); @@ -111,7 +137,8 @@ function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { } } }; - const completion = new Promise((resolve) => { + const completion = new Promise((resolve, reject) => { + rejectCompletion = reject; child.stdout?.on("data", (chunk: Buffer) => { stdoutBuffer += chunk.toString("utf8"); flushStdoutLines(false); @@ -132,20 +159,29 @@ function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { return { completion, cancelAndWait: () => { - cancellation ??= (async () => { - if (windowsPid === null) { - try { - child.kill("SIGTERM"); - } catch { - // The wrapper has already exited, so its close event remains authoritative. + cancellation ??= waitForWindowsInstallerCancellation( + (async () => { + if (windowsPid === null) { + try { + child.kill("SIGTERM"); + } catch { + // The wrapper has already exited, so its close event remains authoritative. + } + } + const pid = await windowsPidReady; + if (pid !== null) { + await terminateWindowsProcessTree(pid); } - } - const pid = await windowsPidReady; - if (pid !== null) { - await terminateWindowsProcessTree(pid); - } - await completion; - })(); + await completion; + })(), + ).catch((error: unknown) => { + reportUnconfirmedWindowsInstallerCancellation(); + // Unblock the install path without treating the process as stopped. + // The rejected completion reaches the mutation owner, which leaves the + // snapshot untouched because cancellation was not confirmed. + rejectCompletion(error); + throw error; + }); return cancellation; }, }; @@ -543,11 +579,11 @@ function applyWindowsOllamaBinding( operations: WindowsOllamaSetupOperations, markMutated: () => void, ): { ok: true } | { ok: false; reason: "binding" | "readiness" } { + markMutated(); if (!operations.persistBinding()) { console.error(" Could not persist the Windows Ollama host binding."); return { ok: false, reason: "binding" }; } - markMutated(); if (opts.announceStop) { console.log(" Stopping existing Ollama on Windows host..."); } @@ -577,11 +613,11 @@ async function installOllamaOnWindowsHost( console.log(" Installing Ollama on Windows host..."); console.log(" This can take several minutes. Output may pause silently"); try { + mutation.markMutated(); if (!operations.persistBinding()) { await mutation.rollback(); return { ok: false, path: "", reason: "binding" }; } - mutation.markMutated(); const installer = operations.startInstaller(); mutation.setInterruptCancellation(installer.cancelAndWait); await installer.completion; From 35751ed2750ac58e03870659a482b6a9a4ea6a5e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 07:58:14 -0700 Subject: [PATCH 43/48] fix(agent): force timed-out Ollama recovery exit Signed-off-by: Prekshi Vyas --- .../agent/ollama-restart-recovery.test.ts | 143 ++++++++++++++++++ .../sandbox/agent/ollama-restart-recovery.ts | 24 ++- src/lib/inference/local.ts | 1 + 3 files changed, 167 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 1a600af6708..79f17c6995d 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -3,6 +3,9 @@ import { EventEmitter } from "node:events"; import type { StdioOptions } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; @@ -229,6 +232,81 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ); }); + it("runs Windows recovery through the isolated Docker transport", async () => { + const fakeDockerDirectory = mkdtempSync(join(tmpdir(), "nemoclaw-recovery-docker-")); + const fakeDockerPath = join(fakeDockerDirectory, "docker"); + writeFileSync(fakeDockerPath, "#!/bin/sh\nprintf '%s\\n' 'Operating System: Docker Desktop'\n"); + chmodSync(fakeDockerPath, 0o755); + const originalPath = process.env.PATH; + process.env.PATH = `${fakeDockerDirectory}${delimiter}${originalPath ?? ""}`; + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.stubEnv("DOCKER_CONTEXT", "default"); + vi.stubEnv("DOCKER_HOST", ""); + vi.stubEnv("WSL_DISTRO_NAME", "Ubuntu"); + const cleanup = vi.fn(() => ({ ok: true as const })); + const prepareDockerEnvironment = vi.fn(() => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })); + const routeProtectionCapture = vi.fn((command: readonly string[]) => { + const rendered = command.join(" "); + return rendered.includes("Get-NetTCPConnection") + ? "127.0.0.1" + : command.includes("Host: rebinding.invalid") + ? "403" + : JSON.stringify({ models: [] }); + }); + const spawnRecoveryChild = completingRecoverySpawner([ + unloadedProbeResult().stdout, + successfulWarmResult().stdout, + ]); + + try { + const result = await maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, + }, + { + dockerContextIsDefault: () => true, + getOllamaHost: () => "host.docker.internal", + prepareDockerEnvironment, + revalidateOllamaHost: () => "host.docker.internal", + routeProtectionCapture, + spawnRecoveryChild, + }, + ); + expect(result).toEqual({ kind: "warmed", ok: true }); + } finally { + Reflect.deleteProperty(process.env, "PATH"); + Object.assign(process.env, originalPath === undefined ? {} : { PATH: originalPath }); + platformSpy.mockRestore(); + vi.unstubAllEnvs(); + rmSync(fakeDockerDirectory, { recursive: true, force: true }); + } + + const commands = spawnRecoveryChild.mock.calls.map(([binary, args]) => [binary, ...args]); + expect(commands.map(([binary]) => binary)).toEqual(["docker", "docker"]); + expect(commands).toEqual([ + expect.arrayContaining([ + CONTAINER_REACHABILITY_IMAGE, + `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, + ]), + expect.arrayContaining([ + CONTAINER_REACHABILITY_IMAGE, + `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, + ]), + ]); + expect(spawnRecoveryChild.mock.calls.map(([, , , env]) => env.DOCKER_CONFIG)).toEqual([ + "/tmp/credential-free-docker", + "/tmp/credential-free-docker", + ]); + expect(prepareDockerEnvironment).toHaveBeenCalled(); + expect(cleanup).toHaveBeenCalled(); + }); + it("stops recovery after an async status probe is cancelled", async () => { const runRecoveryCaptureImpl = vi.fn().mockResolvedValue({ stdout: "", @@ -358,6 +436,71 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(signalEvents.listenerCount("SIGINT")).toBe(0); }); + it("forces a timed-out recovery child to close and releases its Docker environment", async () => { + vi.useFakeTimers(); + const childEvents = new EventEmitter(); + const signalEvents = new EventEmitter(); + const cleanup = vi.fn(() => ({ ok: true as const })); + let child: AgentDispatchChild; + const kill = vi + .fn<(signal: NodeJS.Signals) => boolean>() + .mockReturnValueOnce(true) + .mockImplementationOnce((signal) => { + child.signalCode = signal; + queueMicrotask(() => childEvents.emit("close", null, signal)); + return true; + }); + child = { + exitCode: null, + signalCode: null, + kill, + once: ((event: string, listener: (...args: unknown[]) => void) => + childEvents.once(event, listener)) as AgentDispatchChild["once"], + stderr: new EventEmitter(), + stdout: new EventEmitter(), + }; + const signalSource: SandboxExecSignalSource = { + add: (signal, listener) => signalEvents.on(signal, listener), + remove: (signal, listener) => signalEvents.off(signal, listener), + }; + + try { + const pending = runOllamaRecoveryCapture( + ["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE, "true"], + { + host: "127.0.0.1", + timeoutMilliseconds: 10, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + signalSource, + spawnRecoveryChild: vi.fn(() => child), + }, + ); + + await vi.advanceTimersByTimeAsync(10); + expect(child.kill).toHaveBeenCalledTimes(1); + expect(child.kill).toHaveBeenLastCalledWith("SIGTERM"); + expect(cleanup).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1_000); + await expect(pending).resolves.toMatchObject({ + exitCode: null, + signal: "SIGKILL", + timedOut: true, + }); + expect(child.kill).toHaveBeenCalledTimes(2); + expect(child.kill).toHaveBeenLastCalledWith("SIGKILL"); + expect(cleanup).toHaveBeenCalledOnce(); + expect(signalEvents.listenerCount("SIGTERM")).toBe(0); + expect(signalEvents.listenerCount("SIGINT")).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + it("maps an auth-proxy route back to host loopback", async () => { const runRecoveryCaptureImpl = scriptedRecoveryCapture( unloadedProbeResult(), diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 0d62788d0f3..9fac386a4e8 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -53,10 +53,13 @@ export interface OllamaRestartRecoveryOptions { type PrepareOllamaDockerEnvironment = NonNullable< Parameters[2] >["prepareDockerEnvironment"]; +type OllamaExecutionOptions = NonNullable[2]>; export interface OllamaRestartRecoveryDeps extends OllamaRestartRecoveryOptions { getOllamaHost?: () => string; + dockerContextIsDefault?: OllamaExecutionOptions["dockerContextIsDefault"]; prepareDockerEnvironment?: PrepareOllamaDockerEnvironment; + routeProtectionCapture?: OllamaExecutionOptions["runCaptureImpl"]; runRecoveryCaptureImpl?: OllamaRecoveryCaptureFn; signalSource?: SandboxExecSignalSource; spawnRecoveryChild?: OllamaRecoverySpawner; @@ -90,6 +93,7 @@ export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; const OLLAMA_RESTART_RECOVERY_PROBE_TIMEOUT_MILLISECONDS = 5_000; const OLLAMA_RESTART_RECOVERY_MAX_BUFFER_BYTES = 1024 * 1024; +const OLLAMA_RESTART_RECOVERY_TERMINATION_GRACE_MILLISECONDS = 1_000; const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ OLLAMA_LOCALHOST, @@ -172,7 +176,9 @@ export type OllamaRecoveryCaptureFn = ( options: { host: string; timeoutMilliseconds: number; + dockerContextIsDefault?: OllamaExecutionOptions["dockerContextIsDefault"]; prepareDockerEnvironment?: PrepareOllamaDockerEnvironment; + routeProtectionCapture?: OllamaExecutionOptions["runCaptureImpl"]; signalSource?: SandboxExecSignalSource; spawnRecoveryChild?: OllamaRecoverySpawner; }, @@ -194,9 +200,11 @@ export async function runOllamaRecoveryCapture( options: Parameters[1], ): Promise { const execution = prepareOllamaApiExecution(command, options.host, { + dockerContextIsDefault: options.dockerContextIsDefault, env: buildSubprocessEnv(), prepareDockerEnvironment: options.prepareDockerEnvironment, operation: "Ollama restart recovery", + runCaptureImpl: options.routeProtectionCapture, }); const [binary, ...args] = execution.command; if (!binary) { @@ -212,12 +220,19 @@ export async function runOllamaRecoveryCapture( let timedOut = false; let timeout: ReturnType | undefined; + let forceTimeout: ReturnType | undefined; const spawnRecoveryChild = options.spawnRecoveryChild ?? defaultOllamaRecoverySpawner; const spawnChild: AgentDispatchSpawner = (runBinary, runArgs, stdio) => { const child = spawnRecoveryChild(runBinary, runArgs, stdio, execution.env ?? {}); timeout = setTimeout(() => { timedOut = true; - if (child.exitCode === null && child.signalCode === null) child.kill("SIGTERM"); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + forceTimeout = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, OLLAMA_RESTART_RECOVERY_TERMINATION_GRACE_MILLISECONDS); + forceTimeout.unref?.(); + } }, options.timeoutMilliseconds); timeout.unref?.(); return child; @@ -240,6 +255,7 @@ export async function runOllamaRecoveryCapture( }; } finally { if (timeout) clearTimeout(timeout); + if (forceTimeout) clearTimeout(forceTimeout); execution.cleanup(); } } @@ -391,7 +407,9 @@ export async function maybeWarmOllamaAfterDaemonRestart( { host: rawHost, timeoutMilliseconds: statusTimeoutMilliseconds, + dockerContextIsDefault: deps.dockerContextIsDefault, prepareDockerEnvironment: deps.prepareDockerEnvironment, + routeProtectionCapture: deps.routeProtectionCapture, signalSource: deps.signalSource, spawnRecoveryChild: deps.spawnRecoveryChild, }, @@ -424,7 +442,9 @@ export async function maybeWarmOllamaAfterDaemonRestart( const result = await (deps.runRecoveryCaptureImpl ?? runOllamaRecoveryCapture)(command, { host: rawHost, timeoutMilliseconds: warmupTimeoutMilliseconds, + dockerContextIsDefault: deps.dockerContextIsDefault, prepareDockerEnvironment: deps.prepareDockerEnvironment, + routeProtectionCapture: deps.routeProtectionCapture, signalSource: deps.signalSource, spawnRecoveryChild: deps.spawnRecoveryChild, }); @@ -484,7 +504,9 @@ export async function maybeWarmOllamaAfterDaemonRestart( { host: rawHost, timeoutMilliseconds: inventoryTimeoutMilliseconds, + dockerContextIsDefault: deps.dockerContextIsDefault, prepareDockerEnvironment: deps.prepareDockerEnvironment, + routeProtectionCapture: deps.routeProtectionCapture, signalSource: deps.signalSource, spawnRecoveryChild: deps.spawnRecoveryChild, }, diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 006712e32c1..2b96b3f6d6f 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -608,6 +608,7 @@ export function prepareOllamaApiExecution( host === OLLAMA_HOST_DOCKER_INTERNAL && windowsHostOllamaRouteProtectionProbeDepth === 0 && !probeWindowsHostOllamaRouteProtection(options.runCaptureImpl ?? runCapture, { + dockerContextIsDefault: options.dockerContextIsDefault, env: sourceEnv, prepareDockerEnvironment: options.prepareDockerEnvironment, }).protected From e93da296d50b234624798c45524f43a50f251b6f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 22:25:56 -0700 Subject: [PATCH 44/48] fix(onboard): bound Windows installer PID prefix Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/windows.test.ts | 24 +++++++++ src/lib/inference/ollama/windows.ts | 67 +++++++++++++++--------- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 36d2ccfb3ab..118050cd7b1 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -365,6 +365,30 @@ describe("Windows Ollama helper", () => { } }); + it("streams an oversized unterminated PID prefix and cancels the wrapper", async () => { + const child = createMockChildProcess(); + const spawnProcess = vi.fn(() => child); + const stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn(), spawnProcess); + const output = Buffer.alloc(256, "x"); + + try { + const installer = windows.startWindowsOllamaInstaller(); + child.stdout.emit("data", output); + expect(stdoutWrite).toHaveBeenCalledWith(output); + child.stdout.emit("data", Buffer.from("\n__NEMOCLAW_WINDOWS_INSTALLER_PID__:4321\n")); + + const cancellation = installer.cancelAndWait(); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + child.emit("close", null); + await expect(cancellation).resolves.toBeUndefined(); + expect(spawnProcess).toHaveBeenCalledTimes(1); + } finally { + restore(); + stdoutWrite.mockRestore(); + } + }); + it("describes Docker-client isolation in Windows Ollama timeout diagnostics", () => { const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 8740e503b93..001e0e08e9c 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -37,6 +37,8 @@ type WindowsOllamaInstallerProcess = { }; const WINDOWS_INSTALLER_PID_SENTINEL = "__NEMOCLAW_WINDOWS_INSTALLER_PID__:"; +// A Windows DWORD PID has at most 10 digits; allow CRLF after it. +const WINDOWS_INSTALLER_PID_LINE_MAX_BYTES = Buffer.byteLength(WINDOWS_INSTALLER_PID_SENTINEL) + 12; const WINDOWS_INSTALLER_CANCELLATION_TIMEOUT_MS = 5_000; function reportUnconfirmedWindowsInstallerCancellation(): void { @@ -112,41 +114,58 @@ function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { { stdio: ["ignore", "pipe", "pipe"] }, ); let windowsPid: number | null = null; - let stdoutBuffer = ""; + let stdoutPrefix = Buffer.alloc(0); + let awaitingPidLine = true; let resolveWindowsPid: (pid: number | null) => void = () => {}; let rejectCompletion: (error: unknown) => void = () => {}; const windowsPidReady = new Promise((resolve) => { resolveWindowsPid = resolve; }); - const flushStdoutLines = (final: boolean) => { - while (true) { - const newlineIndex = stdoutBuffer.indexOf("\n"); - if (newlineIndex < 0 && !final) return; - const line = newlineIndex < 0 ? stdoutBuffer : stdoutBuffer.slice(0, newlineIndex + 1); - stdoutBuffer = newlineIndex < 0 ? "" : stdoutBuffer.slice(newlineIndex + 1); - if (!line) return; - const candidate = line.replace(/\r?\n$/, "").slice(WINDOWS_INSTALLER_PID_SENTINEL.length); - const parsedPid = line.startsWith(WINDOWS_INSTALLER_PID_SENTINEL) - ? parseWindowsProcessId(candidate) - : null; - if (parsedPid !== null && windowsPid === null) { - windowsPid = parsedPid; - resolveWindowsPid(parsedPid); - } else { - process.stdout.write(line); - } + const settlePidLine = (line: Buffer): boolean => { + awaitingPidLine = false; + const text = line.toString("utf8").replace(/\r?\n$/, ""); + const candidate = text.slice(WINDOWS_INSTALLER_PID_SENTINEL.length); + const parsedPid = text.startsWith(WINDOWS_INSTALLER_PID_SENTINEL) + ? parseWindowsProcessId(candidate) + : null; + windowsPid = parsedPid; + resolveWindowsPid(parsedPid); + return parsedPid !== null; + }; + const streamInstallerStdout = (chunk: Buffer) => { + if (!awaitingPidLine) { + process.stdout.write(chunk); + return; } + const remainingBytes = WINDOWS_INSTALLER_PID_LINE_MAX_BYTES - stdoutPrefix.length; + const inspected = chunk.subarray(0, remainingBytes); + const newlineIndex = inspected.indexOf(0x0a); + if (newlineIndex >= 0) { + const line = Buffer.concat([stdoutPrefix, inspected.subarray(0, newlineIndex + 1)]); + if (!settlePidLine(line)) process.stdout.write(line); + process.stdout.write(chunk.subarray(newlineIndex + 1)); + stdoutPrefix = Buffer.alloc(0); + return; + } + if (chunk.length >= remainingBytes) { + awaitingPidLine = false; + resolveWindowsPid(null); + process.stdout.write(stdoutPrefix); + process.stdout.write(chunk); + stdoutPrefix = Buffer.alloc(0); + return; + } + stdoutPrefix = Buffer.concat([stdoutPrefix, chunk]); }; const completion = new Promise((resolve, reject) => { rejectCompletion = reject; - child.stdout?.on("data", (chunk: Buffer) => { - stdoutBuffer += chunk.toString("utf8"); - flushStdoutLines(false); - }); + child.stdout?.on("data", streamInstallerStdout); child.stderr?.on("data", (chunk: Buffer) => process.stderr.write(chunk)); child.on("close", () => { - flushStdoutLines(true); - resolveWindowsPid(windowsPid); + if (awaitingPidLine) { + if (!settlePidLine(stdoutPrefix)) process.stdout.write(stdoutPrefix); + stdoutPrefix = Buffer.alloc(0); + } resolve(); }); child.on("error", (err: NodeJS.ErrnoException) => { From 6ac0a7725a6113bc59bbcedcc3a318f679142373 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sun, 6 Sep 2026 20:57:38 -0700 Subject: [PATCH 45/48] test(onboard): remove duplicate Windows boundary suite --- .../onboard-windows-provider-boundary.test.ts | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 test/onboarding/onboard-windows-provider-boundary.test.ts diff --git a/test/onboarding/onboard-windows-provider-boundary.test.ts b/test/onboarding/onboard-windows-provider-boundary.test.ts deleted file mode 100644 index 9d91a3d976e..00000000000 --- a/test/onboarding/onboard-windows-provider-boundary.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import assert from "node:assert/strict"; -import { describe, it } from "vitest"; -import { testTimeout } from "../helpers/timeouts"; -import { runNativeDockerWindowsProviderBoundary } from "../support/onboard-selection-test-helpers.js"; - -const PROVIDER_SELECTION_TEST_TIMEOUT_MS = testTimeout(60_000); -const FORBIDDEN_WINDOWS_ACTIONS = - /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/; - -describe("native Docker WSL provider boundary", () => { - it.each([ - { provider: "start-windows-ollama", installed: true }, - { provider: "install-windows-ollama", installed: false }, - ] as const)("rejects $provider before launching Ollama", (scenario) => { - const boundary = runNativeDockerWindowsProviderBoundary({ - ...scenario, - reachable: false, - timeoutMs: PROVIDER_SELECTION_TEST_TIMEOUT_MS, - }); - - assert.equal(boundary.status, 1, `${scenario.provider} unexpectedly passed`); - assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); - assert.match(boundary.stderr, new RegExp(scenario.provider + " requires Docker Desktop")); - assert.match(boundary.stderr, /Choose WSL-local Ollama/); - assert.doesNotMatch(boundary.stderr, FORBIDDEN_WINDOWS_ACTIONS); - }); - - it.each(["start-windows-ollama", "install-windows-ollama"] as const)( - "rejects an explicit reachable Windows-host provider path [%s]", - (provider) => { - const boundary = runNativeDockerWindowsProviderBoundary({ - provider, - installed: true, - reachable: true, - timeoutMs: PROVIDER_SELECTION_TEST_TIMEOUT_MS, - }); - const effectiveProvider = - provider === "install-windows-ollama" ? "start-windows-ollama" : provider; - - assert.equal(boundary.status, 1, `${provider} unexpectedly passed`); - assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); - assert.match(boundary.stderr, new RegExp(effectiveProvider + " requires Docker Desktop")); - assert.match(boundary.stderr, /Choose WSL-local Ollama/); - assert.doesNotMatch(boundary.stderr, FORBIDDEN_WINDOWS_ACTIONS); - }, - ); -}); From a89c99a73534c107e88187a6c30ad025c4ba85da Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 5 Sep 2026 23:01:20 -0700 Subject: [PATCH 46/48] docs(inference): document Windows installer recovery Signed-off-by: Prekshi Vyas --- docs/inference/set-up-ollama.mdx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/inference/set-up-ollama.mdx b/docs/inference/set-up-ollama.mdx index c38ade35f60..bd57b981174 100644 --- a/docs/inference/set-up-ollama.mdx +++ b/docs/inference/set-up-ollama.mdx @@ -219,6 +219,20 @@ It pulls missing models through the Ollama HTTP API without requiring an Ollama NemoClaw rejects a reachable Windows daemon when it accepts the untrusted `Host` probe. +### Recover After Unconfirmed Installer Cancellation + +If NemoClaw cannot confirm that the Windows installer stopped, it does not restore the previous Windows Ollama state because the installer might still change it. +Complete these steps before you rerun onboarding: + +1. In Windows PowerShell, stop the installer and every Ollama process. +2. Restore the previous User-scope `OLLAMA_HOST` value. + If the value was previously unset, remove it from User scope. +3. From WSL, rerun onboarding: + +```bash +$$nemoclaw onboard +``` + If the endpoint is not reachable, NemoClaw also checks the Windows `ollama.exe` process through PowerShell interop. When the daemon does not become reachable, onboarding prints PowerShell commands for inspecting the Windows process and port state. Choose the WSL-local Ollama install when Docker Desktop cannot reach the protected Windows loopback route. From 90eb9a74a9cd9c86ed040e6a73d69ec24110eba1 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 8 Sep 2026 08:17:49 -0700 Subject: [PATCH 47/48] fix(inference): address Ollama review findings Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 4 ++- src/lib/inference/ollama/windows.test.ts | 32 ++++++++++++------------ src/lib/inference/ollama/windows.ts | 30 +++++++++++----------- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 443058f6292..c1f2be01366 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1146,7 +1146,9 @@ $$nemoclaw dcode-sandbox agent -n "Summarize this repository" $$nemoclaw dcode-sandbox agent -n "Summarize this repository" --json ``` -For non-JSON OpenClaw turns, the wrapper captures `stdout` and `stderr` and replays them only after the in-sandbox command exits. The combined capture limit is `64 MiB`; exceeding it reports an OpenShell invocation error and exits with status `1`. If the captured output contains an embedded-fallback marker, the wrapper suppresses both streams, prints `recover`, `rebuild --yes`, and `onboard --resume` guidance to `stderr`, and exits with status `1`. Otherwise, it writes the captured output to the corresponding host streams and returns the OpenShell command's exit status. The in-sandbox NemoClaw plugin writes its registration banner to `stderr`, so the banner does not prefix the agent reply on `stdout` in non-JSON mode. Because a delivered turn always writes to one of the two streams, the wrapper reports a dispatch with status `0` and no output as a failure. The wrapper prints recovery guidance to `stderr` and exits with status `1`. Pressing `Ctrl+C` interrupts the OpenShell child, and sending `SIGTERM` to the host wrapper forwards `SIGTERM` to that child. NemoClaw waits for OpenShell to stop the in-sandbox turn, replays captured output, and returns status `130` for `SIGINT` or `143` for `SIGTERM`. When the forwarded argv sets `openclaw agent --timeout `, both captured paths bound the OpenShell command at that value plus 30 seconds. The extra seconds let the in-sandbox turn report its own timeout first, so the host bound catches only a turn that stops answering. +For non-JSON OpenClaw turns, the wrapper captures `stdout` and `stderr` and replays them only after the in-sandbox command exits. The combined capture limit is `64 MiB`; exceeding it reports an OpenShell invocation error and exits with status `1`. If the captured output contains an embedded-fallback marker, the wrapper suppresses both streams, prints `recover`, `rebuild --yes`, and `onboard --resume` guidance to `stderr`, and exits with status `1`. Otherwise, it writes the captured output to the corresponding host streams and returns the OpenShell command's exit status. The in-sandbox NemoClaw plugin writes its registration banner to `stderr`, so the banner does not prefix the agent reply on `stdout` in non-JSON mode. Because a delivered turn always writes to one of the two streams, the wrapper reports a dispatch with status `0` and no output as a failure. The wrapper prints recovery guidance to `stderr` and exits with status `1`. Pressing `Ctrl+C` interrupts the OpenShell child, and sending `SIGTERM` to the host wrapper forwards `SIGTERM` to that child. NemoClaw waits for OpenShell to stop the in-sandbox turn, replays captured output, and returns status `130` for `SIGINT` or `143` for `SIGTERM`. + +When the forwarded argv sets a positive whole-number `openclaw agent --timeout `, NemoClaw starts that deadline before its best-effort Ollama recovery. Recovery can use only the remaining time minus a reserved one second for the turn. Before dispatch, NemoClaw rewrites the forwarded timeout to the remaining time rounded up to a whole second, with a one-second minimum. Both captured paths then bound the OpenShell command at that forwarded remainder plus 30 seconds. The extra seconds let the in-sandbox turn report its own timeout first, so the host bound catches only a turn that stops answering. These leave the OpenShell wait unbounded: diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 118050cd7b1..6691cba5adf 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -308,14 +308,22 @@ describe("Windows Ollama helper", () => { vi.unstubAllEnvs(); }); + it("leaves the persistent installer binding under the mutation transaction", () => { + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn()); + + try { + const installerCommand = windows.buildWindowsOllamaInstallerCommand(); + expect(installerCommand).toContain("$env:OLLAMA_HOST='127.0.0.1:11434'"); + expect(installerCommand).not.toContain("SetEnvironmentVariable('OLLAMA_HOST'"); + } finally { + restore(); + } + }); + it("terminates the PowerShell wrapper when cancellation precedes the PID sentinel", async () => { const child = createMockChildProcess(); const spawnProcess = vi.fn(() => child); - const { windows, restore } = loadWindowsOllamaWithMocks( - vi.fn(), - vi.fn(), - spawnProcess, - ); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn(), spawnProcess); try { const installer = windows.startWindowsOllamaInstaller(); @@ -335,18 +343,11 @@ describe("Windows Ollama helper", () => { const spawnProcess = vi.fn((command: string) => command === "taskkill.exe" ? taskkillChild : installerChild, ); - const { windows, restore } = loadWindowsOllamaWithMocks( - vi.fn(), - vi.fn(), - spawnProcess, - ); + const { windows, restore } = loadWindowsOllamaWithMocks(vi.fn(), vi.fn(), spawnProcess); try { const installer = windows.startWindowsOllamaInstaller(); - installerChild.stdout.emit( - "data", - Buffer.from("__NEMOCLAW_WINDOWS_INSTALLER_PID__:4321\n"), - ); + installerChild.stdout.emit("data", Buffer.from("__NEMOCLAW_WINDOWS_INSTALLER_PID__:4321\n")); const cancellation = installer.cancelAndWait(); expect(installerChild.kill).not.toHaveBeenCalled(); await vi.waitFor(() => @@ -1046,8 +1047,7 @@ describe("Windows Ollama helper", () => { (command: string | string[], options?: { env?: NodeJS.ProcessEnv }) => { return commandText(command).includes("Get-NetTCPConnection") ? "127.0.0.1" - : !Array.isArray(command) || - options?.env?.DOCKER_CONFIG !== "/tmp/credential-free-docker" + : !Array.isArray(command) || options?.env?.DOCKER_CONFIG !== "/tmp/credential-free-docker" ? "" : command.includes(REBINDING_PROBE_HOST_HEADER) ? "403" diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 001e0e08e9c..fd6429ac19b 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -93,26 +93,27 @@ function terminateWindowsProcessTree(pid: number): Promise { }); } -// Pre-set OLLAMA_HOST in both User scope (persists across logins) and the -// current PowerShell session (inherited by the installer's auto-spawned -// ollama_app + daemon) so the new daemon stays on Windows loopback. Ollama -// enables its Host-header validation only for loopback listeners, which is -// required to reject same-host DNS-rebinding requests. +// Pre-set OLLAMA_HOST in the current PowerShell session so the installer's +// auto-spawned ollama_app + daemon inherit the loopback binding. The enclosing +// mutation transaction owns the separate User-scope write and its rollback. +// Ollama enables its Host-header validation only for loopback listeners, which +// is required to reject same-host DNS-rebinding requests. // Don't use stdio:inherit here. When powershell.exe is spawned through // WSL interop, its stdout looks like a pipe (not a console), so PowerShell // holds output in an internal buffer and the user sees long silent gaps. // Reading the pipe from Node and re-writing to our own TTY shows progress // as soon as PowerShell flushes a chunk. -function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { - const child = spawn( - "powershell.exe", - [ - "-Command", - `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; [Console]::Out.WriteLine('${WINDOWS_INSTALLER_PID_SENTINEL}' + $PID); [Console]::Out.Flush(); ` + - `[Environment]::SetEnvironmentVariable('OLLAMA_HOST','${OLLAMA_LOOPBACK_HOST}','User'); $env:OLLAMA_HOST='${OLLAMA_LOOPBACK_HOST}'; irm https://ollama.com/install.ps1 | iex`, - ], - { stdio: ["ignore", "pipe", "pipe"] }, +function buildWindowsOllamaInstallerCommand(): string { + return ( + `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; [Console]::Out.WriteLine('${WINDOWS_INSTALLER_PID_SENTINEL}' + $PID); [Console]::Out.Flush(); ` + + `$env:OLLAMA_HOST='${OLLAMA_LOOPBACK_HOST}'; irm https://ollama.com/install.ps1 | iex` ); +} + +function startWindowsOllamaInstaller(): WindowsOllamaInstallerProcess { + const child = spawn("powershell.exe", ["-Command", buildWindowsOllamaInstallerCommand()], { + stdio: ["ignore", "pipe", "pipe"], + }); let windowsPid: number | null = null; let stdoutPrefix = Buffer.alloc(0); let awaitingPidLine = true; @@ -746,6 +747,7 @@ function printWindowsOllamaSnapshotDiagnostics(): void { module.exports = { installOllamaOnWindowsHost, awaitWindowsOllamaReady, + buildWindowsOllamaInstallerCommand, startWindowsOllamaInstaller, setupWindowsOllamaLoopbackBinding, sleep, From 169620afa7ac45ae0fe58d140f9c0a81c3977195 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 8 Sep 2026 08:46:51 -0700 Subject: [PATCH 48/48] test(inference): compose Windows warmup guards Signed-off-by: Prekshi Vyas --- .../local-windows-ollama-transport.test.ts | 57 ++++++++----------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts index c2d536f0e18..6905f456f81 100644 --- a/src/lib/inference/local-windows-ollama-transport.test.ts +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -666,31 +666,26 @@ describe("Windows-host Ollama transport", () => { expect(capture).toHaveBeenCalledTimes(4); }); - it( - "retries an invalid Windows-host inventory before returning installed models (#10259)", - () => { - setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); - const capture = respondsWithOllamaInventorySequence([ - "", - "proxy response", - JSON.stringify({ models: [{ name: "qwen3.5:9b" }] }), - ]); - const sleeps: number[] = []; - - expect(getOllamaModelOptions(capture, (milliseconds) => sleeps.push(milliseconds))).toEqual([ - "qwen3.5:9b", - ]); - expect( - capture.mock.calls.filter( - ([command, options]) => - command.some((argument) => argument.endsWith("/api/tags")) && - options?.timeout !== 10_000, - ), - ).toHaveLength(3); - expect(sleeps).toEqual([500, 1_000]); - }, - 10_000, - ); + it("retries an invalid Windows-host inventory before returning installed models (#10259)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = respondsWithOllamaInventorySequence([ + "", + "proxy response", + JSON.stringify({ models: [{ name: "qwen3.5:9b" }] }), + ]); + const sleeps: number[] = []; + + expect(getOllamaModelOptions(capture, (milliseconds) => sleeps.push(milliseconds))).toEqual([ + "qwen3.5:9b", + ]); + expect( + capture.mock.calls.filter( + ([command, options]) => + command.some((argument) => argument.endsWith("/api/tags")) && options?.timeout !== 10_000, + ), + ).toHaveLength(3); + expect(sleeps).toEqual([500, 1_000]); + }, 10_000); it("rejects an invalid Windows-host inventory after bounded retries (#10259)", () => { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); @@ -1006,18 +1001,14 @@ describe("Windows-host Ollama transport", () => { const [command, options] = run.mock.calls[0] ?? []; expect({ cleanupCalls: cleanup.mock.calls.length, - commandPrefix: command?.slice(0, 5), + commandPrefix: command?.slice(0, 4), + image: command?.find((argument: string) => argument === CONTAINER_REACHABILITY_IMAGE), endpoint: command?.find((argument: string) => argument.endsWith("/api/generate")), options, }).toEqual({ cleanupCalls: 3, - commandPrefix: [ - "docker", - "run", - "--rm", - "-d", - CONTAINER_REACHABILITY_IMAGE, - ], + commandPrefix: ["docker", "run", "--rm", "-d"], + image: CONTAINER_REACHABILITY_IMAGE, endpoint: "http://host.docker.internal:11434/api/generate", options: { ignoreError: true,