diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 98f575c18de..261c6267bce 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { vi } from "vitest"; -import type { ValidationResult } from "../inference/local"; import type { AgentConfigTarget } from "../sandbox/config"; import type { ConfigObject, ConfigValue } from "../security/credential-filter"; import type { Session } from "../state/onboard-session"; @@ -10,6 +9,8 @@ import type { SandboxEntry } from "../state/registry"; import type { InferenceSetDeps } from "./inference-set"; import type { EnsureHttpsPinRuntimeAdapterFn } from "./inference-set-route-containment"; +type LocalValidationResult = ReturnType; + export const OPENCLAW_TARGET: AgentConfigTarget = { agentName: "openclaw", configPath: "/sandbox/.openclaw/openclaw.json", @@ -165,7 +166,7 @@ export function createDeps(options: { session?: Session | null; openshellStatus?: number; captureOpenshell?: InferenceSetDeps["captureOpenshell"]; - localValidation?: ValidationResult; + localValidation?: LocalValidationResult; localReachable?: boolean; contextWindow?: number | null; shieldsMutable?: boolean; @@ -232,7 +233,9 @@ export function createDeps(options: { }), appendAuditEntry: vi.fn(), log: vi.fn(), - validateLocalProvider: vi.fn((): ValidationResult => options.localValidation ?? { ok: true }), + validateLocalProvider: vi.fn( + (): LocalValidationResult => options.localValidation ?? { ok: true }, + ), ensureLocalProviderReachable: vi.fn(() => options.localReachable ?? true), resolveContextWindowForModel: vi.fn((_provider: string, _model: string) => options.contextWindow === undefined ? null : options.contextWindow, diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index 43319de1d81..4e8bdf699a6 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { prepareOllamaApiExecution } from "../../../inference/local"; import { maybeWarmOllamaAfterDaemonRestart, type OllamaRestartRecoveryDeps, @@ -46,8 +47,23 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }); it("uses the persisted direct bridge route for both the default probe and warm-up", () => { - const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); - const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + const cleanup = vi.fn(() => ({ ok: true as const })); + const prepareDockerEnvironment = () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }); + const runCaptureImpl = vi.fn( + (_command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const runCaptureExImpl = vi.fn((_command: string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? successfulWarmResult() + : { stdout: "", exitCode: 1, timedOut: false }, + ); expect( maybeWarmOllamaAfterDaemonRestart( @@ -56,21 +72,37 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { model: "qwen3.6:35b", endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, }, - { runCaptureImpl, runCaptureExImpl }, + { + runCaptureImpl, + runCaptureExImpl, + prepareDockerEnvironment, + prepareOllamaApiExecution: (command, host, options) => + prepareOllamaApiExecution(command, host, { + ...options, + prepareDockerEnvironment, + }), + }, ), ).toEqual({ kind: "warmed", ok: true, timedOut: false }); expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, ); + expect(runCaptureImpl.mock.calls[0][0][0]).toBe("docker"); expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); + expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("docker"); expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({ model: "qwen3.6:35b", stream: false, think: false, }); + expect(runCaptureImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe("/tmp/credential-free-docker"); + expect(runCaptureExImpl.mock.calls[0][1]?.env?.DOCKER_CONFIG).toBe( + "/tmp/credential-free-docker", + ); + expect(cleanup).toHaveBeenCalledTimes(2); }); it("maps an auth-proxy route back to host loopback", () => { @@ -89,9 +121,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, ); + expect(runCaptureImpl.mock.calls[0][0][0]).toBe("curl"); expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, ); + expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("curl"); }); it("falls back to an allowlisted host instead of probing an arbitrary registry URL", () => { @@ -182,7 +216,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: true, reason: "timeout" }); + ).toEqual({ + kind: "warmed", + ok: false, + timedOut: true, + reason: "timeout", + endpoint: "http://127.0.0.1:11434", + detail: "warm-up exceeded 300 seconds", + }); }); it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { @@ -199,7 +240,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); + ).toMatchObject({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "ollama-error", + endpoint: "http://127.0.0.1:11434", + detail: expect.stringContaining("model not found"), + }); }); it("reports an endpoint that no longer holds the model instead of a warm failure (#9455)", () => { @@ -228,7 +276,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { endpoint: `http://host.docker.internal:${OLLAMA_PORT}`, inventoryLabel: "llama3.2:1b", }); - expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", undefined); + expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", expect.any(Function)); }); it("keeps the warm failure when the daemon does hold the model (#9455)", () => { @@ -245,7 +293,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); + ).toMatchObject({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "ollama-error", + endpoint: "http://127.0.0.1:11434", + detail: expect.stringContaining("runner stopped unexpectedly"), + }); }); it("accepts a completed thinking-only response from a thinking model", () => { @@ -278,7 +333,13 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "invalid-response" }); + ).toMatchObject({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "invalid-response", + endpoint: "http://127.0.0.1:11434", + }); }); it("reports a non-zero warm command exit", () => { @@ -290,7 +351,14 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), }, ), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }); + ).toEqual({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "command-failed", + endpoint: "http://127.0.0.1:11434", + detail: "warm-up exited 7", + }); }); it("reports a warm process spawn failure without throwing", () => { @@ -303,6 +371,13 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect( maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local", model: "qwen3.6:35b" }, deps), - ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }); + ).toEqual({ + kind: "warmed", + ok: false, + timedOut: false, + reason: "spawn-failed", + endpoint: "http://127.0.0.1:11434", + detail: "spawn failed", + }); }); }); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 8fa79227865..fd6c78ddd6e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -18,16 +18,19 @@ import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args" import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { describeModelInventory, + createOllamaApiCapture, + getOllamaApiCommand, getResolvedOllamaHost, ollamaInventoryContainsModel, OLLAMA_HOST_DOCKER_INTERNAL, OLLAMA_LOCALHOST, + prepareOllamaApiExecution, probeOllamaEndpointInventory, + type RunCaptureFn, type RunCaptureExFn, } from "../../../inference/local"; import { type OllamaRuntimeModelStatus, - type OllamaRuntimeRunCaptureFn, probeOllamaRuntimeModelStatus, } from "../../../inference/ollama-runtime-context"; import { runCaptureEx } from "../../../runner"; @@ -42,15 +45,14 @@ export interface OllamaRestartRecoveryDeps { probeRuntimeModelStatus?: ( model: string, getOllamaHost: () => string, - runCaptureImpl?: OllamaRuntimeRunCaptureFn, + runCaptureImpl?: RunCaptureFn, ) => OllamaRuntimeModelStatus; - probeModelInventory?: ( - host: string, - runCaptureImpl?: OllamaRuntimeRunCaptureFn, - ) => string[] | null; + probeModelInventory?: (host: string, runCaptureImpl?: RunCaptureFn) => string[] | null; runCaptureExImpl?: RunCaptureExFn; getOllamaHost?: () => string; - runCaptureImpl?: OllamaRuntimeRunCaptureFn; + runCaptureImpl?: RunCaptureFn; + prepareDockerEnvironment?: Parameters[2]; + prepareOllamaApiExecution?: typeof prepareOllamaApiExecution; } export type OllamaRestartRecoveryFailureReason = @@ -69,6 +71,8 @@ export type OllamaRestartRecoveryResult = ok: false; timedOut: boolean; reason: OllamaRestartRecoveryFailureReason; + endpoint: string; + detail: string; }; export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; @@ -150,9 +154,8 @@ function buildWarmCommand(model: string, hostname: string): string[] { keep_alive: "15m", options: { num_predict: 16 }, }); - return [ - "curl", - ...buildValidatedCurlCommandArgs([ + return getOllamaApiCommand( + buildValidatedCurlCommandArgs([ "-sS", "--connect-timeout", "3", @@ -164,7 +167,8 @@ function buildWarmCommand(model: string, hostname: string): string[] { body, `http://${hostname}:${OLLAMA_PORT}/api/generate`, ]), - ]; + hostname, + ); } function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" { @@ -189,6 +193,13 @@ function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid- } } +function boundedWarmFailureDetail(value: unknown, fallback: string): string { + const detail = String(value ?? "") + .replace(/\s+/g, " ") + .trim(); + return (detail || fallback).slice(0, 300); +} + /** * Warm a registered local Ollama model only when `/api/ps` proves that the * daemon is reachable and the selected model is no longer loaded. @@ -208,10 +219,16 @@ export function maybeWarmOllamaAfterDaemonRestart( const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); + const rawEndpoint = `http://${rawHost}:${OLLAMA_PORT}`; const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; + const rawCapture = createOllamaApiCapture( + deps.runCaptureImpl, + rawHost, + deps.prepareDockerEnvironment, + ); let status: OllamaRuntimeModelStatus; try { - status = probe(model, () => rawHost, deps.runCaptureImpl); + status = probe(model, () => rawHost, rawCapture); } catch { return { kind: "skipped", reason: "unreachable" }; } @@ -224,12 +241,44 @@ export function maybeWarmOllamaAfterDaemonRestart( const captureEx = deps.runCaptureExImpl ?? runCaptureEx; try { - const result = captureEx(buildWarmCommand(model, rawHost)); + const execution = (deps.prepareOllamaApiExecution ?? prepareOllamaApiExecution)( + buildWarmCommand(model, rawHost), + rawHost, + { operation: `Ollama restart warm-up for '${model}'` }, + ); + let result; + try { + result = captureEx(execution.command, { + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } if (result.timedOut) { - return { kind: "warmed", ok: false, timedOut: true, reason: "timeout" }; + return { + kind: "warmed", + ok: false, + timedOut: true, + reason: "timeout", + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail( + result.stderr, + `warm-up exceeded ${OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS} seconds`, + ), + }; } if (result.exitCode !== 0) { - return { kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }; + return { + kind: "warmed", + ok: false, + timedOut: false, + reason: "command-failed", + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail( + result.stderr || result.stdout, + `warm-up exited ${String(result.exitCode)}`, + ), + }; } const response = validateWarmResponse(result.stdout); // An Ollama error can mean a broken runner or a daemon that simply does not @@ -239,7 +288,7 @@ export function maybeWarmOllamaAfterDaemonRestart( // an unreadable inventory keeps the original warm-failure reason. if (response === "ollama-error") { const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory; - const inventory = probeInventory(rawHost, deps.runCaptureImpl); + const inventory = probeInventory(rawHost, rawCapture); if (inventory && !ollamaInventoryContainsModel(inventory, model)) { return { kind: "skipped", @@ -250,10 +299,27 @@ export function maybeWarmOllamaAfterDaemonRestart( } } if (response !== "ok") { - return { kind: "warmed", ok: false, timedOut: false, reason: response }; + return { + kind: "warmed", + ok: false, + timedOut: false, + reason: response, + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail(result.stdout, `Ollama returned ${response}`), + }; } return { kind: "warmed", ok: true, timedOut: false }; - } catch { - return { kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }; + } catch (error) { + return { + kind: "warmed", + ok: false, + timedOut: false, + reason: "spawn-failed", + endpoint: rawEndpoint, + detail: boundedWarmFailureDetail( + error instanceof Error ? error.message : error, + "warm-up process could not start", + ), + }; } } diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index d8516b67e5f..8888031343b 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -55,12 +55,17 @@ describe("runOllamaRestartRecovery", () => { ok: false, timedOut: true, reason: "timeout", + endpoint: "http://host.docker.internal:11434", + detail: "curl timed out after 300 seconds", })); const stderr = writes.join(""); - expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); - expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b' timed out"); - expect(stderr).toContain("continuing to OpenClaw dispatch"); + expect(stderr).toContain("Checking whether the Ollama model is loaded"); + expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b'"); + expect(stderr).toContain("timed out"); + expect(stderr).toContain("at http://host.docker.internal:11434"); + expect(stderr).toContain("OpenClaw dispatch will continue"); + expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); }); it.each([ @@ -76,16 +81,22 @@ describe("runOllamaRestartRecovery", () => { ok: false, timedOut: false, reason, + endpoint: "http://host.docker.internal:11434", + detail: "bounded failure detail", })); - expect(writes.join("")).toContain(message); + const stderr = writes.join(""); + expect(stderr).toContain(message); + expect(stderr).toContain("http://host.docker.internal:11434"); + expect(stderr).toContain("OpenClaw dispatch will continue"); + expect(stderr).toContain("confirm that it serves 'qwen3.6:35b'"); }); it.each([ ["already-loaded", "Ollama model 'qwen3.6:35b' is already loaded"], - ["unreachable", "Ollama was unreachable during the restart check"], + ["unreachable", "Ollama was unreachable during the model check"], ["missing-model", "No Ollama model is recorded for this sandbox"], - ["not-ollama", "Checking Ollama model readiness after daemon restart"], + ["not-ollama", "Checking whether the Ollama model is loaded"], ] as const)("handles the %s skip reason", (reason, message) => { const { writes, proc } = makeProcMock(); @@ -125,7 +136,7 @@ describe("runOllamaRestartRecovery", () => { expect(stderr).not.toContain("Ollama was unreachable during the restart check"); }); - it("contains an unexpected recovery exception", () => { + it("continues OpenClaw dispatch when Ollama recovery throws", () => { const { writes, proc } = makeProcMock(); expect(() => diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index dd1c0556942..4687c5ca201 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -46,7 +46,10 @@ function reportRecovery( return; } proc.stderr.write( - ` Ollama warm-up for '${model}' ${describeWarmFailure(result.reason)}; continuing to OpenClaw dispatch.\n`, + ` Ollama warm-up for '${model}' at ${result.endpoint} ${describeWarmFailure(result.reason)} ` + + `(${result.detail}). OpenClaw dispatch will continue. To retry the warm-up, restore ` + + `Ollama access to ${result.endpoint} and confirm that it serves '${model}', then rerun ` + + `this command.\n`, ); return; } @@ -71,7 +74,7 @@ function reportRecovery( break; case "unreachable": proc.stderr.write( - " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", + " Ollama was unreachable during the model check; continuing to OpenClaw dispatch.\n", ); break; case "missing-model": @@ -94,7 +97,7 @@ export function runOllamaRestartRecovery( proc: OllamaRestartRecoveryProcess, recoverOllama: OllamaRestartRecoveryFn = maybeWarmOllamaAfterDaemonRestart, ): void { - proc.stderr.write(" Checking Ollama model readiness after daemon restart...\n"); + proc.stderr.write(" Checking whether the Ollama model is loaded...\n"); try { reportRecovery(route, recoverOllama(route), proc); } catch { diff --git a/src/lib/actions/sandbox/destroy.test.ts b/src/lib/actions/sandbox/destroy.test.ts index 9af10ead97a..5884c3a7809 100644 --- a/src/lib/actions/sandbox/destroy.test.ts +++ b/src/lib/actions/sandbox/destroy.test.ts @@ -4,7 +4,11 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { assertUnambiguousDestroyContainerIdentity, cleanupSandboxServices } from "./destroy"; +import type { SandboxEntry } from "../../state/registry"; +import { + assertUnambiguousDestroyContainerIdentity, + cleanupSandboxServices, +} from "./destroy"; const SANDBOX = "mybox"; const mainPidDir = path.resolve("/tmp", `nemoclaw-services-${SANDBOX}`); @@ -71,6 +75,41 @@ describe("cleanupSandboxServices Google Chat tunnel cleanup (#7317)", () => { }); }); +describe("cleanupSandboxServices Ollama ownership", () => { + it("keeps a model shared through a compatible endpoint at the same local daemon", () => { + const own = { + name: SANDBOX, + provider: "ollama-local", + model: "llama3", + } as SandboxEntry; + const peer = { + name: "peer", + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + model: "llama3:latest", + } as SandboxEntry; + const unloadOllamaModels = vi.fn(); + + cleanupSandboxServices( + SANDBOX, + { stopHostServices: false }, + { + getSandbox: () => own, + listSandboxes: () => ({ sandboxes: [own, peer], defaultSandbox: null }), + loadPersistedOllamaHost: () => "127.0.0.1", + unloadOllamaModels, + withOllamaModelOwnershipLock: (operation) => operation(), + rmSync: vi.fn(), + runOpenshell: vi.fn(() => ({ status: 0 })), + stopGooglechatWebhookTunnel: vi.fn(() => googlechatPidDir), + googlechatWebhookTunnelPidDir: vi.fn(() => googlechatPidDir), + }, + ); + + expect(unloadOllamaModels).not.toHaveBeenCalled(); + }); +}); + describe("assertUnambiguousDestroyContainerIdentity (#8999)", () => { const dockerSandbox = { openshellDriver: "docker" } as { openshellDriver: string | null }; diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 2521f0c6ed8..a63595ab2d1 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -6,13 +6,13 @@ import path from "node:path"; import { CLI_NAME } from "../../cli/branding"; import { G, R, YW } from "../../cli/terminal-style"; -import { isNonInteractiveEnv } from "../../core/non-interactive"; import { prompt as askPrompt } from "../../credentials/store"; import { type DestroySandboxOptions, normalizeDestroySandboxOptions, } from "../../domain/lifecycle/options"; import { + isDestroyNonInteractiveEnv, resolveDestroyGatewayCleanupDecision, shouldStopHostServicesAfterDestroy, } from "../../domain/sandbox/destroy"; @@ -22,6 +22,12 @@ import { revokeHttpsPinRuntimeAdapterRoute, } from "../../inference/https-pin-runtime-adapter"; import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup"; +import { + isLocalOllamaRouteOwner, + loadPersistedOllamaHost, + type OllamaUnloadResult, + withOllamaModelOwnershipTransaction, +} from "../../inference/ollama/proxy"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, normalizeRuntimeProviderIdentity, @@ -38,7 +44,6 @@ import { validateName } from "../../runner"; import { killTimer as defaultKillShieldsTimer } from "../../shields/timer-control"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import * as onboardSession from "../../state/onboard-session"; -import type { RetainedSandboxRecoveryRecord } from "../../state/onboard-session/retained-sandbox-recovery"; import { resolveNemoclawStateDir } from "../../state/paths"; import * as registry from "../../state/registry"; import { @@ -89,8 +94,8 @@ type RemoveSandboxRegistryEntryWithReceiptDeps = { function selectRetainedSandboxRecoveryAuthority( sandboxName: string, sandbox: registry.SandboxEntry | null, - records: readonly RetainedSandboxRecoveryRecord[], -): RetainedSandboxRecoveryRecord | null { + records: readonly onboardSession.RetainedSandboxRecoveryRecord[], +): onboardSession.RetainedSandboxRecoveryRecord | null { const candidates = records.filter( (record) => record.sandboxName === sandboxName && record.sandboxIdentityFingerprint !== null, ); @@ -115,7 +120,9 @@ function selectRetainedSandboxRecoveryAuthority( return observedMatches.length === 1 ? observedMatches[0]! : null; } - const matchesRegistryAuthority = (record: RetainedSandboxRecoveryRecord): boolean => { + const matchesRegistryAuthority = ( + record: onboardSession.RetainedSandboxRecoveryRecord, + ): boolean => { const pending = sandbox.pendingCreateIdentity; if (pending) { return ( @@ -165,8 +172,21 @@ type RunOpenshell = (args: string[], opts?: Record) => { status export type CleanupSandboxServicesDeps = { getSandbox?: typeof registry.getSandbox; - stopAll?: (opts: { sandboxName: string }) => void; - unloadOllamaModels?: () => void; + listSandboxes?: typeof registry.listSandboxes; + stopAll?: (opts: { + sandboxName: string; + cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; + }) => OllamaUnloadResult | void; + unloadOllamaModels?: (onlyModels?: readonly string[]) => OllamaUnloadResult | void; + loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; + clearPendingOllamaModelCleanup?: ( + sandboxName: string, + releasedModels?: readonly string[], + ) => void; + loadPersistedOllamaHost?: () => "127.0.0.1" | "host.docker.internal" | null; + withOllamaModelOwnershipLock?: (operation: () => T) => T; + ollamaModelRefsMatch?: (left: string, right: string) => boolean; runOpenshell?: RunOpenshell; rmSync?: typeof fs.rmSync; stopGooglechatWebhookTunnel?: (sandboxName: string) => string; @@ -191,7 +211,7 @@ type RemoveShieldsStateDeps = { async function resolveCleanupGatewayDecision(options: DestroySandboxOptions): Promise { const decision = resolveDestroyGatewayCleanupDecision(options, { - nonInteractive: isNonInteractiveEnv(), + nonInteractive: isDestroyNonInteractiveEnv(), platform: process.platform, }); if (decision === "cleanup") return true; @@ -223,21 +243,66 @@ export function cleanupSandboxServices( const validatedSandboxName = validateName(sandboxName, "sandbox name"); const servicesPidDir = path.resolve("/tmp", `nemoclaw-services-${validatedSandboxName}`); const getSandbox = deps.getSandbox ?? registry.getSandbox; + const listSandboxes = deps.listSandboxes ?? registry.listSandboxes; const stopAll = deps.stopAll ?? - ((opts: { sandboxName: string }) => { + ((opts: { + sandboxName: string; + cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; + }) => { const services = require("../../tunnel/services") as { - stopAll: (opts: { sandboxName: string }) => void; + stopAll: (opts: { + sandboxName: string; + cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; + }) => OllamaUnloadResult | void; }; - services.stopAll(opts); + return services.stopAll(opts); }); const unloadOllamaModels = deps.unloadOllamaModels ?? - (() => { + ((onlyModels?: readonly string[]) => { const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as { - unloadOllamaModels: () => void; + unloadOllamaModels: (onlyModels?: readonly string[]) => OllamaUnloadResult; + }; + return unload(onlyModels); + }); + const loadPendingOllamaModelCleanup = + deps.loadPendingOllamaModelCleanup ?? + ((name: string) => { + const local = require("../../inference/ollama/proxy") as { + loadPendingOllamaModelCleanup(sandboxName: string): readonly string[]; + }; + return local.loadPendingOllamaModelCleanup(name); + }); + const clearPendingOllamaModelCleanup = + deps.clearPendingOllamaModelCleanup ?? + ((name: string, releasedModels?: readonly string[]) => { + const local = require("../../inference/ollama/proxy") as { + clearPendingOllamaModelCleanup(sandboxName: string, models?: readonly string[]): void; }; - unload(); + local.clearPendingOllamaModelCleanup(name, releasedModels); + }); + const loadPersistedOllamaHost = + deps.loadPersistedOllamaHost ?? + (require("../../inference/local") as typeof import("../../inference/local")) + .loadPersistedOllamaHost; + const withOllamaModelOwnershipLock = + deps.withOllamaModelOwnershipLock ?? + ((operation: () => T): T => { + const proxy = require("../../inference/ollama/proxy") as { + withOllamaModelOwnershipLock(operation: () => T): T; + }; + return proxy.withOllamaModelOwnershipLock(operation); + }); + const ollamaModelRefsMatch = + deps.ollamaModelRefsMatch ?? + ((left: string, right: string) => { + const proxy = require("../../inference/ollama/proxy") as { + ollamaModelRefsMatch(leftModel: string, rightModel: string): boolean; + }; + return proxy.ollamaModelRefsMatch(left, right); }); const runOpenshell = deps.runOpenshell ?? @@ -286,18 +351,79 @@ export function cleanupSandboxServices( ); } + let ollamaCleanup: OllamaUnloadResult | void = undefined; if (stopHostServices) { - // `stopAll()` already runs `unloadOllamaModels()` unconditionally — - // see src/lib/tunnel/services.ts. Don't double-call here. - stopAll({ sandboxName: validatedSandboxName }); + // `stopAll()` owns the host-wide unload when this sandbox has an Ollama + // route or retained cleanup work. Don't probe an unrelated daemon for a + // sandbox with no Ollama ownership, and don't double-call cleanup here. + try { + ollamaCleanup = withOllamaModelOwnershipLock(() => { + const sandbox = getSandbox(validatedSandboxName); + const pending = loadPendingOllamaModelCleanup(validatedSandboxName); + const selectedHost = loadPersistedOllamaHost(); + const cleanupOllamaModels = Boolean( + (sandbox && isLocalOllamaRouteOwner(sandbox, selectedHost)) || pending.length > 0, + ); + return stopAll({ + sandboxName: validatedSandboxName, + cleanupOllamaModels, + unloadOllamaModels: () => unloadOllamaModels(), + }); + }); + } catch (error) { + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .trim() + .slice(0, 300); + throw new Error( + `Host-service cleanup failed after sandbox '${validatedSandboxName}' was deleted: ${detail || "unknown error"}. ` + + `The local registry and cleanup state for '${validatedSandboxName}' were retained for recovery; ` + + `restore the reported dependency, then retry \`nemoclaw ${validatedSandboxName} destroy\`.`, + { cause: error }, + ); + } } else { // No global stop, so `stopAll()` did not run; explicitly free Ollama // models for this sandbox if its provider used Ollama. Without this // branch a single-sandbox destroy would leave models loaded on the GPU. - const sb = getSandbox(validatedSandboxName); - if (sb?.provider?.includes("ollama")) { - unloadOllamaModels(); - } + withOllamaModelOwnershipLock(() => { + const sb = getSandbox(validatedSandboxName); + const selectedHost = loadPersistedOllamaHost(); + const peers = listSandboxes().sandboxes.filter( + (candidate) => + candidate.name !== validatedSandboxName && + isLocalOllamaRouteOwner(candidate, selectedHost), + ); + const pending = loadPendingOllamaModelCleanup(validatedSandboxName); + const currentModel = String(sb?.model ?? "").trim(); + const candidates = [ + ...pending, + ...(sb && isLocalOllamaRouteOwner(sb, selectedHost) && currentModel ? [currentModel] : []), + ].filter( + (model, index, models) => + models.findIndex((candidate) => ollamaModelRefsMatch(candidate, model)) === index && + !peers.some( + (candidate) => candidate.model && ollamaModelRefsMatch(model, candidate.model), + ), + ); + if (candidates.length === 0) return; + ollamaCleanup = unloadOllamaModels(candidates); + if (!ollamaCleanup || ollamaCleanup.ok) { + clearPendingOllamaModelCleanup(validatedSandboxName, candidates); + } + }); + } + if (ollamaCleanup && !ollamaCleanup.ok) { + const recoveryAction = + ollamaCleanup.outcome === "discovery-failed" + ? `restore access to ${ollamaCleanup.endpoint}` + : ollamaCleanup.outcome === "still-resident" + ? `stop the recorded model at ${ollamaCleanup.endpoint}` + : `allow the model unload request at ${ollamaCleanup.endpoint}`; + throw new Error( + `Ollama model cleanup failed at ${ollamaCleanup.endpoint} (${ollamaCleanup.outcome}: ${ollamaCleanup.message ?? "no detail"}). ` + + `The sandbox registry and saved route were retained; ${recoveryAction}, then retry destroy.`, + ); } try { @@ -957,6 +1083,25 @@ async function destroySandboxUnlocked( ` ${YW}⚠${R} Failed to retire portable lifecycle authority for '${sandboxName}': ${redactDestroyError(error)}`, ); } + const localInference = require("../../inference/local") as { + clearPersistedOllamaHostIfUnused( + routes: readonly { provider?: string | null; endpointUrl?: string | null }[], + ): boolean; + }; + if (sandbox && isLocalOllamaRouteOwner(sandbox)) { + try { + await withOllamaModelOwnershipTransaction(() => { + const selectedHost = loadPersistedOllamaHost(); + if (!isLocalOllamaRouteOwner(sandbox, selectedHost)) return; + const remainingSandboxes = registry.listSandboxes().sandboxes; + localInference.clearPersistedOllamaHostIfUnused(remainingSandboxes); + }); + } catch (error) { + console.warn( + ` ${YW}⚠${R} Failed to retire the final local Ollama route receipt: ${redactDestroyError(error)}`, + ); + } + } } if (deleteSucceededOrAlreadyGone && removed && priorHttpsPinRouteId) { await revokeDestroyedSandboxHttpsPinRoute(cleanupGatewayName, priorHttpsPinRouteId); diff --git a/src/lib/actions/sandbox/policy-channel-conflict.test.ts b/src/lib/actions/sandbox/policy-channel-conflict.test.ts index 640fe2eca9b..bf6f2209ba3 100644 --- a/src/lib/actions/sandbox/policy-channel-conflict.test.ts +++ b/src/lib/actions/sandbox/policy-channel-conflict.test.ts @@ -590,7 +590,7 @@ describe("addSandboxChannel cross-sandbox conflict check (#4305)", () => { ); expect(removePresetMock).toHaveBeenCalledWith("alpha", "telegram"); - }); + }, 15_000); // Scenario 5b it("different hash on the other sandbox is NOT a conflict (no warning, add proceeds)", async () => { diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index cd02dc35e5d..b52ac71629a 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -83,7 +83,6 @@ const localProviderScenarios = [ validateLocalProvider: () => ({ ok: true }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:11435/v1", applyLocalInferenceRoute, - getOllamaWarmupCommand: () => ["true"], run: () => ({ status: 0 }), shouldFrontOllamaWithProxy: () => false, ensureOllamaAuthProxy: vi.fn(), @@ -93,6 +92,8 @@ const localProviderScenarios = [ localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, + persistResolvedOllamaHost: () => () => {}, }, OLLAMA_PROXY_CREDENTIAL_ENV: "NEMOCLAW_OLLAMA_PROXY_TOKEN", ...unusedCommonInferenceDeps, @@ -139,101 +140,96 @@ function makeRouteApplier() { installRebuildFlowTestHooks({ acceptThirdPartySoftware: true }); describe("rebuild local-provider recreation", () => { - it.each( - localProviderScenarios, - )("recreates a missing $provider gateway provider through the resumed local setup path", async ({ - provider, - model, - baseUrl, - credentialEnv, - setup, - }) => { - let sourceDeleted = false; - let harness!: RebuildFlowHarness; - let setupResult: SetupResult | undefined; - harness = createRebuildFlowHarness({ - sandboxEntry: { provider, model, credentialEnv: null }, - onboard: async (session) => { - const callsBeforeSetup = harness.runOpenshellSpy.mock.calls.map( - (call) => call[0] as string[], - ); - expect(callsBeforeSetup).not.toContainEqual(["provider", "get", provider]); - expect(session.provider).toBe(provider); - expect(session.model).toBe(model); - expect(session.steps.provider_selection.status).toBe("pending"); - expect(session.steps.inference.status).toBe("pending"); + it.each(localProviderScenarios)( + "recreates a missing $provider gateway provider through the resumed local setup path", + async ({ provider, model, baseUrl, credentialEnv, setup }) => { + let sourceDeleted = false; + let harness!: RebuildFlowHarness; + let setupResult: SetupResult | undefined; + harness = createRebuildFlowHarness({ + sandboxEntry: { provider, model, credentialEnv: null }, + onboard: async (session) => { + const callsBeforeSetup = harness.runOpenshellSpy.mock.calls.map( + (call) => call[0] as string[], + ); + expect(callsBeforeSetup).not.toContainEqual(["provider", "get", provider]); + expect(session.provider).toBe(provider); + expect(session.model).toBe(model); + expect(session.steps.provider_selection.status).toBe("pending"); + expect(session.steps.inference.status).toBe("pending"); - setupResult = await setup(makeRouteApplier()); - }, - }); - harness.session.provider = provider; - harness.session.model = model; - harness.runOpenshellSpy.mockImplementation((args: string[]) => { - sourceDeleted ||= args.join(" ") === "sandbox delete -g nemoclaw alpha"; - return args[0] === "sandbox" && args[1] === "get" - ? { - status: 1, - stdout: "", - stderr: "sandbox alpha not found", - } - : { - status: args[0] === "provider" && args[1] === "get" ? 1 : 0, - stdout: "", - stderr: "", - }; - }); - const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; - harness.captureOpenshellSpy.mockImplementation((args: unknown) => { - const argv = Array.isArray(args) ? args.map(String) : []; - return argv.join(" ") === "sandbox get -g nemoclaw alpha" && !sourceDeleted - ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } - : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; - }); + setupResult = await setup(makeRouteApplier()); + }, + }); + harness.session.provider = provider; + harness.session.model = model; + harness.runOpenshellSpy.mockImplementation((args: string[]) => { + sourceDeleted ||= args.join(" ") === "sandbox delete -g nemoclaw alpha"; + return args[0] === "sandbox" && args[1] === "get" + ? { + status: 1, + stdout: "", + stderr: "sandbox alpha not found", + } + : { + status: args[0] === "provider" && args[1] === "get" ? 1 : 0, + stdout: "", + stderr: "", + }; + }); + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + harness.captureOpenshellSpy.mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args.map(String) : []; + return argv.join(" ") === "sandbox get -g nemoclaw alpha" && !sourceDeleted + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } + : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; + }); - await expect( - harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).resolves.toBeUndefined(); + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).resolves.toBeUndefined(); - const calls = harness.runOpenshellSpy.mock.calls.map((call) => call[0] as string[]); - const deleteCall = calls.findIndex( - (args) => args.join(" ") === "sandbox delete -g nemoclaw alpha", - ); - const providerLookup = calls.findIndex( - (args) => args[0] === "provider" && args[1] === "get" && args[2] === provider, - ); - expect(setupResult).toEqual({ done: false }); - expect(harness.onboardSpy).toHaveBeenCalledWith( - expect.objectContaining({ resume: true, nonInteractive: true, recreateSandbox: true }), - ); - expect(deleteCall).toBeGreaterThanOrEqual(0); - expect(providerLookup).toBeGreaterThan(deleteCall); - expect(calls).toContainEqual(["provider", "get", provider]); - expect(calls).toContainEqual([ - "provider", - "create", - "--name", - provider, - "--type", - "openai", - "--credential", - credentialEnv, - "--config", - `OPENAI_BASE_URL=${baseUrl}`, - ]); - expect(calls).toContainEqual([ - "inference", - "set", - "--no-verify", - "--provider", - provider, - "--model", - model, - "--timeout", - "30", - ]); - expect(calls.some((args) => args[0] === "provider" && args[1] === "update")).toBe(false); - expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith("alpha", harness.backupPath, { - targetAgentType: "openclaw", - }); - }); + const calls = harness.runOpenshellSpy.mock.calls.map((call) => call[0] as string[]); + const deleteCall = calls.findIndex( + (args) => args.join(" ") === "sandbox delete -g nemoclaw alpha", + ); + const providerLookup = calls.findIndex( + (args) => args[0] === "provider" && args[1] === "get" && args[2] === provider, + ); + expect(setupResult).toEqual({ done: false }); + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ resume: true, nonInteractive: true, recreateSandbox: true }), + ); + expect(deleteCall).toBeGreaterThanOrEqual(0); + expect(providerLookup).toBeGreaterThan(deleteCall); + expect(calls).toContainEqual(["provider", "get", provider]); + expect(calls).toContainEqual([ + "provider", + "create", + "--name", + provider, + "--type", + "openai", + "--credential", + credentialEnv, + "--config", + `OPENAI_BASE_URL=${baseUrl}`, + ]); + expect(calls).toContainEqual([ + "inference", + "set", + "--no-verify", + "--provider", + provider, + "--model", + model, + "--timeout", + "30", + ]); + expect(calls.some((args) => args[0] === "provider" && args[1] === "update")).toBe(false); + expect(harness.restoreSandboxStateSpy).toHaveBeenCalledWith("alpha", harness.backupPath, { + targetAgentType: "openclaw", + }); + }, + ); }); diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 4af84fda5b8..fd332ae4b16 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -765,6 +765,27 @@ describe("stopSandbox Ollama GPU release", () => { ); }); + it("never unloads a model a compatible local endpoint sibling also uses", () => { + const unloadOllamaModels = vi.fn(() => successfulUnload()); + const peer = sandbox({ + endpointUrl: "http://127.0.0.1:11434/v1", + model: "qwen2.5:7b", + name: "peer", + provider: "compatible-endpoint", + }); + const h = harness({ + listSandboxes: registryOf(ollamaSandbox, peer), + loadPersistedOllamaHost: () => "127.0.0.1", + unloadOllamaModels, + }); + h.getSandbox.mockReturnValue(ollamaSandbox); + + const result = stopSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(0); + expect(unloadOllamaModels).not.toHaveBeenCalled(); + }); + it("ignores a stopped sibling registry row and releases the exclusive model (#10074)", () => { const unloadOllamaModels = vi.fn(() => successfulUnload()); const stoppedPeer = sandbox({ diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 356ea66e0f1..381f8b2783c 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -4,7 +4,9 @@ import { CLI_NAME } from "../../cli/branding"; import { decideOllamaModelOwnership, + isLocalOllamaRouteOwner, matchingOllamaModelPeers, + type OllamaHostRoute, } from "../../inference/ollama/model-ownership"; import type { OllamaUnloadResult } from "../../inference/ollama/proxy"; import { @@ -139,16 +141,19 @@ function releaseStoppedSandboxOllamaModel( deps: SandboxStopDeps, log: (message: string) => void, ): OllamaStopReleaseResult { - if (!sandbox.provider?.includes("ollama")) return { ok: true }; + if (!isLocalOllamaRouteOwner(sandbox)) return { ok: true }; try { + const proxy = require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"); const withOwnershipLock = - deps.withOllamaModelOwnershipLock ?? - (require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy")) - .withOllamaModelOwnershipLock; + deps.withOllamaModelOwnershipLock ?? proxy.withOllamaModelOwnershipLock; + const loadPersistedOllamaHost = + deps.loadPersistedOllamaHost ?? proxy.loadPersistedOllamaHost; return withOwnershipLock(() => { + const selectedHost = loadPersistedOllamaHost(); + if (!isLocalOllamaRouteOwner(sandbox, selectedHost)) return { ok: true }; const { sandboxes } = (deps.listSandboxes ?? registry.listSandboxes)(); - const matchingPeers = matchingOllamaModelPeers(sandbox, sandboxes); + const matchingPeers = matchingOllamaModelPeers(sandbox, sandboxes, selectedHost); const discovery = ( deps.discoverActiveOllamaSandboxNames ?? discoverActiveOllamaSandboxNames )(matchingPeers, deps.environment ?? process.env); @@ -166,6 +171,7 @@ function releaseStoppedSandboxOllamaModel( sandbox, sandboxes, discovery.activeSandboxNames, + selectedHost, ); if (ownership.kind === "missing-model") { log(" Ollama model release skipped: the sandbox registry has no model."); @@ -238,6 +244,7 @@ export interface SandboxStopDeps { ) => OllamaActiveOwnershipDiscovery; unloadOllamaModels?: (onlyModels: readonly string[]) => OllamaUnloadResult; decideOllamaModelOwnership?: typeof decideOllamaModelOwnership; + loadPersistedOllamaHost?: () => OllamaHostRoute | null; withOllamaModelOwnershipLock?: typeof import("../../inference/ollama/proxy").withOllamaModelOwnershipLock; withLifecycleLockSync?: typeof withSandboxLifecycleLockSync; log?: (message: string) => void; diff --git a/src/lib/adapters/http/container-curl-probe.ts b/src/lib/adapters/http/container-curl-probe.ts index 90955adca3c..54fb020fbee 100644 --- a/src/lib/adapters/http/container-curl-probe.ts +++ b/src/lib/adapters/http/container-curl-probe.ts @@ -11,7 +11,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -export const CONTAINER_REACHABILITY_IMAGE = "curlimages/curl:8.10.1"; +export const CONTAINER_REACHABILITY_IMAGE = + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b"; const MAX_CONTAINER_CURL_STDOUT_BYTES = 8 * 1024 * 1024 + 256; const HTTP_STATUS_MARKER_PREFIX = "\n__NEMOCLAW_CONTAINER_HTTP_STATUS_"; diff --git a/src/lib/domain/sandbox/destroy.ts b/src/lib/domain/sandbox/destroy.ts index e41b430f28b..c3faa255bc5 100644 --- a/src/lib/domain/sandbox/destroy.ts +++ b/src/lib/domain/sandbox/destroy.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { parseLiveSandboxEntries } from "../../runtime-recovery"; +import { isNonInteractiveEnv } from "../../core/non-interactive"; import { resolveSandboxContainerOwner } from "./container-owner"; const ANSI_RE = /\x1b\[[0-9;]*m/g; @@ -97,6 +98,10 @@ export function shouldStopHostServicesAfterDestroy(input: { ); } +export function isDestroyNonInteractiveEnv(): boolean { + return isNonInteractiveEnv(); +} + export function shouldCleanupGatewayAfterDestroy(input: { deleteSucceededOrAlreadyGone: boolean; removedRegistryEntry: boolean; diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index 7c9f48fa6ed..e57ec9d095c 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -8,7 +8,7 @@ import { isSafeModelId, shouldSkipResponsesProbe } from "../validation"; import { isSafeLlamaCppServedModelAlias, LLAMA_CPP_CREDENTIAL_ENV } from "./llama-cpp/contract"; -import { DEFAULT_OLLAMA_MODEL } from "./local"; +import { DEFAULT_OLLAMA_MODEL_TAG as DEFAULT_OLLAMA_MODEL } from "./ollama-model-registry"; import { OLLAMA_LOCAL_CREDENTIAL_ENV } from "./ollama/contract"; import { OPENROUTER_CREDENTIAL_ENV, OPENROUTER_PROVIDER_NAME } from "./openrouter"; diff --git a/src/lib/inference/context-window.test.ts b/src/lib/inference/context-window.test.ts index e2e46d3d17c..8a1127dd5e1 100644 --- a/src/lib/inference/context-window.test.ts +++ b/src/lib/inference/context-window.test.ts @@ -6,12 +6,17 @@ import { afterEach, describe, expect, it, vi } from "vitest"; // Most tests inject deps, so these mocks replace the real inference stack under // vitest. The default-deps suite below calls through to them. vi.mock("./local", () => ({ + createOllamaApiCaptureEx: vi.fn((capture) => capture), getOllamaProbeCommand: vi.fn(() => ["curl", "ollama-probe"]), resolveOllamaRuntimeContextWindow: vi.fn(() => null), })); vi.mock("./vllm-runtime-context", () => ({ resolveVllmContextWindowFromModels: vi.fn() })); -import { getOllamaProbeCommand, resolveOllamaRuntimeContextWindow } from "./local"; +import { + createOllamaApiCaptureEx, + getOllamaProbeCommand, + resolveOllamaRuntimeContextWindow, +} from "./local"; import { type ContextWindowDeps, resolveContextWindowForModel } from "./context-window"; // The default dependencies reach ../runner through a lazy CJS require, so swap the @@ -19,7 +24,7 @@ import { type ContextWindowDeps, resolveContextWindowForModel } from "./context- type CaptureStub = { stdout: string; exitCode: number | null; timedOut: boolean }; const runner = require("../runner") as { - runCaptureEx: (cmd: readonly string[]) => CaptureStub; + runCaptureEx: (cmd: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => CaptureStub; }; function captured(timedOut = false): CaptureStub { @@ -101,6 +106,7 @@ describe("resolveContextWindowForModel default dependencies (#8974)", () => { afterEach(() => { runner.runCaptureEx = originalRunCaptureEx; + vi.mocked(createOllamaApiCaptureEx).mockImplementation((capture) => capture!); }); it("ollama-local: runs the blocking probe command, not a backgrounded warm-up", () => { @@ -134,6 +140,37 @@ describe("resolveContextWindowForModel default dependencies (#8974)", () => { expect(getOllamaProbeCommand).toHaveBeenLastCalledWith("qwen3.5:9b", 300); }); + it("ollama-local: isolates Docker credentials for the initial and retry probes", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + const cleanup = vi.fn(); + const environments: Array = []; + let attempts = 0; + runner.runCaptureEx = (_command, options) => { + environments.push(options?.env); + attempts += 1; + return captured(attempts === 1); + }; + vi.mocked(createOllamaApiCaptureEx).mockImplementation((capture) => (command, options) => { + try { + return capture!(command, { + ...options, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }); + } finally { + cleanup(); + } + }); + vi.mocked(getOllamaProbeCommand).mockReturnValue(["docker", "run", "ollama-probe"]); + vi.mocked(resolveOllamaRuntimeContextWindow).mockReturnValue(16384); + + expect(resolveContextWindowForModel("ollama-local", "qwen3.5:9b")).toBe(16384); + expect(environments).toEqual([ + { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + ]); + expect(cleanup).toHaveBeenCalledTimes(2); + }); + it("ollama-local: does not retry when the first probe fails without timing out", () => { vi.spyOn(console, "log").mockImplementation(() => {}); let attempts = 0; diff --git a/src/lib/inference/context-window.ts b/src/lib/inference/context-window.ts index d0179f16380..ee2ef459c47 100644 --- a/src/lib/inference/context-window.ts +++ b/src/lib/inference/context-window.ts @@ -12,6 +12,7 @@ import { DEFAULT_CONTEXT_WINDOW } from "./config"; import { + createOllamaApiCaptureEx, getLocalProviderHealthEndpoint, getManagedVllmProviderBinding, getOllamaProbeCommand, @@ -44,6 +45,7 @@ const defaultContextWindowDeps: ContextWindowDeps = { // Lazy require: ../runner is CJS and a top-level require fails to resolve // under the test runner. Runs only for the real (non-injected) deps. const { runCaptureEx } = require("../runner") as { runCaptureEx: RunCaptureExFn }; + const captureEx = createOllamaApiCaptureEx(runCaptureEx); console.log(` Priming Ollama model: ${model}`); // Blocking probe, the command onboarding also waits for. A backgrounded // warm-up returns before the daemon has the model resident, and `/api/ps` @@ -51,8 +53,8 @@ const defaultContextWindowDeps: ContextWindowDeps = { // model can exceed the 120 s default on unified-memory and tight-VRAM // hosts, so retry once at 300 s as onboarding does. // A connection-refused result keeps `timedOut` false and skips the retry. - if (runCaptureEx(getOllamaProbeCommand(model)).timedOut) { - runCaptureEx(getOllamaProbeCommand(model, 300)); + if (captureEx(getOllamaProbeCommand(model)).timedOut) { + captureEx(getOllamaProbeCommand(model, 300)); } }, // currentContextWindow = null → always probe (we recompute on every switch diff --git a/src/lib/inference/local-adapter-lifecycle.ts b/src/lib/inference/local-adapter-lifecycle.ts index be37413ba43..ae5fda2aca2 100644 --- a/src/lib/inference/local-adapter-lifecycle.ts +++ b/src/lib/inference/local-adapter-lifecycle.ts @@ -8,7 +8,7 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; -import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../core/ports"; +import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT, OLLAMA_PORT } from "../core/ports"; import { waitUntilAsync } from "../core/wait"; import { rejectSymlinksOnPath } from "../state/config-io"; import { nemoclawStateRoot } from "../state/state-root"; @@ -42,6 +42,55 @@ export function resolveSharedLocalAdapterStateRoot(homeDir: string = os.homedir( export const SHARED_LOCAL_ADAPTER_STATE_DIR = resolveSharedLocalAdapterStateRoot(); export const LOCAL_ADAPTER_HEALTH_MAX_RESPONSE_BYTES = 64 * 1024; +export const OLLAMA_LOCALHOST = "127.0.0.1"; +export const OLLAMA_HOST_DOCKER_INTERNAL = "host.docker.internal"; + +export type OllamaHostRoute = + | typeof OLLAMA_LOCALHOST + | typeof OLLAMA_HOST_DOCKER_INTERNAL; + +/** Registry fields that identify a route backed by NemoClaw's host Ollama daemon. */ +export type OllamaRouteHolder = { + readonly provider?: string | null; + readonly endpointUrl?: string | null; +}; + +function isSupportedOllamaRouteHost(host: string): host is OllamaHostRoute { + return host === OLLAMA_LOCALHOST || host === OLLAMA_HOST_DOCKER_INTERNAL; +} + +/** + * Return whether a recorded inference route uses NemoClaw's host Ollama daemon. + * + * Direct and legacy Ollama providers own that daemon by definition. A + * compatible endpoint owns it only when its credential-free HTTP URL names + * the selected fixed host route and Ollama port. Remote compatible endpoints + * are never classified as local owners. + */ +export function isLocalOllamaRouteOwner( + route: OllamaRouteHolder, + selectedHost: OllamaHostRoute | null = null, +): boolean { + if (route.provider === "ollama-local" || route.provider?.startsWith("ollama/")) return true; + if (route.provider !== "compatible-endpoint" || !route.endpointUrl) return false; + + try { + const endpoint = new URL(route.endpointUrl); + const endpointHost = endpoint.hostname.toLowerCase(); + const hostMatches = selectedHost + ? endpointHost === selectedHost + : isSupportedOllamaRouteHost(endpointHost); + return ( + endpoint.protocol === "http:" && + endpoint.username === "" && + endpoint.password === "" && + Number(endpoint.port) === OLLAMA_PORT && + hostMatches + ); + } catch { + return false; + } +} export function ensureLocalAdapterStateDir(stateDir = DEFAULT_LOCAL_ADAPTER_STATE_DIR): void { rejectSymlinksOnPath(stateDir); diff --git a/src/lib/inference/local-windows-ollama-transport.test.ts b/src/lib/inference/local-windows-ollama-transport.test.ts new file mode 100644 index 00000000000..63b8bb3ab98 --- /dev/null +++ b/src/lib/inference/local-windows-ollama-transport.test.ts @@ -0,0 +1,508 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + applyOllamaRuntimeContextWindow, + clearPersistedOllamaHostIfUnused, + CONTAINER_REACHABILITY_IMAGE, + createOllamaApiCapture, + findReachableOllamaHost, + getOllamaApiCommand, + getOllamaModelOptions, + getResolvedOllamaHost, + OLLAMA_HOST_DOCKER_INTERNAL, + loadPersistedOllamaHost, + persistResolvedOllamaHost, + probeLocalProviderHealth, + probeOllamaModelCapabilities, + resetOllamaHostCache, + resetOllamaRuntimeContextWindowAutoState, + runOllamaWarmup, + setResolvedOllamaHost, + validateLocalProvider, + validateOllamaModel, +} from "./local"; +import { withOllamaModelOwnershipTransaction } from "./ollama/proxy"; + +function respondsOnlyThroughDockerDesktop(apiPath: string, response: string) { + return vi.fn((command: readonly string[]) => { + const expectedUrl = `http://host.docker.internal:11434${apiPath}`; + const usesExpectedTransport = + command[0] === "docker" && + command.includes("run") && + command.includes("--rm") && + command.includes(CONTAINER_REACHABILITY_IMAGE) && + command.some((argument) => argument === expectedUrl); + return usesExpectedTransport ? response : ""; + }); +} + +describe("Windows-host Ollama transport", () => { + afterEach(() => { + resetOllamaHostCache(); + resetOllamaRuntimeContextWindowAutoState(); + }); + + it("selects Docker Desktop only for the Windows-host transport owner", () => { + expect( + getOllamaApiCommand( + ["-sf", "http://host.docker.internal:11434/api/tags"], + OLLAMA_HOST_DOCKER_INTERNAL, + ), + ).toEqual([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "-sf", + "http://host.docker.internal:11434/api/tags", + ]); + expect(getOllamaApiCommand(["-sf", "http://127.0.0.1:11434/api/tags"], "127.0.0.1")).toEqual([ + "curl", + "-sf", + "http://127.0.0.1:11434/api/tags", + ]); + }); + + it("restores the accepted route receipt in a fresh process", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-receipt-")); + try { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + persistResolvedOllamaHost(undefined, stateRoot); + resetOllamaHostCache(); + + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("restores the prior receipt when staged provider setup rolls back", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-rollback-")); + try { + persistResolvedOllamaHost("127.0.0.1", stateRoot); + const rollback = persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + + rollback(); + + expect(loadPersistedOllamaHost(stateRoot)).toBe("127.0.0.1"); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("retires the final persisted route after Ollama ownership ends", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-retire-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect(clearPersistedOllamaHostIfUnused([{ provider: "nvidia-prod" }], stateRoot)).toBe(true); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("retains the persisted route while another Ollama sandbox owns it", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-retain-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect(clearPersistedOllamaHostIfUnused([{ provider: "ollama-local" }], stateRoot)).toBe( + false, + ); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("retains the route for a compatible endpoint at the selected local daemon", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-compatible-retain-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect( + clearPersistedOllamaHostIfUnused( + [ + { + provider: "compatible-endpoint", + endpointUrl: "http://host.docker.internal:11434/v1", + }, + ], + stateRoot, + ), + ).toBe(false); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("retires the route when only a remote compatible endpoint remains", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-compatible-remote-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + + expect( + clearPersistedOllamaHostIfUnused( + [ + { + provider: "compatible-endpoint", + endpointUrl: "https://ollama.example.com:11434/v1", + }, + ], + stateRoot, + ), + ).toBe(true); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("serializes route publication with final ownership retirement", async () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-transition-")); + const routes: Array<{ provider: string }> = []; + let staged!: () => void; + let resume!: () => void; + const stagedRoute = new Promise((resolve) => { + staged = resolve; + }); + const resumeOnboarding = new Promise((resolve) => { + resume = resolve; + }); + + try { + const onboarding = withOllamaModelOwnershipTransaction(async () => { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + staged(); + await resumeOnboarding; + routes.push({ provider: "ollama-local" }); + }); + await stagedRoute; + + let retirementEntered = false; + const retirement = withOllamaModelOwnershipTransaction(() => { + retirementEntered = true; + clearPersistedOllamaHostIfUnused(routes, stateRoot); + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(retirementEntered).toBe(false); + + resume(); + await onboarding; + await retirement; + + expect(retirementEntered).toBe(true); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("re-probes a stale persisted route before fresh-process connect discovery", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-connect-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + resetOllamaHostCache(); + const capture = vi.fn((command: readonly string[]) => + command.some((argument) => argument === "http://127.0.0.1:11434/api/tags") + ? JSON.stringify({ models: [] }) + : "", + ); + + expect(findReachableOllamaHost(capture, { isWsl: true }, stateRoot)).toBe("127.0.0.1"); + expect(capture).toHaveBeenCalledTimes(2); + expect(capture.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(["docker", "run", "http://host.docker.internal:11434/api/tags"]), + ); + expect(capture.mock.calls[1]?.[0]).toEqual( + expect.arrayContaining(["curl", "http://127.0.0.1:11434/api/tags"]), + ); + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + expect(getResolvedOllamaHost()).toBe("127.0.0.1"); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("rejects an untrusted persisted host", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-invalid-")); + try { + writeFileSync( + join(stateRoot, "ollama-host.json"), + JSON.stringify({ schemaVersion: 1, host: "example.com" }), + ); + resetOllamaHostCache(); + + expect(loadPersistedOllamaHost(stateRoot)).toBeNull(); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("probes a resolved Windows-host route through Docker Desktop", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-health-")); + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + resetOllamaHostCache(); + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + + const captureEx = vi.fn((command: readonly string[]) => ({ + stdout: + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command[3] === CONTAINER_REACHABILITY_IMAGE && + command.some((argument) => argument === "http://host.docker.internal:11434/api/tags") + ? JSON.stringify({ models: [] }) + : "", + stderr: "", + exitCode: 0, + timedOut: false, + })); + const result = probeLocalProviderHealth("ollama-local", { + findReachableOllamaHostImpl: () => OLLAMA_HOST_DOCKER_INTERNAL, + loadOllamaProxyTokenImpl: () => null, + ollamaRunCaptureExImpl: captureEx, + }); + + expect(result).toMatchObject({ + ok: true, + endpoint: "http://host.docker.internal:11434/api/tags", + }); + expect(captureEx).toHaveBeenCalledOnce(); + } finally { + resetOllamaHostCache(); + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("reads the model inventory through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = respondsOnlyThroughDockerDesktop( + "/api/tags", + JSON.stringify({ models: [{ name: "qwen3.5:9b" }] }), + ); + + expect(getOllamaModelOptions(capture)).toEqual(["qwen3.5:9b"]); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("validates a Windows-host model through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = respondsOnlyThroughDockerDesktop( + "/api/show", + JSON.stringify({ capabilities: ["tools"] }), + ); + const captureEx = vi.fn((command: readonly string[]) => { + const expected = + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command[3] === CONTAINER_REACHABILITY_IMAGE && + command.some((argument) => argument === "http://host.docker.internal:11434/api/generate"); + return { + stdout: expected ? JSON.stringify({ done: true, response: "ready" }) : "", + stderr: "", + exitCode: expected ? 0 : 1, + timedOut: false, + }; + }); + + expect(validateOllamaModel("qwen3.5:9b", capture, () => false, captureEx)).toEqual({ + ok: true, + }); + expect(captureEx).toHaveBeenCalledOnce(); + }); + + it("validates health and container reachability through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((command: readonly string[]) => { + const usesDockerDesktop = + command[0] === "docker" && + command[1] === "run" && + command[2] === "--rm" && + command.includes(CONTAINER_REACHABILITY_IMAGE); + const endpoint = command.find((argument) => argument.startsWith("http://")); + return usesDockerDesktop && + (endpoint === "http://host.docker.internal:11434/api/tags" || + endpoint === "http://host.openshell.internal:11434/api/tags" || + endpoint === "http://host.openshell.internal:11435/api/tags") + ? JSON.stringify({ models: [] }) + : ""; + }); + + const result = validateLocalProvider( + "ollama-local", + capture, + () => {}, + () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + ); + expect(result).toEqual({ ok: true }); + const endpoints = capture.mock.calls.map(([command]) => + command.find((argument: string) => argument.startsWith("http://")), + ); + expect(endpoints).toContain("http://host.docker.internal:11434/api/tags"); + expect(endpoints).toContain("http://host.openshell.internal:11434/api/tags"); + }); + + it("rejects Windows-host health when the container route is unreachable", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = vi.fn((command: readonly string[]) => + command.some((argument) => argument === "http://host.docker.internal:11434/api/tags") + ? JSON.stringify({ models: [] }) + : "", + ); + + const result = validateLocalProvider( + "ollama-local", + capture, + () => {}, + () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + ); + + expect(result).toMatchObject({ + ok: false, + message: expect.stringContaining("container reachability check failed"), + }); + const endpoints = capture.mock.calls.map(([command]) => + command.find((argument: string) => argument.startsWith("http://")), + ); + expect(endpoints).toContain("http://host.docker.internal:11434/api/tags"); + expect( + endpoints.some((endpoint) => endpoint?.startsWith("http://host.openshell.internal:")), + ).toBe(true); + }); + + it("checks the Hermes context window through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = respondsOnlyThroughDockerDesktop( + "/api/ps", + JSON.stringify({ + models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }], + }), + ); + const env: NodeJS.ProcessEnv = {}; + + expect( + applyOllamaRuntimeContextWindow("qwen3.5:9b", { + contextWindowFloor: 64_000, + env, + logger: { log: vi.fn(), warn: vi.fn() }, + runCaptureImpl: capture, + }), + ).toEqual({ ok: true }); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("checks model capability metadata through Docker Desktop (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = respondsOnlyThroughDockerDesktop( + "/api/show", + JSON.stringify({ capabilities: ["tools"] }), + ); + + expect(probeOllamaModelCapabilities("qwen3.5:9b", capture)).toMatchObject({ + source: "api", + supportsTools: true, + }); + expect(capture).toHaveBeenCalledOnce(); + }); + + it("isolates Docker credentials for Windows-host API requests", () => { + const cleanup = vi.fn(() => ({ ok: true as const })); + const capture = vi.fn((_command: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const isolatedCapture = createOllamaApiCapture(capture, OLLAMA_HOST_DOCKER_INTERNAL, () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })); + + expect(isolatedCapture(["curl", "-sf", "http://host.docker.internal:11434/api/tags"])).toBe( + JSON.stringify({ models: [] }), + ); + expect(capture).toHaveBeenCalledWith( + expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]), + { env: { DOCKER_CONFIG: "/tmp/credential-free-docker" } }, + ); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("runs Windows-host warm-up with an isolated Docker client", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const cleanup = vi.fn(() => ({ ok: true as const })); + const run = vi.fn(); + + runOllamaWarmup("qwen3.5:9b", run, () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })); + + expect(run).toHaveBeenCalledWith( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/generate", + ]), + { + ignoreError: true, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }, + ); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const capture = respondsOnlyThroughDockerDesktop( + "/api/ps", + JSON.stringify({ models: [{ name: "qwen3.5:9b", context_length: "invalid" }] }), + ); + + const result = applyOllamaRuntimeContextWindow("qwen3.5:9b", { + contextWindowFloor: 64_000, + env: {}, + logger: { log: vi.fn(), warn: vi.fn() }, + runCaptureImpl: capture, + }); + + expect(result).toMatchObject({ + ok: false, + message: expect.stringContaining("cannot verify the required 64000-token window"), + }); + expect(capture).toHaveBeenCalledWith( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/ps", + ]), + expect.any(Object), + ); + }); +}); diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 5233cdcef62..79f0b392e8c 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -176,10 +176,11 @@ describe("local inference helpers", () => { "http://127.0.0.1:11434/api/tags", "http://host.docker.internal:11434/api/tags", ]); - expect(commands.map((command) => command.slice(2, 6))).toEqual([ - ["--connect-timeout", "3", "--max-time", "5"], - ["--connect-timeout", "3", "--max-time", "5"], - ]); + expect(commands.map((command) => command[0])).toEqual(["curl", "docker"]); + expect(commands[0]).toEqual(expect.arrayContaining(["--connect-timeout", "3", "--max-time", "5"])); + expect(commands[1]).toEqual( + expect.arrayContaining([CONTAINER_REACHABILITY_IMAGE, "--connect-timeout", "3", "--max-time", "5"]), + ); }); it("returns the expected base URL for vllm-local", () => { @@ -334,7 +335,7 @@ describe("local inference helpers", () => { expect(result.message).toMatch(/not an Ollama networking failure/); expect(result.message).not.toMatch(/Docker container reachability check failed/); expect(result.message).not.toMatch(/sandbox uses a different network path/); - expect(result.diagnostic).toMatch(/DOCKER_CONFIG=\$\(mktemp -d\) docker pull curlimages\/curl/); + expect(result.diagnostic).toContain(CONTAINER_REACHABILITY_IMAGE); expect(result.diagnostic).toMatch(/credential helper/); expect(result.diagnostic).toMatch(/onboard --resume/); }); @@ -353,7 +354,7 @@ describe("local inference helpers", () => { expect(result.ok).toBe(false); expect(result.message).toMatch(/Docker image-pull failure/); expect(result.message).toMatch(/not a vLLM networking failure/); - expect(result.diagnostic).toMatch(/docker pull curlimages\/curl/); + expect(result.diagnostic).toContain(`docker pull ${CONTAINER_REACHABILITY_IMAGE}`); }); it("keeps the runtime-failure report when the probe image is present locally (#9308)", () => { @@ -1215,7 +1216,8 @@ describe("local inference helpers", () => { it("builds a background warmup command for ollama models", () => { const command = getOllamaWarmupCommand("nemotron-3-nano:30b"); expect(command).toEqual(expect.arrayContaining(["bash", "-c"])); - expect(command[2]).toMatch(/^nohup curl -s http:\/\/127.0.0.1:11434\/api\/generate /); + expect(command[2]).toContain("'--connect-timeout' '10' '--max-time' '120'"); + expect(command[2]).toContain("http://127.0.0.1:11434/api/generate"); expect(command[2]).toMatch(/"model":"nemotron-3-nano:30b"/); expect(command[2]).toMatch(/"keep_alive":"15m"/); }); @@ -1226,7 +1228,7 @@ describe("local inference helpers", () => { const probe1 = getOllamaProbeCommand("qwen3.5:9b", 30, "5m"); expect(probe1).toContain("--max-time"); expect(probe1).toContain("30"); - const payload1 = probe1[probe1.length - 1]; + const payload1 = probe1[probe1.indexOf("-d") + 1]; expect(payload1).toMatch(/"keep_alive":"5m"/); }); @@ -1237,7 +1239,7 @@ describe("local inference helpers", () => { expect(command).toContain("--max-time"); expect(command).toContain("120"); expect(command).toContain("http://127.0.0.1:11434/api/generate"); - const payload = command[command.length - 1]; + const payload = command[command.indexOf("-d") + 1]; expect(payload).toMatch(/"model":"nemotron-3-nano:30b"/); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index bc8cb325fab..29c2563fb72 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -27,13 +27,23 @@ import { OLLAMA_PORT, OLLAMA_PROXY_PORT, VLLM_PORT } from "../core/ports"; import { retryUntil } from "../core/retry"; import { sleepSeconds } from "../core/wait"; import { containerCanReachHostLoopback, isWsl, type WslDetectionOptions } from "../platform"; -import { type CaptureResult, runCapture, runCaptureEx, shellQuote } from "../runner"; +import { type CaptureResult, run, runCapture, runCaptureEx, shellQuote } from "../runner"; import { buildSubprocessEnv } from "../subprocess-env"; -import { resolveSharedLocalAdapterStateRoot } from "./local-adapter-lifecycle"; +import { + isLocalOllamaRouteOwner, + OLLAMA_HOST_DOCKER_INTERNAL, + OLLAMA_LOCALHOST, + readLocalAdapterJsonFile, + removeLocalAdapterFile, + resolveSharedLocalAdapterStateRoot, + type OllamaRouteHolder, + writeLocalAdapterJsonFile, +} from "./local-adapter-lifecycle"; import { detectNvidiaPlatform } from "./nim"; import { anyRegistryModelFits, + DEFAULT_OLLAMA_MODEL_TAG, effectiveGpuMemoryMB, fittableOllamaModelTags, largestFittableOllamaModelTag, @@ -107,7 +117,7 @@ function assertRegistryTag(tag: string): string { } export const SMALL_OLLAMA_MODEL = SMALLEST_OLLAMA_MODEL_TAG; -export const DEFAULT_OLLAMA_MODEL = assertRegistryTag("nemotron-3-nano:30b"); +export const DEFAULT_OLLAMA_MODEL = assertRegistryTag(DEFAULT_OLLAMA_MODEL_TAG); export const QWEN3_6_OLLAMA_MODEL = assertRegistryTag("qwen3.6:35b"); export type RunCaptureFn = ( @@ -123,13 +133,16 @@ export { MIN_OLLAMA_VERSION, } from "./ollama-version"; -export type RunCaptureExFn = (cmd: string[]) => CaptureResult; +export type RunCaptureExFn = (cmd: string[], opts?: { env?: NodeJS.ProcessEnv }) => CaptureResult; // Hosts that local-provider discovery may try when probing Ollama. The Windows // onboarding path separately checks host.docker.internal from Docker Desktop's // network context because the alias may not resolve from the WSL host. -export const OLLAMA_LOCALHOST = "127.0.0.1"; -export const OLLAMA_HOST_DOCKER_INTERNAL = "host.docker.internal"; +export { + isLocalOllamaRouteOwner, + OLLAMA_HOST_DOCKER_INTERNAL, + OLLAMA_LOCALHOST, +} from "./local-adapter-lifecycle"; /** Build the credential-free Docker Desktop probe for Windows-host Ollama. */ export function getWindowsHostOllamaDockerReachabilityArgs(): string[] { @@ -147,6 +160,22 @@ export function getWindowsHostOllamaDockerReachabilityArgs(): string[] { } let _resolvedOllamaHost: string | null = null; +const OLLAMA_HOST_RECEIPT_NAME = "ollama-host.json"; + +type OllamaHostReceipt = { + readonly schemaVersion: 1; + readonly host: typeof OLLAMA_LOCALHOST | typeof OLLAMA_HOST_DOCKER_INTERNAL; +}; + +function isSupportedOllamaHost( + host: unknown, +): host is typeof OLLAMA_LOCALHOST | typeof OLLAMA_HOST_DOCKER_INTERNAL { + return host === OLLAMA_LOCALHOST || host === OLLAMA_HOST_DOCKER_INTERNAL; +} + +function ollamaHostReceiptPath(stateRoot: string): string { + return nodePath.join(stateRoot, OLLAMA_HOST_RECEIPT_NAME); +} function ollamaCandidateHosts(wslDetection: WslDetectionOptions = {}): string[] { return isWsl(wslDetection) ? [OLLAMA_LOCALHOST, OLLAMA_HOST_DOCKER_INTERNAL] : [OLLAMA_LOCALHOST]; @@ -161,15 +190,21 @@ function ollamaCandidateHosts(wslDetection: WslDetectionOptions = {}): string[] export function findReachableOllamaHost( runCaptureImpl?: RunCaptureFn, wslDetection: WslDetectionOptions = {}, + stateRoot: string = resolveSharedLocalAdapterStateRoot(), ): string | null { if (_resolvedOllamaHost !== null) return _resolvedOllamaHost; + const persistedHost = loadPersistedOllamaHost(stateRoot); const capture = runCaptureImpl ?? runCapture; - for (const host of ollamaCandidateHosts(wslDetection)) { + const candidates = [ + ...(persistedHost ? [persistedHost] : []), + ...ollamaCandidateHosts(wslDetection).filter((host) => host !== persistedHost), + ]; + for (const host of candidates) { // Explicit timeouts: a blackholed host (e.g., firewalled host.docker.internal) // would otherwise stall the synchronous onboard probe for the OS connect // timeout (~75-130s on Linux). Matches the convention used in // getLocalProviderHealthStatus probes. - const result = capture( + const result = createOllamaApiCapture(capture, host)( [ "curl", "-sf", @@ -185,6 +220,7 @@ export function findReachableOllamaHost( _resolvedOllamaHost = host; return host; } + if (host === persistedHost) clearPersistedOllamaHost(stateRoot); } return null; } @@ -195,6 +231,152 @@ export function getResolvedOllamaHost(): string { return _resolvedOllamaHost ?? OLLAMA_LOCALHOST; } +/** + * Persist the accepted host-global Ollama route for later CLI processes. + * `ollama-local` is one gateway provider backed by one host auth proxy, so all + * sandboxes using that provider share the same daemon target. Discovery probes + * this receipt first and changes it only after the recorded target is stale. + */ +export function persistResolvedOllamaHost( + host: string = getResolvedOllamaHost(), + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): () => void { + if (!isSupportedOllamaHost(host)) { + throw new Error(`Refusing to persist unexpected Ollama host: ${host}`); + } + const receiptPath = ollamaHostReceiptPath(stateRoot); + const previousHost = loadPersistedOllamaHost(stateRoot); + writeLocalAdapterJsonFile(receiptPath, { + schemaVersion: 1, + host, + } satisfies OllamaHostReceipt); + return () => { + if (previousHost) { + writeLocalAdapterJsonFile(receiptPath, { + schemaVersion: 1, + host: previousHost, + } satisfies OllamaHostReceipt); + } else { + removeLocalAdapterFile(receiptPath); + } + }; +} + +/** Read only the two fixed local Ollama routes NemoClaw can establish. */ +export function loadPersistedOllamaHost( + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): typeof OLLAMA_LOCALHOST | typeof OLLAMA_HOST_DOCKER_INTERNAL | null { + const receipt = readLocalAdapterJsonFile(ollamaHostReceiptPath(stateRoot)); + return receipt?.schemaVersion === 1 && isSupportedOllamaHost(receipt.host) ? receipt.host : null; +} + +export function clearPersistedOllamaHost( + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + removeLocalAdapterFile(ollamaHostReceiptPath(stateRoot)); + _resolvedOllamaHost = null; +} + +export function clearPersistedOllamaHostIfUnused( + routes: readonly OllamaRouteHolder[], + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): boolean { + const selectedHost = loadPersistedOllamaHost(stateRoot); + if (routes.some((route) => isLocalOllamaRouteOwner(route, selectedHost))) return false; + clearPersistedOllamaHost(stateRoot); + return true; +} + +/** Keep Windows-host Ollama requests in Docker Desktop's verified network context. */ +export function getOllamaApiCommand( + curlArgs: readonly string[], + host: string = getResolvedOllamaHost(), +): string[] { + return host === OLLAMA_HOST_DOCKER_INTERNAL + ? ["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE, ...curlArgs] + : ["curl", ...curlArgs]; +} + +export type PreparedOllamaApiExecution = { + readonly command: string[]; + readonly env?: NodeJS.ProcessEnv; + cleanup(): void; +}; + +/** Own command translation and Docker-client isolation for one Ollama API process. */ +export function prepareOllamaApiExecution( + command: readonly string[], + host: string = getResolvedOllamaHost(), + options: { + env?: NodeJS.ProcessEnv; + operation?: string; + prepareDockerEnvironment?: PrepareDockerEnvironmentFn; + } = {}, +): PreparedOllamaApiExecution { + const [executable, ...args] = command; + const translated = executable === "curl" ? getOllamaApiCommand(args, host) : [...command]; + if (translated[0] !== "docker") { + return { command: translated, env: options.env, cleanup: () => {} }; + } + const prepared = (options.prepareDockerEnvironment ?? prepareIsolatedDockerEnvironment)(); + let cleaned = false; + return { + command: translated, + env: mergeIsolatedDockerClientEnv(options.env ?? {}, prepared), + cleanup: () => { + if (cleaned) return; + cleaned = true; + warnIfDockerBuildEnvironmentCleanupFailed( + prepared.cleanup(), + options.operation ?? "Windows-host Ollama API request", + ); + }, + }; +} + +export function createOllamaApiCapture( + runCaptureImpl?: RunCaptureFn, + host: string = getResolvedOllamaHost(), + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, +): RunCaptureFn { + const capture = runCaptureImpl ?? runCapture; + return (command, options) => { + const execution = prepareOllamaApiExecution(command, host, { + env: options?.env, + prepareDockerEnvironment, + }); + try { + return capture(execution.command, { + ...options, + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } + }; +} + +export function createOllamaApiCaptureEx( + runCaptureExImpl: RunCaptureExFn = runCaptureEx, + host: string = getResolvedOllamaHost(), + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, +): RunCaptureExFn { + return (command, options) => { + const execution = prepareOllamaApiExecution(command, host, { + env: options?.env, + prepareDockerEnvironment, + }); + try { + return runCaptureExImpl(execution.command, { + ...options, + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } + }; +} + export function resetOllamaHostCache(): void { _resolvedOllamaHost = null; } @@ -285,6 +467,9 @@ export interface LocalProviderHealthProbeOptions { /** Configured runtime model that must be present in the provider inventory. */ model?: string | null; runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; + /** Executes the translated Windows-host Docker probe. Injectable for transport tests. */ + ollamaRunCaptureExImpl?: RunCaptureExFn; + findReachableOllamaHostImpl?: () => string | null; /** * Lets callers that perform their own Ollama auth-proxy check avoid the * legacy inline proxy subprobe. The inline subprobe is retained for status @@ -322,6 +507,30 @@ function runLocalCurlProbe(argv: string[], opts: CurlProbeOptions = {}): CurlPro return runCurlProbe(argv, { ...opts, env: buildSubprocessEnv(), replaceEnv: true }); } +function runOllamaLocalCurlProbe( + argv: string[], + host: string, + runCaptureExImpl: RunCaptureExFn = runCaptureEx, +): CurlProbeResult { + const command = getOllamaApiCommand(buildValidatedCurlCommandArgs(["-f", ...argv]), host); + const result = createOllamaApiCaptureEx(runCaptureExImpl, host)(command); + const ok = result.exitCode === 0; + const stderr = String(result.stderr ?? ""); + return { + ok, + httpStatus: ok ? 200 : 0, + curlStatus: result.exitCode ?? 1, + body: result.stdout, + stderr, + message: ok + ? "HTTP 200" + : (stderr || result.stdout || `Docker Ollama probe exited ${String(result.exitCode)}`) + .replace(/\s+/g, " ") + .trim() + .slice(0, 300), + }; +} + export interface VllmModelsProbeOptions { runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; } @@ -772,7 +981,9 @@ export function getLocalProviderHealthCheck(provider: string): string[] | null { endpoint, ]; } - return endpoint ? ["curl", ...buildValidatedCurlCommandArgs(["-sf", endpoint])] : null; + if (!endpoint) return null; + const curlArgs = buildValidatedCurlCommandArgs(["-sf", endpoint]); + return provider === "ollama-local" ? getOllamaApiCommand(curlArgs) : ["curl", ...curlArgs]; } /** @@ -792,9 +1003,10 @@ export function isLocalProviderHostHealthy( const command = getLocalProviderHealthCheck(provider); if (!command) return false; const capture = runCaptureImpl ?? runCapture; + const hostCapture = provider === "ollama-local" ? createOllamaApiCapture(capture) : capture; return isLocalProviderProbeOutputHealthy( command.at(-1) ?? "", - capture(command, { ignoreError: true }), + hostCapture(command, { ignoreError: true }), ); } @@ -934,6 +1146,9 @@ export function probeLocalProviderHealth( ): LocalProviderHealthStatus | null { const providerLabel = getLocalProviderLabel(provider); if (!providerLabel) return null; + if (provider === "ollama-local") { + (options.findReachableOllamaHostImpl ?? findReachableOllamaHost)(); + } let managedState: ManagedVllmProviderState = { kind: "absent" }; if (provider === "vllm-local") { @@ -979,7 +1194,14 @@ export function probeLocalProviderHealth( : getLocalProviderHealthEndpoint(provider); if (!endpoint) return null; - const runCurlProbeImpl = options.runCurlProbeImpl ?? runLocalCurlProbe; + const resolvedOllamaHost = + provider === "ollama-local" ? getResolvedOllamaHost() : OLLAMA_LOCALHOST; + const runCurlProbeImpl = + options.runCurlProbeImpl ?? + (provider === "ollama-local" && resolvedOllamaHost === OLLAMA_HOST_DOCKER_INTERNAL + ? (argv: string[]) => + runOllamaLocalCurlProbe(argv, resolvedOllamaHost, options.ollamaRunCaptureExImpl) + : runLocalCurlProbe); let result: CurlProbeResult; if (managedBinding) { result = probeVllmModels(managedValidationBaseUrl!, managedBinding.apiKey, { @@ -1125,7 +1347,10 @@ export function getLocalProviderContainerReachabilityCheck( // requires a Bearer token on every endpoint (#3338) and the ephemeral // probe container doesn't carry one, but the goal here is connectivity // not authorisation. - const containerPort = getOllamaContainerPort(); + const containerPort = + getResolvedOllamaHost() === OLLAMA_HOST_DOCKER_INTERNAL + ? OLLAMA_PORT + : getOllamaContainerPort(); if (responseMode === "body" && containerPort !== OLLAMA_PORT) return null; return [ "docker", @@ -1152,7 +1377,7 @@ export function probeOllamaEndpointInventory( host: string, runCaptureImpl?: RunCaptureFn, ): string[] | null { - const capture = runCaptureImpl ?? runCapture; + const capture = createOllamaApiCapture(runCaptureImpl, host); const body = capture( [ "curl", @@ -1263,7 +1488,8 @@ export function validateLocalProvider( return { ok: true }; } - const output = capture(command, { ignoreError: true }); + const hostCapture = provider === "ollama-local" ? createOllamaApiCapture(capture) : capture; + const output = hostCapture(command, { ignoreError: true }); if (!isLocalProviderProbeOutputHealthy(command.at(-1) ?? "", output)) { switch (provider) { case "vllm-local": @@ -1528,7 +1754,11 @@ export function probeOllamaRuntimeModelStatus( model: string, runCaptureImpl?: RunCaptureFn, ): OllamaRuntimeModelStatus { - return probeOllamaRuntimeModelStatusWithHost(model, getResolvedOllamaHost, runCaptureImpl); + return probeOllamaRuntimeModelStatusWithHost( + model, + getResolvedOllamaHost, + createOllamaApiCapture(runCaptureImpl), + ); } export function resolveOllamaRuntimeContextWindow( @@ -1540,7 +1770,7 @@ export function resolveOllamaRuntimeContextWindow( model, currentContextWindow, getResolvedOllamaHost, - runCaptureImpl, + createOllamaApiCapture(runCaptureImpl), ); } @@ -1549,9 +1779,15 @@ export { resetOllamaRuntimeContextWindowAutoState }; /** Apply Ollama runtime context-window adoption using the resolved local host. */ export function applyOllamaRuntimeContextWindow( selectedModel: string, - options: Pick = {}, + options: Pick< + ApplyOllamaRuntimeContextWindowOptions, + "contextWindowFloor" | "env" | "logger" | "runCaptureImpl" + > = {}, ): ApplyOllamaRuntimeContextWindowResult { - return applyOllamaRuntimeContextWindowWithHost(selectedModel, getResolvedOllamaHost, options); + return applyOllamaRuntimeContextWindowWithHost(selectedModel, getResolvedOllamaHost, { + ...options, + runCaptureImpl: createOllamaApiCapture(options.runCaptureImpl), + }); } export function applyVllmRuntimeContextWindow( @@ -1578,24 +1814,24 @@ export function getOllamaModelOptions( sleepMilliseconds: (milliseconds: number) => void = (milliseconds) => sleepSeconds(milliseconds / 1_000), ): string[] { - const capture = runCaptureImpl ?? runCapture; const host = getResolvedOllamaHost(); + const capture = createOllamaApiCapture(runCaptureImpl, host); const modelDiscoveryRetryDelaysMs = [500, 1_000] as const; + // Docker Desktop owns Windows-host reachability because host.docker.internal + // may not resolve from WSL. Keep model discovery on the verified transport. + const tagsCommand = [ + "curl", + ...buildValidatedCurlCommandArgs([ + "-sf", + "--connect-timeout", + "3", + "--max-time", + "5", + `http://${host}:${OLLAMA_PORT}/api/tags`, + ]), + ]; const readTags = () => { - const tagsOutput = capture( - [ - "curl", - ...buildValidatedCurlCommandArgs([ - "-sf", - "--connect-timeout", - "3", - "--max-time", - "5", - `http://${host}:${OLLAMA_PORT}/api/tags`, - ]), - ], - { ignoreError: true }, - ); + const tagsOutput = capture(tagsCommand, { ignoreError: true }); return parseOllamaModelInventory(String(tagsOutput || "")); }; // The daemon can become unreachable after the earlier readiness check. @@ -1720,7 +1956,7 @@ export function selectDefaultOllamaModel( return OLLAMA_MODEL_REGISTRY.find((entry) => pool.includes(entry.tag))?.tag ?? pool[0]; } -export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string[] { +export function getOllamaWarmupRequestCommand(model: string, keepAlive = "15m"): string[] { const payload = JSON.stringify({ model, prompt: "Hello, reply in less than 5 words", @@ -1729,6 +1965,25 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string options: { num_predict: 16 }, }); const host = getResolvedOllamaHost(); + return getOllamaApiCommand( + [ + "-s", + "--connect-timeout", + "10", + "--max-time", + "120", + `http://${host}:${OLLAMA_PORT}/api/generate`, + "-H", + "Content-Type: application/json", + "-d", + payload, + ], + host, + ); +} + +export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string[] { + const command = getOllamaWarmupRequestCommand(model, keepAlive); // backgrounding (nohup ... &) and output redirection require a shell wrapper. // The payload is safe: model name is JSON-serialized (escaping all special // chars) then shellQuote'd (single-quoted), so injection through model @@ -1736,10 +1991,36 @@ export function getOllamaWarmupCommand(model: string, keepAlive = "15m"): string return [ "bash", "-c", - `nohup curl -s http://${host}:${OLLAMA_PORT}/api/generate -H 'Content-Type: application/json' -d ${shellQuote(payload)} >/dev/null 2>&1 &`, + `nohup ${command.map((arg) => shellQuote(arg)).join(" ")} >/dev/null 2>&1 &`, ]; } +export function runOllamaWarmup( + model: string, + runImpl: ( + command: readonly string[], + options?: { ignoreError?: boolean; env?: NodeJS.ProcessEnv }, + ) => unknown = run, + prepareDockerEnvironment: PrepareDockerEnvironmentFn = prepareIsolatedDockerEnvironment, +): void { + const windowsHost = getResolvedOllamaHost() === OLLAMA_HOST_DOCKER_INTERNAL; + const command = windowsHost + ? getOllamaWarmupRequestCommand(model) + : getOllamaWarmupCommand(model); + const execution = prepareOllamaApiExecution(command, getResolvedOllamaHost(), { + prepareDockerEnvironment, + operation: `Windows-host Ollama warm-up for '${model}'`, + }); + try { + runImpl(execution.command, { + ignoreError: true, + ...(execution.env === undefined ? {} : { env: execution.env }), + }); + } finally { + execution.cleanup(); + } +} + export function getOllamaProbeCommand( model: string, timeoutSeconds = 120, @@ -1754,27 +2035,19 @@ export function getOllamaProbeCommand( }); const host = getResolvedOllamaHost(); const endpoint = `http://${host}:${OLLAMA_PORT}/api/generate`; - buildValidatedCurlCommandArgs([ - "-sS", - "--max-time", - String(timeoutSeconds), - "-H", - "Content-Type: application/json", - "-d", - payload, - endpoint, - ]); - return [ - "curl", - "-sS", - "--max-time", - String(timeoutSeconds), - endpoint, - "-H", - "Content-Type: application/json", - "-d", - payload, - ]; + return getOllamaApiCommand( + buildValidatedCurlCommandArgs([ + "-sS", + "--max-time", + String(timeoutSeconds), + "-H", + "Content-Type: application/json", + "-d", + payload, + endpoint, + ]), + host, + ); } export function validateOllamaModel( @@ -1785,7 +2058,7 @@ export function validateOllamaModel( options: { allowToolsIncompatible?: boolean } = {}, ): ValidationResult { const capture = runCaptureImpl ?? runCapture; - const captureEx = runCaptureExImpl ?? runCaptureEx; + const captureEx = createOllamaApiCaptureEx(runCaptureExImpl ?? runCaptureEx); const isSpark = isSparkImpl ?? (() => detectNvidiaPlatform() === "spark"); const sparkHost = isSpark(); const probeCmd = getOllamaProbeCommand(model); @@ -1934,7 +2207,11 @@ export function probeOllamaModelCapabilities( model: string, runCaptureImpl?: RunCaptureFn, ): OllamaCapabilities { - const metadata = fetchOllamaModelShowMetadata(model, getResolvedOllamaHost, runCaptureImpl); + const metadata = fetchOllamaModelShowMetadata( + model, + getResolvedOllamaHost, + createOllamaApiCapture(runCaptureImpl), + ); if (!metadata.ok) { return { source: "unknown", diff --git a/src/lib/inference/ollama-model-registry.ts b/src/lib/inference/ollama-model-registry.ts index 933c2a92156..2a334f7e242 100644 --- a/src/lib/inference/ollama-model-registry.ts +++ b/src/lib/inference/ollama-model-registry.ts @@ -26,6 +26,8 @@ import type { GpuInfo } from "./local"; +export const DEFAULT_OLLAMA_MODEL_TAG = "nemotron-3-nano:30b"; + export interface OllamaModelEntry { tag: string; requiredMemoryMB: number; diff --git a/src/lib/inference/ollama/model-ownership.test.ts b/src/lib/inference/ollama/model-ownership.test.ts index 84454236390..557df044639 100644 --- a/src/lib/inference/ollama/model-ownership.test.ts +++ b/src/lib/inference/ollama/model-ownership.test.ts @@ -1,22 +1,65 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { + clearPendingOllamaModelCleanup, decideOllamaModelOwnership, exclusivelyHeldOllamaModel, + loadPendingOllamaModelCleanup, type OllamaModelHolder, + type OllamaModelRoute, + persistPendingOllamaModelCleanup, supersededOllamaModel, } from "./model-ownership"; +import { isLocalOllamaRouteOwner } from "./model-ownership"; function holder(overrides: Partial = {}): OllamaModelHolder { return { name: "test-box", provider: "ollama-local", model: "llama3", ...overrides }; } +function route(model: string, overrides: Partial = {}): OllamaModelRoute { + return { provider: "ollama-local", model, ...overrides }; +} + +describe("isLocalOllamaRouteOwner", () => { + it.each([["ollama-local"], ["ollama/qwen3-vl:4b"]])( + "recognizes the direct local provider %s", + (provider) => { + expect(isLocalOllamaRouteOwner({ provider }, "127.0.0.1")).toBe(true); + }, + ); + + it("recognizes a compatible endpoint at the selected local daemon", () => { + expect( + isLocalOllamaRouteOwner( + { + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + }, + "127.0.0.1", + ), + ).toBe(true); + }); + + it.each([ + ["a remote endpoint", "https://ollama.example.com:11434/v1"], + ["the other fixed host route", "http://host.docker.internal:11434/v1"], + ["a different port", "http://127.0.0.1:11435/v1"], + ])("excludes %s", (_label, endpointUrl) => { + expect( + isLocalOllamaRouteOwner({ provider: "compatible-endpoint", endpointUrl }, "127.0.0.1"), + ).toBe(false); + }); +}); + describe("supersededOllamaModel", () => { it("releases the previous model when a re-onboard moves to a different one (#9110)", () => { - expect(supersededOllamaModel(holder(), "qwen2.5:7b", [holder()])).toBe("llama3"); + expect(supersededOllamaModel(holder(), route("qwen2.5:7b"), [holder()])).toBe("llama3"); }); it.each([ @@ -25,41 +68,69 @@ describe("supersededOllamaModel", () => { ["an explicit latest tag on the previous model", "llama3:latest", "llama3"], ])("keeps the model when the next ref is %s (#9110)", (_label, previousModel, nextModel) => { const previous = holder({ model: previousModel }); - expect(supersededOllamaModel(previous, nextModel, [previous])).toBeNull(); + expect(supersededOllamaModel(previous, route(nextModel), [previous])).toBeNull(); }); it.each([["llama3"], ["llama3:latest"]])( "keeps a model an Ollama peer records as %s (#9110)", (peerModel) => { const peer = holder({ model: peerModel, name: "peer" }); - expect(supersededOllamaModel(holder(), "qwen2.5:7b", [holder(), peer])).toBeNull(); + expect(supersededOllamaModel(holder(), route("qwen2.5:7b"), [holder(), peer])).toBeNull(); }, ); it("releases the model when peers hold different ones (#9110)", () => { const peer = holder({ model: "llama3:8b", name: "peer" }); - expect(supersededOllamaModel(holder(), "qwen2.5:7b", [holder(), peer])).toBe("llama3"); + expect(supersededOllamaModel(holder(), route("qwen2.5:7b"), [holder(), peer])).toBe("llama3"); }); it.each([["nvidia-prod"], ["vllm-local"], [undefined]])( "does nothing when the previous provider is %s (#9110)", (provider) => { const previous = holder({ provider }); - expect(supersededOllamaModel(previous, "qwen2.5:7b", [previous])).toBeNull(); + expect(supersededOllamaModel(previous, route("qwen2.5:7b"), [previous])).toBeNull(); }, ); it("does nothing when the previous model is unrecorded (#9110)", () => { const previous = holder({ model: undefined }); - expect(supersededOllamaModel(previous, "qwen2.5:7b", [previous])).toBeNull(); + expect(supersededOllamaModel(previous, route("qwen2.5:7b"), [previous])).toBeNull(); }); it.each([[""], [" "]])("does nothing when the next model is %j (#9110)", (nextModel) => { - expect(supersededOllamaModel(holder(), nextModel, [holder()])).toBeNull(); + expect(supersededOllamaModel(holder(), route(nextModel), [holder()])).toBeNull(); }); it("does nothing when there is no previous entry (#9110)", () => { - expect(supersededOllamaModel(null, "qwen2.5:7b", [])).toBeNull(); + expect(supersededOllamaModel(null, route("qwen2.5:7b"), [])).toBeNull(); + }); + + it("keeps a model selected through a compatible endpoint at the same local daemon", () => { + expect( + supersededOllamaModel( + holder(), + route("llama3:latest", { + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + }), + [holder()], + "127.0.0.1", + ), + ).toBeNull(); + }); + + it("does not mistake a remote compatible endpoint for the local daemon", () => { + expect( + supersededOllamaModel( + holder(), + route("llama3", { + provider: "compatible-endpoint", + endpointUrl: "https://ollama.example.com:11434/v1", + }), + [holder()], + "127.0.0.1", + ), + ).toBe("llama3"); }); }); @@ -95,9 +166,11 @@ describe("decideOllamaModelOwnership", () => { it.each([[undefined], [""], [" "]])( "returns missing-model for registry model %j (#10074)", (model) => { - expect(decideOllamaModelOwnership(holder({ model }), [holder({ model })], new Set())).toEqual({ - kind: "missing-model", - }); + expect(decideOllamaModelOwnership(holder({ model }), [holder({ model })], new Set())).toEqual( + { + kind: "missing-model", + }, + ); }, ); @@ -113,6 +186,28 @@ describe("decideOllamaModelOwnership", () => { ), ).toEqual({ kind: "exclusive", model: "llama3", stalePeers: [] }); }); + + it("protects a matching compatible endpoint at the same local daemon", () => { + const activePeer = holder({ + name: "compatible-peer", + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:11434/v1", + }); + + expect( + decideOllamaModelOwnership( + holder(), + [holder(), activePeer], + new Set(["compatible-peer"]), + "127.0.0.1", + ), + ).toEqual({ + kind: "shared-active", + model: "llama3", + activePeers: ["compatible-peer"], + stalePeers: [], + }); + }); }); describe("exclusivelyHeldOllamaModel", () => { @@ -121,3 +216,30 @@ describe("exclusivelyHeldOllamaModel", () => { expect(exclusivelyHeldOllamaModel(holder(), [holder(), peer])).toBe("llama3"); }); }); + +describe("pending Ollama model cleanup", () => { + it("persists exact sandbox-scoped models until verified release", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-pending-ollama-cleanup-")); + try { + persistPendingOllamaModelCleanup("test-box", ["llama3", "llama3:latest"], stateRoot); + persistPendingOllamaModelCleanup("test-box", ["qwen3.5:9b"], stateRoot); + + expect(loadPendingOllamaModelCleanup("test-box", stateRoot)).toEqual([ + "llama3", + "qwen3.5:9b", + ]); + clearPendingOllamaModelCleanup("test-box", ["llama3:latest"], stateRoot); + expect(loadPendingOllamaModelCleanup("test-box", stateRoot)).toEqual(["qwen3.5:9b"]); + clearPendingOllamaModelCleanup("test-box", undefined, stateRoot); + expect(loadPendingOllamaModelCleanup("test-box", stateRoot)).toEqual([]); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("rejects an unsafe sandbox name before state access", () => { + expect(() => loadPendingOllamaModelCleanup("../peer")).toThrow( + "Invalid sandbox name for pending Ollama cleanup", + ); + }); +}); diff --git a/src/lib/inference/ollama/model-ownership.ts b/src/lib/inference/ollama/model-ownership.ts index 392edfbb21f..84a3d69e5f6 100644 --- a/src/lib/inference/ollama/model-ownership.ts +++ b/src/lib/inference/ollama/model-ownership.ts @@ -1,11 +1,101 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import path from "node:path"; + import type { SandboxEntry } from "../../state/registry"; +import { + isLocalOllamaRouteOwner, + readLocalAdapterJsonFile, + removeLocalAdapterFile, + resolveSharedLocalAdapterStateRoot, + type OllamaHostRoute, + writeLocalAdapterJsonFile, +} from "../local-adapter-lifecycle"; import { ollamaModelRefsMatch } from "./model-discovery"; +export { isLocalOllamaRouteOwner } from "../local-adapter-lifecycle"; +export type { OllamaHostRoute, OllamaRouteHolder } from "../local-adapter-lifecycle"; + +const PENDING_CLEANUP_DIRECTORY = "ollama-pending-model-cleanup"; +const SAFE_SANDBOX_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; + +function pendingCleanupPath(sandboxName: string, stateRoot: string): string { + if (!SAFE_SANDBOX_NAME.test(sandboxName)) { + throw new Error(`Invalid sandbox name for pending Ollama cleanup: ${sandboxName}`); + } + return path.join(stateRoot, PENDING_CLEANUP_DIRECTORY, `${sandboxName}.json`); +} + +function validPendingModel(value: unknown): value is string { + return ( + typeof value === "string" && + value === value.trim() && + value.length > 0 && + Buffer.byteLength(value, "utf8") <= 512 && + !/[\u0000\r\n]/u.test(value) + ); +} + +export function loadPendingOllamaModelCleanup( + sandboxName: string, + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): readonly string[] { + const record = readLocalAdapterJsonFile(pendingCleanupPath(sandboxName, stateRoot)); + return record?.schemaVersion === 1 && Array.isArray(record.models) + ? record.models.filter(validPendingModel) + : []; +} + +export function persistPendingOllamaModelCleanup( + sandboxName: string, + models: readonly string[], + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + const pending = [...loadPendingOllamaModelCleanup(sandboxName, stateRoot)]; + for (const model of models) { + const normalized = model.trim(); + if (!validPendingModel(normalized)) continue; + if (!pending.some((existing) => ollamaModelRefsMatch(existing, normalized))) { + pending.push(normalized); + } + } + if (pending.length === 0) return; + writeLocalAdapterJsonFile(pendingCleanupPath(sandboxName, stateRoot), { + schemaVersion: 1, + sandboxName, + models: pending, + }); +} + +export function clearPendingOllamaModelCleanup( + sandboxName: string, + releasedModels?: readonly string[], + stateRoot: string = resolveSharedLocalAdapterStateRoot(), +): void { + const receiptPath = pendingCleanupPath(sandboxName, stateRoot); + if (!releasedModels) { + removeLocalAdapterFile(receiptPath); + return; + } + const remaining = loadPendingOllamaModelCleanup(sandboxName, stateRoot).filter( + (pending) => !releasedModels.some((released) => ollamaModelRefsMatch(pending, released)), + ); + if (remaining.length === 0) { + removeLocalAdapterFile(receiptPath); + return; + } + writeLocalAdapterJsonFile(receiptPath, { + schemaVersion: 1, + sandboxName, + models: remaining, + }); +} + /** The registry fields an Ollama GPU-release decision reads. */ -export type OllamaModelHolder = Pick; +export type OllamaModelHolder = Pick; + +export type OllamaModelRoute = Pick; export type OllamaModelOwnershipDecision = | { readonly kind: "missing-model" } @@ -32,13 +122,14 @@ export type OllamaModelOwnershipDecision = export function matchingOllamaModelPeers( sandbox: OllamaModelHolder, peers: readonly T[], + selectedHost: OllamaHostRoute | null = null, ): T[] { const model = sandbox.model?.trim(); if (!model) return []; return peers.filter( (peer) => peer.name !== sandbox.name && - !!peer.provider?.includes("ollama") && + isLocalOllamaRouteOwner(peer, selectedHost) && !!peer.model && ollamaModelRefsMatch(peer.model, model), ); @@ -48,11 +139,12 @@ export function decideOllamaModelOwnership( sandbox: OllamaModelHolder, peers: readonly OllamaModelHolder[], activeSandboxNames: ReadonlySet, + selectedHost: OllamaHostRoute | null = null, ): OllamaModelOwnershipDecision { const model = sandbox.model?.trim(); if (!model) return { kind: "missing-model" }; - const matchingPeers = matchingOllamaModelPeers(sandbox, peers); + const matchingPeers = matchingOllamaModelPeers(sandbox, peers, selectedHost); const activePeers = matchingPeers .filter((peer) => activeSandboxNames.has(peer.name)) .map((peer) => peer.name) @@ -74,11 +166,13 @@ export function decideOllamaModelOwnership( export function exclusivelyHeldOllamaModel( sandbox: OllamaModelHolder, peers: readonly OllamaModelHolder[], + selectedHost: OllamaHostRoute | null = null, ): string | null { const decision = decideOllamaModelOwnership( sandbox, peers, new Set(peers.map((peer) => peer.name)), + selectedHost, ); return decision.kind === "exclusive" ? decision.model : null; } @@ -97,13 +191,16 @@ export function exclusivelyHeldOllamaModel( */ export function supersededOllamaModel( previous: OllamaModelHolder | null, - nextModel: string, + next: OllamaModelRoute, peers: readonly OllamaModelHolder[], + selectedHost: OllamaHostRoute | null = null, ): string | null { - if (!previous?.provider?.includes("ollama")) return null; - const next = nextModel?.trim(); - if (!next) return null; - const held = exclusivelyHeldOllamaModel(previous, peers); + if (!previous || !isLocalOllamaRouteOwner(previous, selectedHost)) return null; + const nextModel = next.model?.trim(); + if (!nextModel) return null; + const held = exclusivelyHeldOllamaModel(previous, peers, selectedHost); if (!held) return null; - return ollamaModelRefsMatch(held, next) ? null : held; + return isLocalOllamaRouteOwner(next, selectedHost) && ollamaModelRefsMatch(held, nextModel) + ? null + : held; } diff --git a/src/lib/inference/ollama/proxy.test.ts b/src/lib/inference/ollama/proxy.test.ts index 1e09c4b2f93..f6bd9d4daeb 100644 --- a/src/lib/inference/ollama/proxy.test.ts +++ b/src/lib/inference/ollama/proxy.test.ts @@ -32,7 +32,7 @@ function loadProxyWithMocks(setup: MockSetup): { const childProcess = require(CHILD_PROCESS_DIST) as typeof import("node:child_process"); const runner = require(RUNNER_DIST); const originalGetOllamaModelOptions = local.getOllamaModelOptions; - const originalGetOllamaWarmupCommand = local.getOllamaWarmupCommand; + const originalRunOllamaWarmup = local.runOllamaWarmup; const originalPrompt = creds.prompt; const originalProbeOllamaModelCapabilities = local.probeOllamaModelCapabilities; const originalRun = runner.run; @@ -68,9 +68,9 @@ function loadProxyWithMocks(setup: MockSetup): { capabilities: ["tools"], supportsTools: true, }); - local.getOllamaWarmupCommand = (model: string) => { + local.runOllamaWarmup = (model: string, runImpl: typeof runner.run) => { warmupModels.push(model); - return ["warmup", model]; + runImpl(["warmup", model], { ignoreError: true }); }; local.validateOllamaModel = (...args: unknown[]) => { validateCalls.push(args); @@ -95,7 +95,7 @@ function loadProxyWithMocks(setup: MockSetup): { restore() { delete require.cache[PROXY_DIST]; local.getOllamaModelOptions = originalGetOllamaModelOptions; - local.getOllamaWarmupCommand = originalGetOllamaWarmupCommand; + local.runOllamaWarmup = originalRunOllamaWarmup; creds.prompt = originalPrompt; local.probeOllamaModelCapabilities = originalProbeOllamaModelCapabilities; runner.run = originalRun; @@ -382,13 +382,17 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { host: string; hasLocalCli: boolean; httpCloseCode?: number; + isolatedDockerConfig?: string; }) { const local = require(LOCAL_DIST); const runner = require(RUNNER_DIST); const childProcess = require(CHILD_PROCESS_DIST) as typeof import("node:child_process"); const originalRunCapture = runner.runCapture; + const originalPrepareOllamaApiExecution = local.prepareOllamaApiExecution; const cliCommands: string[][] = []; const httpCommands: string[][] = []; + const httpEnvs: NodeJS.ProcessEnv[] = []; + let cleanupCalls = 0; runner.runCapture = () => (setup.hasLocalCli ? "/usr/bin/ollama" : ""); @@ -398,23 +402,42 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { cliCommands.push([String(file), ...(((args as string[]) ?? []) as string[]).map(String)]); return { status: 0, signal: null, output: [], pid: 1, stdout: "", stderr: "" } as never; }); - const spawn = vi.spyOn(childProcess, "spawn").mockImplementation((file: unknown, args) => { - httpCommands.push([String(file), ...(((args as string[]) ?? []) as string[]).map(String)]); - const child = new EventEmitter() as EventEmitter & { - stdout: PassThrough; - stderr: PassThrough; - }; - child.stdout = new PassThrough(); - child.stderr = new PassThrough(); - process.nextTick(() => { - const closeCode = setup.httpCloseCode ?? 0; - const output = closeCode === 0 ? '{"status":"success"}\n' : ""; - child.stdout.end(output, () => { - setImmediate(() => child.emit("close", closeCode)); + local.prepareOllamaApiExecution = ( + command: readonly string[], + host: string, + options: NonNullable[2]>, + ) => + originalPrepareOllamaApiExecution(command, host, { + ...options, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: setup.isolatedDockerConfig ?? "/tmp/test-docker-config" }, + isolatedCredentialConfig: true, + cleanup: () => { + cleanupCalls += 1; + return { ok: true }; + }, + }), + }); + const spawn = vi + .spyOn(childProcess, "spawn") + .mockImplementation((file: unknown, args, options) => { + httpCommands.push([String(file), ...(((args as string[]) ?? []) as string[]).map(String)]); + httpEnvs.push((options?.env ?? {}) as NodeJS.ProcessEnv); + const child = new EventEmitter() as EventEmitter & { + stdout: PassThrough; + stderr: PassThrough; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + process.nextTick(() => { + const closeCode = setup.httpCloseCode ?? 0; + const output = closeCode === 0 ? '{"status":"success"}\n' : ""; + child.stdout.end(output, () => { + setImmediate(() => child.emit("close", closeCode)); + }); }); + return child as never; }); - return child as never; - }); local.setResolvedOllamaHost(setup.host); delete require.cache[PROXY_DIST]; @@ -423,9 +446,14 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { proxy, cliCommands, httpCommands, + httpEnvs, + get cleanupCalls() { + return cleanupCalls; + }, restore() { delete require.cache[PROXY_DIST]; runner.runCapture = originalRunCapture; + local.prepareOllamaApiExecution = originalPrepareOllamaApiExecution; spawnSync.mockRestore(); spawn.mockRestore(); local.setResolvedOllamaHost(null); @@ -442,14 +470,37 @@ describe("pullOllamaModel CLI-vs-HTTP dispatch", () => { vi.restoreAllMocks(); }); - it("pulls over HTTP when the daemon resolves on the Windows host", async () => { + it("pulls through Docker when the daemon resolves on the Windows host (#10553)", async () => { vi.spyOn(console, "log").mockImplementation(() => {}); - active = loadProxyForDispatch({ host: "host.docker.internal", hasLocalCli: true }); + active = loadProxyForDispatch({ + host: "host.docker.internal", + hasLocalCli: true, + isolatedDockerConfig: "/tmp/credential-free-docker", + }); - await active.proxy.pullOllamaModel("qwen3.5:9b"); + const result = await active.proxy.pullOllamaModel("qwen3.5:9b"); - expect(active.httpCommands.map((command) => command[0])).toContain("curl"); + expect(result).toBe(true); + expect(active.httpCommands.map((command) => command[0])).toContain("docker"); + const request = active.httpCommands[0]; + expect(request).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + "-X", + "POST", + "Content-Type: application/json", + "http://host.docker.internal:11434/api/pull", + ]), + ); + expect(JSON.parse(request[request.indexOf("-d") + 1])).toEqual({ + model: "qwen3.5:9b", + stream: true, + }); expect(active.cliCommands.map((command) => command[0])).not.toContain("bash"); + expect(active.httpEnvs[0]?.DOCKER_CONFIG).toBe("/tmp/credential-free-docker"); + expect(active.cleanupCalls).toBe(1); }); it("pulls over HTTP when a loopback daemon has no local ollama binary (#7472)", async () => { diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 94354ab059a..411278ba747 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -22,19 +22,30 @@ const { redirectInheritedChildStdoutToStderr, }: typeof import("../../cli/stdout-guard") = require("../../cli/stdout-guard"); const { OLLAMA_PORT, OLLAMA_PROXY_PORT } = require("../../core/ports"); -const { isNonInteractiveEnv }: typeof import("../../core/non-interactive") = - require("../../core/non-interactive"); +const { + isNonInteractiveEnv, +}: typeof import("../../core/non-interactive") = require("../../core/non-interactive"); const { sleepMs, waitForPort } = require("../../core/wait"); -const { ensurePulledOllamaModel }: typeof import("./model-discovery") = - require("./model-discovery"); +const { + ensurePulledOllamaModel, +}: typeof import("./model-discovery") = require("./model-discovery"); const { ollamaModelRefsMatch }: typeof import("./model-discovery") = require("./model-discovery"); +const { + clearPendingOllamaModelCleanup, + isLocalOllamaRouteOwner, + loadPendingOllamaModelCleanup, +}: typeof import("./model-ownership") = require("./model-ownership"); const { getBootstrapOllamaModelOptions, + findReachableOllamaHost, getOllamaModelOptions, - getOllamaWarmupCommand, getResolvedOllamaHost, + loadPersistedOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL, + prepareOllamaApiExecution, probeOllamaModelCapabilities, + persistResolvedOllamaHost, + runOllamaWarmup, selectDefaultOllamaModel, validateOllamaModel, } = require("../local"); @@ -124,6 +135,11 @@ function withOllamaModelOwnershipLock(operation: () => T): T { return withMcpLifecycleLockSync(OLLAMA_MODEL_OWNERSHIP_LOCK, operation); } +/** Serialize async host-route publication with final ownership retirement. */ +function withOllamaModelOwnershipTransaction(operation: () => Promise | T): Promise { + return withMcpLifecycleLock(OLLAMA_MODEL_OWNERSHIP_LOCK, operation); +} + function withOllamaProxyLifecycleTransaction(operation: () => Promise | T): Promise { // Async setup steps can call the synchronous helpers below while retaining // this lock through the shared re-entrant lifecycle-lock context. @@ -534,7 +550,9 @@ function attemptStartOllamaAuthProxyWithTokenUnlocked( printProxyPortConflict(owners); } else { console.error(` Error: Ollama auth proxy exited during startup on :${OLLAMA_PROXY_PORT}.`); - console.error(" Containers will not be able to reach the inference endpoint without the proxy."); + console.error( + " Containers will not be able to reach the inference endpoint without the proxy.", + ); console.error(` Check the proxy port owner: lsof -ti :${OLLAMA_PROXY_PORT}`); } } @@ -1043,30 +1061,47 @@ function pullOllamaModelViaHttp(model: string): Promise { // The endpoint is restricted to the local Ollama hosts NemoClaw probes and // the model id is normalized before being serialized as JSON request data. - const proc = spawn( - "curl", - [ - "-sN", - "--connect-timeout", - "10", - "--max-time", - String(TIMEOUT_MS / 1000), - "-X", - "POST", - "-H", - "Content-Type: application/json", - "-d", - // codeql[js/file-access-to-http]: local-only Ollama API with a normalized model id. - body, - url, - ], - { + let execution; + let proc; + try { + execution = prepareOllamaApiExecution( + [ + "curl", + "-sN", + "--connect-timeout", + "10", + "--max-time", + String(TIMEOUT_MS / 1000), + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + // codeql[js/file-access-to-http]: local-only Ollama API with a normalized model id. + body, + url, + ], + host, + { + env: buildSubprocessEnv(), + operation: `Windows-host Ollama model pull for '${model}'`, + }, + ); + const [executable, ...args] = execution.command; + proc = spawn(executable, args, { stdio: ["ignore", "pipe", "pipe"], // #2616: inject NO_PROXY=localhost so the streamed pull against the // local Ollama daemon doesn't tunnel through the user's host proxy. - env: buildSubprocessEnv(), - }, - ); + env: execution.env, + }); + } catch (error) { + execution?.cleanup(); + console.error( + ` Docker request failed to start: ${error instanceof Error ? error.message : String(error)}`, + ); + resolve(false); + return; + } const readline = require("readline"); const rl = readline.createInterface({ input: proc.stdout }); @@ -1146,6 +1181,7 @@ function pullOllamaModelViaHttp(model: string): Promise { }); proc.on("error", (err: Error) => { + execution.cleanup(); finishLine(); console.error(` Pull failed to start: ${err.message}`); resolve(false); @@ -1155,6 +1191,7 @@ function pullOllamaModelViaHttp(model: string): Promise { // child's stdio streams are fully drained, ensuring readline has emitted // the final 'line' event for the trailing `success` JSON. proc.on("close", (code: number | null) => { + execution.cleanup(); finishLine(); if (sawError) { resolve(false); @@ -1164,7 +1201,9 @@ function pullOllamaModelViaHttp(model: string): Promise { // curl exit 28 covers both the connection timeout and the complete // request limit. Elapsed time distinguishes the operator actions. if (code === 28) { - console.error(httpPullTimeoutErrorHint(performance.now() - startedAtMs, TIMEOUT_MS, host)); + console.error( + httpPullTimeoutErrorHint(performance.now() - startedAtMs, TIMEOUT_MS, host), + ); } else { console.error(` Model pull exited with code ${String(code)} (network error).`); console.error(" Already-downloaded layers are kept; re-running the pull resumes them."); @@ -1311,7 +1350,7 @@ async function prepareOllamaModel( } console.log(` Loading Ollama model: ${model}`); - run(getOllamaWarmupCommand(model), { ignoreError: true }); + runOllamaWarmup(model, run); const allowToolsIncompatible = capCheck.allowToolsIncompatible === true; const result = validateOllamaModel(model, undefined, undefined, undefined, { allowToolsIncompatible, @@ -1357,9 +1396,11 @@ export type OllamaUnloadResult = { type OllamaUnloadOptions = { readonly getResolvedOllamaHost?: typeof getResolvedOllamaHost; + readonly ollamaHostStateRoot?: string; readonly maxAttempts?: number; readonly sleep?: (milliseconds: number) => void; readonly spawnSync?: typeof spawnSync; + readonly prepareOllamaApiExecution?: typeof prepareOllamaApiExecution; }; function boundedCurlError(result): string | undefined { @@ -1368,7 +1409,9 @@ function boundedCurlError(result): string | undefined { } function transientCurlFailure(status: number | null): boolean { - return status === 6 || status === 7 || status === 18 || status === 28 || status === 52 || status === 56; + return ( + status === 6 || status === 7 || status === 18 || status === 28 || status === 52 || status === 56 + ); } function defaultReleaseSleep(milliseconds: number): void { @@ -1379,19 +1422,28 @@ function defaultReleaseSleep(milliseconds: number): void { function discoverResidentOllamaModels( attempt: number, selectedModels: readonly string[] | null, + releaseHost: string, releaseEndpoint: string, spawnSyncImpl: typeof spawnSync, + prepareExecution: typeof prepareOllamaApiExecution, ): OllamaModelDiscoveryEvidence { const endpoint = `${releaseEndpoint}/api/ps`; let result; try { - result = spawnSyncImpl( - "curl", - ["-sS", "--fail-with-body", "--max-time", "3", endpoint], - // #2616: env-sanitize so an ambient HTTP proxy cannot intercept the - // loopback-only Ollama ownership and release checks. - { encoding: "utf8", env: buildSubprocessEnv() }, + const execution = prepareExecution( + ["curl", "-sS", "--fail-with-body", "--max-time", "3", endpoint], + releaseHost, + { + env: buildSubprocessEnv(), + operation: "Ollama resident-model discovery", + }, ); + const [command, ...args] = execution.command; + try { + result = spawnSyncImpl(command, args, { encoding: "utf8", env: execution.env }); + } finally { + execution.cleanup(); + } } catch (error) { return { attempt, @@ -1471,20 +1523,48 @@ function unloadOllamaModels( onlyModels?: readonly string[], options: OllamaUnloadOptions = {}, ): OllamaUnloadResult { - const releaseEndpoint = buildLocalOllamaEndpoint( - options.getResolvedOllamaHost ?? getResolvedOllamaHost, - ); + const requestedModels = onlyModels?.map((model) => model.trim()).filter(Boolean) ?? []; + let selectedModels: readonly string[] | null = onlyModels?.length ? requestedModels : null; + let releaseHost: string | null; + if (options.getResolvedOllamaHost) { + releaseHost = options.getResolvedOllamaHost(); + } else { + const persistedHost = loadPersistedOllamaHost(options.ollamaHostStateRoot); + releaseHost = + persistedHost ?? findReachableOllamaHost(undefined, {}, options.ollamaHostStateRoot); + if (releaseHost && !persistedHost) { + persistResolvedOllamaHost(releaseHost, options.ollamaHostStateRoot); + } + } + if (!releaseHost) { + return { + ok: false, + outcome: "discovery-failed", + endpoint: buildLocalOllamaEndpoint(), + selectedModels: selectedModels ?? [], + discoveries: [], + requests: [], + message: "No reachable local Ollama endpoint was found for cleanup", + }; + } + const releaseEndpoint = buildLocalOllamaEndpoint(() => releaseHost!); const spawnSyncImpl = options.spawnSync ?? spawnSync; + const prepareExecution = options.prepareOllamaApiExecution ?? prepareOllamaApiExecution; const sleepImpl = options.sleep ?? defaultReleaseSleep; const maxAttempts = Math.max(1, options.maxAttempts ?? OLLAMA_RELEASE_MAX_ATTEMPTS); - const requestedModels = onlyModels?.map((model) => model.trim()).filter(Boolean) ?? []; - let selectedModels: readonly string[] | null = onlyModels?.length ? requestedModels : null; const discoveries: OllamaModelDiscoveryEvidence[] = []; const requests: OllamaUnloadRequestEvidence[] = []; let lastMatchedModels: readonly string[] = []; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const discovery = discoverResidentOllamaModels(attempt, selectedModels, releaseEndpoint, spawnSyncImpl); + const discovery = discoverResidentOllamaModels( + attempt, + selectedModels, + releaseHost, + releaseEndpoint, + spawnSyncImpl, + prepareExecution, + ); discoveries.push(discovery); if (discovery.error) { if (attempt < maxAttempts && transientCurlFailure(discovery.status)) { @@ -1520,9 +1600,9 @@ function unloadOllamaModels( const endpoint = `${releaseEndpoint}/api/generate`; let result; try { - result = spawnSyncImpl( - "curl", + const execution = prepareExecution( [ + "curl", "-sS", "--fail-with-body", "-o", @@ -1537,8 +1617,18 @@ function unloadOllamaModels( JSON.stringify({ model, keep_alive: 0 }), endpoint, ], - { encoding: "utf8", env: buildSubprocessEnv() }, + releaseHost, + { + env: buildSubprocessEnv(), + operation: `Ollama model release for '${model}'`, + }, ); + const [command, ...args] = execution.command; + try { + result = spawnSyncImpl(command, args, { encoding: "utf8", env: execution.env }); + } finally { + execution.cleanup(); + } } catch (error) { result = { status: null, @@ -1577,7 +1667,14 @@ function unloadOllamaModels( } sleepImpl(OLLAMA_RELEASE_VERIFY_DELAY_MS); - const verification = discoverResidentOllamaModels(attempt, selectedModels, releaseEndpoint, spawnSyncImpl); + const verification = discoverResidentOllamaModels( + attempt, + selectedModels, + releaseHost, + releaseEndpoint, + spawnSyncImpl, + prepareExecution, + ); discoveries.push(verification); if (verification.error) { if (attempt < maxAttempts && transientCurlFailure(verification.status)) { @@ -1621,12 +1718,17 @@ function unloadOllamaModels( export { checkOllamaModelToolSupport, + clearPendingOllamaModelCleanup, ensureOllamaAuthProxy, getOllamaProxyToken, getOllamaPullTimeoutMs, + isLocalOllamaRouteOwner, isProxyHealthy, killStaleProxy, + loadPendingOllamaModelCleanup, + loadPersistedOllamaHost, noAuthProxy, + ollamaModelRefsMatch, persistAndProbeOllamaProxy, persistProxyToken, prepareOllamaModel, @@ -1637,5 +1739,6 @@ export { startOllamaAuthProxy, unloadOllamaModels, withOllamaModelOwnershipLock, + withOllamaModelOwnershipTransaction, withOllamaProxyLifecycleTransaction, }; diff --git a/src/lib/inference/ollama/windows.test.ts b/src/lib/inference/ollama/windows.test.ts index 5b02de34c20..554a3750e0d 100644 --- a/src/lib/inference/ollama/windows.test.ts +++ b/src/lib/inference/ollama/windows.test.ts @@ -5,7 +5,6 @@ import { createRequire } from "node:module"; import { describe, expect, it, vi } from "vitest"; const require = createRequire(import.meta.url); -const childProcess = require("node:child_process"); const WINDOWS_DIST_PATH = require.resolve("./windows"); const RUNNER_PATH = require.resolve("../../runner"); const LOCAL_INFERENCE_PATH = require.resolve("../local"); @@ -23,12 +22,7 @@ function loadWindowsOllamaWithMocks( const originalRun = runner.run; const originalRunCapture = runner.runCapture; // Stub the blocking wait so this test does not spend time on retry delays. - const atomicsWaitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); - // Prove the retired subprocess-sleep path stays unused: the module must not - // call child_process.spawnSync for its fixed readiness delays. - const originalSpawnSync = childProcess.spawnSync; - const spawnSyncSpy = vi.fn(() => ({ status: 0 })); - childProcess.spawnSync = spawnSyncSpy; + const atomicsWaitStub = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); delete require.cache[WINDOWS_DIST_PATH]; runner.run = run; @@ -36,14 +30,11 @@ function loadWindowsOllamaWithMocks( return { windows: require(WINDOWS_DIST_PATH), - atomicsWaitSpy, - spawnSyncSpy, restore() { delete require.cache[WINDOWS_DIST_PATH]; runner.run = originalRun; runner.runCapture = originalRunCapture; - childProcess.spawnSync = originalSpawnSync; - atomicsWaitSpy.mockRestore(); + atomicsWaitStub.mockRestore(); }, }; } @@ -51,20 +42,42 @@ function loadWindowsOllamaWithMocks( describe("Windows Ollama helper", () => { it("rejects a nonempty invalid Docker readiness response (#10100)", () => { const run = vi.fn(); - const runCapture = vi.fn((command: string | string[]) => - Array.isArray(command) && command.at(-1) === WINDOWS_OLLAMA_TAGS_URL - ? "proxy response" - : "", - ); const localInference = require(LOCAL_INFERENCE_PATH); + const runCapture = vi.fn((command: string | string[]) => { + expect(command).toEqual( + expect.arrayContaining([ + "docker", + "run", + "--rm", + localInference.CONTAINER_REACHABILITY_IMAGE, + WINDOWS_OLLAMA_TAGS_URL, + ]), + ); + expect(command.slice(0, 4)).toEqual([ + "docker", + "run", + "--rm", + localInference.CONTAINER_REACHABILITY_IMAGE, + ]); + expect(command.at(-1)).toBe(WINDOWS_OLLAMA_TAGS_URL); + return "proxy response"; + }); localInference.resetOllamaHostCache(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - const { windows, restore, atomicsWaitSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { - expect(windows.awaitWindowsOllamaReady()).toBe(false); - expect(atomicsWaitSpy).toHaveBeenCalledTimes(15); - expect(runCapture).toHaveBeenCalledTimes(15); + expect( + windows.awaitWindowsOllamaReady({ + delay: vi.fn(), + prepareDockerEnvironment: () => ({ + env: {}, + isolatedCredentialConfig: false, + cleanup: () => ({ ok: true }), + }), + }), + ).toBe(false); + expect(runCapture.mock.calls.length).toBeGreaterThan(0); expect(localInference.getResolvedOllamaHost()).toBe("127.0.0.1"); } finally { localInference.resetOllamaHostCache(); @@ -106,24 +119,11 @@ describe("Windows Ollama helper", () => { }); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const delay = vi.fn(); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { - expect(windows.setupWindowsOllamaWith0000Binding({ installedPath })).toBe(true); - // The blocking wait settles for 1s after the kill, pauses 1s between - // launch attempts, then polls readiness with a 2s delay. - expect(atomicsWaitSpy).toHaveBeenCalledTimes(3); - // The wait must target the module's shared backing store (a plain - // ArrayBuffer would be rejected by Atomics.wait). - atomicsWaitSpy.mock.calls.forEach(([array]) => { - expect(array).toBeInstanceOf(Int32Array); - expect(array.buffer).toBeInstanceOf(SharedArrayBuffer); - }); - expect(atomicsWaitSpy).toHaveBeenNthCalledWith(1, expect.any(Int32Array), 0, 0, 1000); - expect(atomicsWaitSpy).toHaveBeenNthCalledWith(2, expect.any(Int32Array), 0, 0, 1000); - expect(atomicsWaitSpy).toHaveBeenNthCalledWith(3, expect.any(Int32Array), 0, 0, 2000); - // The retired subprocess-sleep path must not be exercised. - expect(spawnSyncSpy).not.toHaveBeenCalled(); + expect(windows.setupWindowsOllamaWith0000Binding({ installedPath, delay })).toBe(true); } finally { restore(); logSpy.mockRestore(); @@ -144,7 +144,7 @@ describe("Windows Ollama helper", () => { "docker", "run", "--rm", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", "2", @@ -152,22 +152,51 @@ describe("Windows Ollama helper", () => { "5", "http://host.docker.internal:11434/api/tags", ], - { ignoreError: true }, + expect.objectContaining({ ignoreError: true }), ); + expect(delay).toHaveBeenCalled(); + expect(delay.mock.calls.every(([seconds]) => seconds > 0 && seconds <= 2)).toBe(true); }); - it("skips the blocking wait for non-positive delays", () => { + it("isolates Docker credentials while waiting for the Windows-host daemon", () => { const run = vi.fn(); - const runCapture = vi.fn(); - const { windows, restore, atomicsWaitSpy, spawnSyncSpy } = loadWindowsOllamaWithMocks(run, runCapture); + const cleanup = vi.fn(() => ({ ok: true as const })); + const runCapture = vi.fn((command: string | string[], options?: { env?: NodeJS.ProcessEnv }) => + Array.isArray(command) && + command[0] === "docker" && + command.at(-1) === WINDOWS_OLLAMA_TAGS_URL && + options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? JSON.stringify({ models: [] }) + : "", + ); + const localInference = require(LOCAL_INFERENCE_PATH); + localInference.resetOllamaHostCache(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const { windows, restore } = loadWindowsOllamaWithMocks(run, runCapture); try { - windows.sleep(0); - windows.sleep(-1); - expect(atomicsWaitSpy).not.toHaveBeenCalled(); - expect(spawnSyncSpy).not.toHaveBeenCalled(); + expect( + windows.awaitWindowsOllamaReady({ + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + }), + ).toBe(true); + expect(localInference.getResolvedOllamaHost()).toBe("host.docker.internal"); + expect(runCapture).toHaveBeenCalledWith( + expect.arrayContaining(["docker", "run", "--rm", WINDOWS_OLLAMA_TAGS_URL]), + expect.objectContaining({ + ignoreError: true, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }), + ); + expect(cleanup).toHaveBeenCalledOnce(); } finally { + localInference.resetOllamaHostCache(); restore(); + logSpy.mockRestore(); } }); }); diff --git a/src/lib/inference/ollama/windows.ts b/src/lib/inference/ollama/windows.ts index 58dcd9a61fa..cda316ad6cd 100644 --- a/src/lib/inference/ollama/windows.ts +++ b/src/lib/inference/ollama/windows.ts @@ -6,9 +6,9 @@ // Detection lives in onboard.ts; this module owns the action side. const { spawn } = require("child_process"); -const { dockerCapture } = require("../../adapters/docker/command"); const { run, runCapture } = require("../../runner"); const { + createOllamaApiCapture, getWindowsHostOllamaDockerReachabilityArgs, isValidOllamaTagsResponseBody, OLLAMA_HOST_DOCKER_INTERNAL, @@ -120,13 +120,30 @@ function killWindowsOllamaProcesses(): void { ); } -function awaitWindowsOllamaReady(): boolean { +function awaitWindowsOllamaReady( + opts: { prepareDockerEnvironment?: () => unknown; delay?: (seconds: number) => void } = {}, +): boolean { console.log(" Waiting for Ollama to respond on host.docker.internal..."); + const delay = opts.delay ?? sleep; + const capture = createOllamaApiCapture( + runCapture, + OLLAMA_HOST_DOCKER_INTERNAL, + opts.prepareDockerEnvironment, + ); for (let attempt = 0; attempt < 15; attempt++) { - sleep(2); - const probe = dockerCapture(getWindowsHostOllamaDockerReachabilityArgs(), { - ignoreError: true, - }); + delay(2); + const probe = capture( + [ + "curl", + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + `http://${OLLAMA_HOST_DOCKER_INTERNAL}:${OLLAMA_PORT}/api/tags`, + ], + { ignoreError: true }, + ); if (isValidOllamaTagsResponseBody(probe)) { setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); return true; @@ -139,9 +156,10 @@ function awaitWindowsOllamaReady(): boolean { // watcher's auto-restart survive; fall back through the verified installed // path and finally refreshed PATH because stale watcher paths are possible. function launchAndAwaitWindowsOllama( - opts: { watcherPath?: string; installedPath?: string } = {}, + opts: { watcherPath?: string; installedPath?: string; delay?: (seconds: number) => void } = {}, ): boolean { console.log(" Starting Ollama on Windows host via WSL interop..."); + const delay = opts.delay ?? sleep; const watcherPath = typeof opts.watcherPath === "string" ? opts.watcherPath.trim() : ""; const installedPath = typeof opts.installedPath === "string" ? opts.installedPath.trim() : ""; const launchAttempts: Array<{ label: string; script: string }> = []; @@ -174,7 +192,7 @@ function launchAndAwaitWindowsOllama( ignoreError: true, suppressOutput: true, }); - if (result.status === 0 && awaitWindowsOllamaReady()) { + if (result.status === 0 && awaitWindowsOllamaReady({ delay })) { return true; } @@ -187,7 +205,7 @@ function launchAndAwaitWindowsOllama( console.error(` PowerShell launch via ${attempt.label} failed: ${detail}`); if (i < launchAttempts.length - 1) { killWindowsOllamaProcesses(); - sleep(1); + delay(1); } } return false; @@ -197,18 +215,24 @@ function launchAndAwaitWindowsOllama( // installed Ollama. Fresh install fallback passes installedPath to avoid // relying on a newly-mutated Windows PATH from this process. function setupWindowsOllamaWith0000Binding( - opts: { announceStop?: boolean; installedPath?: string } = {}, + opts: { + announceStop?: boolean; + installedPath?: string; + delay?: (seconds: number) => void; + } = {}, ): boolean { + const delay = opts.delay ?? sleep; const watcherPath = captureWindowsOllamaWatcherPath(); persistOllamaHostEnvVar(); if (opts.announceStop) { console.log(" Stopping existing Ollama on Windows host..."); } killWindowsOllamaProcesses(); - sleep(1); + delay(1); return launchAndAwaitWindowsOllama({ watcherPath: watcherPath || undefined, installedPath: opts.installedPath, + delay, }); } diff --git a/src/lib/inference/onboard-host-docker-internal.test.ts b/src/lib/inference/onboard-host-docker-internal.test.ts index 62cd833ee53..61ce1e96e87 100644 --- a/src/lib/inference/onboard-host-docker-internal.test.ts +++ b/src/lib/inference/onboard-host-docker-internal.test.ts @@ -106,7 +106,12 @@ describe("host.docker.internal onboarding inference policy", () => { expect(seenCommands).toHaveLength(1); seenCommands.forEach(({ command, args }) => { expect(command).toBe("docker"); - expect(args).toContain("curlimages/curl:8.10.1"); + expect(args.slice(0, 3)).toEqual([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + ]); + expect(args).not.toContain("curlimages/curl:8.10.1"); expect(args).toContain("http://host.docker.internal:11434/v1/chat/completions"); expect(args).not.toContain("--volume"); }); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b1eca0570db..dad6121b2ae 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -207,7 +207,6 @@ const { getLocalProviderBaseUrl, getLocalProviderHealthCheck, getLocalProviderValidationBaseUrl, - getOllamaWarmupCommand, validateLocalProvider, } = localInference; const resolveNonInteractiveModel = localInference.resolveNonInteractiveOllamaModel; @@ -2453,7 +2452,6 @@ function getSetupInferenceDeps(): SetupInferenceDeps { getLocalProviderBaseUrl, run, vllmLocalCredentialEnv: VLLM_LOCAL_CREDENTIAL_ENV, - getOllamaWarmupCommand, shouldFrontOllamaWithProxy, ensureOllamaAuthProxy, isProxyHealthy, diff --git a/src/lib/onboard/inference-providers/ollama-local.test.ts b/src/lib/onboard/inference-providers/ollama-local.test.ts index d560bf7d292..96b4c161377 100644 --- a/src/lib/onboard/inference-providers/ollama-local.test.ts +++ b/src/lib/onboard/inference-providers/ollama-local.test.ts @@ -1,18 +1,37 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CONTAINER_REACHABILITY_IMAGE, + loadPersistedOllamaHost, + OLLAMA_HOST_DOCKER_INTERNAL, + persistResolvedOllamaHost, + resetOllamaHostCache, + runOllamaWarmup, + setResolvedOllamaHost, +} from "../../inference/local"; import { setupOllamaLocalInference } from "./ollama-local"; import type { OllamaDeps } from "./types"; const CREDENTIAL_ENV = "NEMOCLAW_OLLAMA_PROXY_TOKEN"; +afterEach(() => resetOllamaHostCache()); + const SANDBOX_ENDPOINT_MISMATCH = "Selected Ollama model 'llama3.2:1b' answers on http://127.0.0.1:11434, but the daemon the " + "sandbox reaches through http://host.openshell.internal:11434 does not serve it " + "(reported models: qwen3.5:2b, gemma4:26b)."; -function deps(overrides: Partial = {}): OllamaDeps { +type OllamaDepsOverrides = Omit, "localInference"> & { + localInference?: Partial; +}; + +function deps(overrides: OllamaDepsOverrides = {}): OllamaDeps { + const { localInference, ...rest } = overrides; return { runOpenshell: vi.fn(() => ({ status: 0 })), upsertProvider: vi.fn(() => ({ ok: true })), @@ -28,7 +47,6 @@ function deps(overrides: Partial = {}): OllamaDeps { validateLocalProvider: () => ({ ok: true }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:11434/v1", applyLocalInferenceRoute: async () => false, - getOllamaWarmupCommand: () => ["ollama", "run", "llama3.2:1b"], run: vi.fn(() => ({ status: 0 })), shouldFrontOllamaWithProxy: () => false, ensureOllamaAuthProxy: vi.fn(), @@ -38,9 +56,12 @@ function deps(overrides: Partial = {}): OllamaDeps { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: vi.fn(), + persistResolvedOllamaHost: () => () => {}, + ...localInference, }, OLLAMA_PROXY_CREDENTIAL_ENV: CREDENTIAL_ENV, - ...overrides, + ...rest, }; } @@ -74,7 +95,7 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("blocks even when every host-side check passes (#9454)", async () => { const validateOllamaModelWithToolsOverride = vi.fn(() => ({ ok: true })); - const run = vi.fn(() => ({ status: 0 })); + const run = vi.fn((_command) => ({ status: 0 })); await expect( setupOllamaLocalInference( @@ -99,20 +120,235 @@ describe("Ollama local provider sandbox-facing model gate", () => { it("records the route when the sandbox endpoint serves the model", async () => { const upsertProvider = vi.fn(() => ({ ok: true })); + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-provider-route-")); + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + try { + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + upsertProvider, + localInference: { + persistResolvedOllamaHost: () => persistResolvedOllamaHost(undefined, stateRoot), + }, + }), + ), + ).resolves.toEqual({ done: false }); + + expect(upsertProvider).toHaveBeenCalledWith( + "ollama-local", + "openai", + CREDENTIAL_ENV, + "http://host.openshell.internal:11434/v1", + { [CREDENTIAL_ENV]: "ollama" }, + ); + expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL); + } finally { + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + + it("dispatches Windows-host warm-up through Docker Desktop", async () => { + setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL); + const run = vi.fn((_command: unknown) => ({ status: 0 })); + const cleanup = vi.fn(() => ({ ok: true as const })); await expect( setupOllamaLocalInference( { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, - deps({ upsertProvider }), + deps({ + run, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: (model, runImpl) => + runOllamaWarmup(model, runImpl, () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + })), + persistResolvedOllamaHost: vi.fn(() => () => {}), + }, + }), ), ).resolves.toEqual({ done: false }); - expect(upsertProvider).toHaveBeenCalledWith( - "ollama-local", - "openai", - CREDENTIAL_ENV, - "http://host.openshell.internal:11434/v1", - { [CREDENTIAL_ENV]: "ollama" }, + expect(run).toHaveBeenCalledWith( + expect.arrayContaining([ + "docker", + "run", + "--rm", + CONTAINER_REACHABILITY_IMAGE, + "http://host.docker.internal:11434/api/generate", + ]), + { + ignoreError: true, + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + }, ); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("fails before provider registration when the cleanup route cannot be staged", async () => { + const upsertProvider = vi.fn(() => ({ ok: true })); + const error = vi.fn(); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + upsertProvider, + error, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => { + throw new Error("state path is unsafe"); + }, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(upsertProvider).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("state path is unsafe")); + }); + + it("restores the prior cleanup route when provider registration fails", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + upsertProvider: () => ({ ok: false, status: 1, message: "provider rejected" }), + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("restores the prior cleanup route when route application requests reselection", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + applyLocalInferenceRoute: async () => true, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).resolves.toEqual({ done: true, result: { retry: "selection" } }); + + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("restores the prior cleanup route when route application throws", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + applyLocalInferenceRoute: async () => { + throw new Error("route application failed"); + }, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("route application failed"); + + expect(persistResolvedOllamaHost).toHaveBeenCalledOnce(); + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("restores the prior cleanup route when provider-owned proof mismatches", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + providerOwnedInferenceProof: { + protocol: "openai-chat-completions", + model: "ollama/wrong-model", + toolCallingRequired: true, + }, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("restores the prior cleanup route when model validation fails", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + const persistResolvedOllamaHost = vi.fn(() => rollbackPersistedOllamaHost); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + localInference: { + validateOllamaModelWithToolsOverride: () => ({ + ok: false, + message: "model validation failed", + }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost, + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); + }); + + it("restores the prior cleanup route when model warm-up throws", async () => { + const rollbackPersistedOllamaHost = vi.fn(); + + await expect( + setupOllamaLocalInference( + { model: "llama3.2:1b", provider: "ollama-local", allowToolsIncompatible: false }, + deps({ + localInference: { + runOllamaWarmup: () => { + throw new Error("warm-up transport failed"); + }, + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + persistResolvedOllamaHost: () => rollbackPersistedOllamaHost, + }, + }), + ), + ).rejects.toThrow("warm-up transport failed"); + + expect(rollbackPersistedOllamaHost).toHaveBeenCalledOnce(); }); }); diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index e2552a7aa0c..2a476332194 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -22,7 +22,6 @@ export async function setupOllamaLocalInference( validateLocalProvider, getLocalProviderBaseUrl, applyLocalInferenceRoute, - getOllamaWarmupCommand, run, shouldFrontOllamaWithProxy, ensureOllamaAuthProxy, @@ -95,22 +94,61 @@ export async function setupOllamaLocalInference( await persistAndProbeOllamaProxy(proxyToken); } } + let rollbackPersistedOllamaHost: () => void; + try { + rollbackPersistedOllamaHost = localInference.persistResolvedOllamaHost(); + } catch (persistError) { + error( + ` Could not stage the selected local Ollama route for later stop/destroy cleanup: ${ + persistError instanceof Error ? persistError.message : String(persistError) + }`, + ); + return exitProcess(1); + } + const rollbackCleanupRoute = (): boolean => { + try { + rollbackPersistedOllamaHost(); + return true; + } catch (rollbackError) { + error( + ` Could not restore the prior local Ollama cleanup route: ${ + rollbackError instanceof Error ? rollbackError.message : String(rollbackError) + }`, + ); + return false; + } + }; // Use a dedicated internal credential env (NEMOCLAW_OLLAMA_PROXY_TOKEN) // so the gateway never reads the user's host OPENAI_API_KEY for local // Ollama. GH #2519: a stale host OPENAI_API_KEY was leaking into the // inference path and producing 401s. - const providerResult = upsertProvider( - "ollama-local", - "openai", - OLLAMA_PROXY_CREDENTIAL_ENV, - baseUrl, - { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, - ); + let providerResult: ReturnType; + try { + providerResult = upsertProvider( + "ollama-local", + "openai", + OLLAMA_PROXY_CREDENTIAL_ENV, + baseUrl, + { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, + ); + } catch (providerError) { + rollbackCleanupRoute(); + throw providerError; + } if (!providerResult.ok) { + rollbackCleanupRoute(); error(` ${providerResult.message}`); return exitProcess(providerResult.status || 1); } - if (await applyLocalInferenceRoute("ollama-local", model)) { + let retrySelection: boolean; + try { + retrySelection = await applyLocalInferenceRoute("ollama-local", model); + } catch (routeError) { + rollbackCleanupRoute(); + throw routeError; + } + if (retrySelection) { + if (!rollbackCleanupRoute()) return exitProcess(1); return { done: true, result: { retry: "selection" } }; } if (providerOwnedInferenceProof) { @@ -119,17 +157,22 @@ export async function setupOllamaLocalInference( providerOwnedInferenceProof.model !== normalizeHostLocalOllamaModelRef(model) || providerOwnedInferenceProof.toolCallingRequired !== !allowToolsIncompatible ) { + rollbackCleanupRoute(); error(" Provider-owned Ollama proof does not match the accepted model capability request."); return exitProcess(1); } } else { - log(` Priming Ollama model: ${model}`); - run(getOllamaWarmupCommand(model), { ignoreError: true }); - const probe = localInference.validateOllamaModelWithToolsOverride( - model, - allowToolsIncompatible, - ); + let probe: ReturnType; + try { + log(` Priming Ollama model: ${model}`); + localInference.runOllamaWarmup(model, run); + probe = localInference.validateOllamaModelWithToolsOverride(model, allowToolsIncompatible); + } catch (probeError) { + rollbackCleanupRoute(); + throw probeError; + } if (!probe.ok) { + rollbackCleanupRoute(); error(` ${probe.message}`); return exitProcess(1); } diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index ce6d3f6cf48..40d56416959 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -213,7 +213,7 @@ export type HermesDeps = CommonDeps & { // loosely so callers can pass either shape without casting. export type RunFn = ( cmd: any, - opts?: { ignoreError?: boolean; suppressOutput?: boolean }, + opts?: { ignoreError?: boolean; suppressOutput?: boolean; env?: NodeJS.ProcessEnv }, ) => RunResult; export type VllmDeps = CommonDeps & { @@ -242,7 +242,6 @@ export type OllamaDeps = CommonDeps & { }; getLocalProviderBaseUrl: (provider: string) => any; applyLocalInferenceRoute: (provider: string, model: string) => Promise; - getOllamaWarmupCommand: (model: string) => any; run: RunFn; shouldFrontOllamaWithProxy: () => boolean; ensureOllamaAuthProxy: () => void; @@ -255,6 +254,15 @@ export type OllamaDeps = CommonDeps & { allowToolsIncompatible: boolean, ): { ok: boolean; message?: string }; validateSandboxFacingOllamaModel(model: string): { ok: boolean; message?: string }; + runOllamaWarmup(model: string, runImpl: RunFn): void; + loadPendingOllamaModelCleanup?(sandboxName: string): readonly string[]; + persistPendingOllamaModelCleanup?(sandboxName: string, models: readonly string[]): void; + clearPendingOllamaModelCleanup?(sandboxName: string, releasedModels?: readonly string[]): void; + persistResolvedOllamaHost(): () => void; + loadPersistedOllamaHost?(): "127.0.0.1" | "host.docker.internal" | null; + clearPersistedOllamaHostIfUnused?( + routes: readonly { provider?: string | null; endpointUrl?: string | null }[], + ): boolean; }; /** Exact provider-owned proof used instead of legacy host warmup/probes. */ providerOwnedInferenceProof?: { diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 4d115791344..911cdcf39b8 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -229,7 +229,7 @@ describe("detectInferenceProviderHostState", () => { [ "run", "--rm", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", "2", diff --git a/src/lib/onboard/sandbox-readiness-tracing.test.ts b/src/lib/onboard/sandbox-readiness-tracing.test.ts index 0701b040248..aa49d4ae27f 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.test.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.test.ts @@ -81,7 +81,6 @@ describe("createSandboxReadyWaiter", () => { isLinuxDockerDriverGatewayEnabled: () => true, now: () => 0, sleep, - now: () => 0, }); await expect(waitForSandboxReady(NAME, 2, 3)).resolves.toEqual({ diff --git a/src/lib/onboard/setup-inference.test.ts b/src/lib/onboard/setup-inference.test.ts index 2b569282a85..ecffb3ad17e 100644 --- a/src/lib/onboard/setup-inference.test.ts +++ b/src/lib/onboard/setup-inference.test.ts @@ -324,7 +324,6 @@ describe("createProviderReviewDeps", () => { validateLocalProvider: () => ({ ok: true }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:11435/v1", applyLocalInferenceRoute: async () => false, - getOllamaWarmupCommand: () => ["ollama", "run", "qwen3.5:9b"], run: vi.fn() as never, shouldFrontOllamaWithProxy: () => true, ensureOllamaAuthProxy, @@ -334,6 +333,8 @@ describe("createProviderReviewDeps", () => { localInference: { validateOllamaModelWithToolsOverride: () => ({ ok: true }), validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, + persistResolvedOllamaHost: () => () => {}, }, OLLAMA_PROXY_CREDENTIAL_ENV: "NEMOCLAW_OLLAMA_PROXY_TOKEN", }, diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 86efa3b5dda..d6747dcf46b 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -19,12 +19,21 @@ import { withModelRouterPortLifecycleLock, } from "../inference/gateway-route-mutation-lock"; import { getManagedVllmProviderBinding } from "../inference/local"; -import { type OllamaModelHolder, supersededOllamaModel } from "../inference/ollama/model-ownership"; +import { + clearPendingOllamaModelCleanup, + isLocalOllamaRouteOwner, + loadPendingOllamaModelCleanup, + type OllamaModelHolder, + persistPendingOllamaModelCleanup, + supersededOllamaModel, +} from "../inference/ollama/model-ownership"; import { getOllamaProxyToken, persistAndProbeOllamaProxy, startOllamaAuthProxy, + type OllamaUnloadResult, withOllamaModelOwnershipLock, + withOllamaModelOwnershipTransaction, } from "../inference/ollama/proxy"; import { assertNoOpenShellGatewayEndpointOverride, @@ -178,7 +187,6 @@ type ProviderBranchDeps = Pick< > & Pick< OllamaDeps, - | "getOllamaWarmupCommand" | "shouldFrontOllamaWithProxy" | "ensureOllamaAuthProxy" | "isProxyHealthy" @@ -218,8 +226,9 @@ export type SetupInferenceDeps = ProviderBranchDeps & { // by hand, so every read below must stay optional-chained. getSandbox?: typeof import("../state/registry").getSandbox; listSandboxes?: typeof import("../state/registry").listSandboxes; - unloadOllamaModels?: (onlyModels: readonly string[]) => void; + unloadOllamaModels?: (onlyModels: readonly string[]) => OllamaUnloadResult | void; withOllamaModelOwnershipLock?: typeof withOllamaModelOwnershipLock; + withOllamaModelOwnershipTransaction?: typeof withOllamaModelOwnershipTransaction; localInferenceTimeoutSecs: number; vllmLocalCredentialEnv: string; getManagedVllmProviderBinding?: () => { @@ -513,7 +522,9 @@ export type SetupInference = ( */ function releaseSupersededOllamaModel( previous: OllamaModelHolder | null, + nextProvider: string, nextModel: string, + nextEndpointUrl: string | null, result: SetupInferenceResult, deps: SetupInferenceDeps, revalidateSandboxIdentity?: (operation: string) => void, @@ -522,23 +533,112 @@ function releaseSupersededOllamaModel( // still owns its model. if (!previous || result.retry) return; let authorityRefusal: unknown; + let cleanupWarning: string | null = null; + let attemptedModels: readonly string[] = []; + let pendingRecordFailure: string | null = null; + const loadPending = + deps.localInference.loadPendingOllamaModelCleanup ?? loadPendingOllamaModelCleanup; + const persistPending = + deps.localInference.persistPendingOllamaModelCleanup ?? persistPendingOllamaModelCleanup; + const clearPending = + deps.localInference.clearPendingOllamaModelCleanup ?? clearPendingOllamaModelCleanup; + const persistRetry = (): string | null => { + if (attemptedModels.length === 0) return null; + try { + persistPending(previous.name, attemptedModels); + return null; + } catch (error) { + return (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .slice(0, 240); + } + }; try { const withOwnershipLock = deps.withOllamaModelOwnershipLock ?? withOllamaModelOwnershipLock; withOwnershipLock(() => { const peers = deps.listSandboxes?.().sandboxes ?? []; - const superseded = supersededOllamaModel(previous, nextModel, peers); - if (!superseded) return; + const selectedHost = deps.localInference.loadPersistedOllamaHost?.() ?? null; + const nextRoute = { provider: nextProvider, model: nextModel, endpointUrl: nextEndpointUrl }; + const superseded = supersededOllamaModel(previous, nextRoute, peers, selectedHost); + const pending = loadPending(previous.name); + const retryablePending = pending.filter((model) => + supersededOllamaModel( + { name: previous.name, provider: "ollama-local", model, endpointUrl: null }, + nextRoute, + peers, + selectedHost, + ), + ); + attemptedModels = [...new Set([...(superseded ? [superseded] : []), ...retryablePending])]; + const retireRoute = + isLocalOllamaRouteOwner(previous, selectedHost) && + !isLocalOllamaRouteOwner(nextRoute, selectedHost) && + !peers.some((peer) => isLocalOllamaRouteOwner(peer, selectedHost)); + if (attemptedModels.length === 0 && !retireRoute) return; try { revalidateSandboxIdentity?.("release the superseded Ollama model"); } catch (error) { authorityRefusal = error; return; } - deps.unloadOllamaModels?.([superseded]); + if (attemptedModels.length > 0 && deps.unloadOllamaModels) { + // The committed route no longer names the old model. Record it before + // release so later lifecycle commands retain a scoped retry target. + pendingRecordFailure = persistRetry(); + try { + const cleanup = deps.unloadOllamaModels(attemptedModels); + if (cleanup && !cleanup.ok) { + if (pendingRecordFailure) pendingRecordFailure = persistRetry(); + const detail = cleanup.message + ? `: ${cleanup.message.replace(/\s+/g, " ").slice(0, 240)}` + : ""; + const recoveryAction = + cleanup.outcome === "discovery-failed" + ? `Restore access to ${cleanup.endpoint}` + : cleanup.outcome === "still-resident" + ? `Stop the recorded model at ${cleanup.endpoint}` + : `Allow the model unload request at ${cleanup.endpoint}`; + cleanupWarning = + ` Warning: Ollama did not release recorded model cleanup for '${previous.name}' from ` + + `${cleanup.endpoint} (outcome: ${cleanup.outcome}${detail}). The new inference ` + + `route remains active. ${recoveryAction}. ` + + (pendingRecordFailure + ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ")} at ${cleanup.endpoint}.` + : `Re-run onboarding or destroy '${previous.name}' to retry only: ${attemptedModels.join(", ")}.`); + } else { + clearPending(previous.name, attemptedModels); + } + } catch (error) { + if (pendingRecordFailure) pendingRecordFailure = persistRetry(); + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .slice(0, 240); + cleanupWarning = + ` Warning: Ollama cleanup for '${previous.name}' failed: ${detail}. The new inference ` + + `route remains active. ` + + (pendingRecordFailure + ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ")} from the saved local Ollama endpoint.` + : `Re-run onboarding or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ")}.`); + } + } + const pendingAfterCleanup = loadPending(previous.name); + if (retireRoute && !cleanupWarning && pendingAfterCleanup.length === 0) { + deps.localInference.clearPersistedOllamaHostIfUnused?.(peers); + } }); - } catch { - /* Best-effort: a failed unload must not fail an onboarding that already committed its route. */ + } catch (error) { + if (!pendingRecordFailure) pendingRecordFailure = persistRetry(); + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .slice(0, 240); + cleanupWarning = + ` Warning: NemoClaw could not finish superseded Ollama cleanup: ${detail}. The new ` + + `inference route remains active. ` + + (pendingRecordFailure + ? `Cleanup retry state could not be recorded: ${pendingRecordFailure}. Manually release only ${attemptedModels.join(", ") || "the superseded model"} from the saved local Ollama endpoint.` + : `Re-run onboarding or destroy '${previous.name}' to retry only the recorded models: ${attemptedModels.join(", ") || "none"}.`); } + if (cleanupWarning) console.warn(cleanupWarning); if (authorityRefusal) throw authorityRefusal; } @@ -925,45 +1025,50 @@ export function createSetupInference( return outcome.result; } } else if (provider === "ollama-local") { - const outcome = await inferenceProviders.setupOllamaLocalInference( - { - model, - provider, - allowToolsIncompatible: options.allowToolsIncompatible === true, - ...(hostLocalRoute ? {} : { preparedProxyToken: options.preparedOllamaProxyToken }), - }, - { - ...commonDeps, - validateLocalProvider: hostLocalRoute - ? () => ({ ok: true as const }) - : deps.validateLocalProvider, - getLocalProviderBaseUrl: hostLocalRoute - ? () => hostLocalRoute.gatewayProviderBaseUrl - : deps.getLocalProviderBaseUrl, - applyLocalInferenceRoute: resolveLocalInferenceRouteApplier( - hostLocalRoute - ? { - ...deps, - exitProcess: commonDeps.exitProcess, - error: commonDeps.error, - } - : deps, - runGatewayOpenshell, - revalidateSandboxIdentity, - ), - getOllamaWarmupCommand: deps.getOllamaWarmupCommand, - run: deps.run, - shouldFrontOllamaWithProxy: hostLocalRoute - ? () => false - : deps.shouldFrontOllamaWithProxy, - ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, - isProxyHealthy: deps.isProxyHealthy, - getOllamaProxyToken: deps.getOllamaProxyToken, - persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, - localInference: deps.localInference, - providerOwnedInferenceProof: hostLocalRoute?.receipt.inference, - OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, - }, + const withOwnershipTransaction = + deps.withOllamaModelOwnershipTransaction ?? withOllamaModelOwnershipTransaction; + const outcome = await withOwnershipTransaction(() => + inferenceProviders.setupOllamaLocalInference( + { + model, + provider, + allowToolsIncompatible: options.allowToolsIncompatible === true, + ...(hostLocalRoute + ? {} + : { preparedProxyToken: options.preparedOllamaProxyToken }), + }, + { + ...commonDeps, + validateLocalProvider: hostLocalRoute + ? () => ({ ok: true as const }) + : deps.validateLocalProvider, + getLocalProviderBaseUrl: hostLocalRoute + ? () => hostLocalRoute.gatewayProviderBaseUrl + : deps.getLocalProviderBaseUrl, + applyLocalInferenceRoute: resolveLocalInferenceRouteApplier( + hostLocalRoute + ? { + ...deps, + exitProcess: commonDeps.exitProcess, + error: commonDeps.error, + } + : deps, + runGatewayOpenshell, + revalidateSandboxIdentity, + ), + run: deps.run, + shouldFrontOllamaWithProxy: hostLocalRoute + ? () => false + : deps.shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, + isProxyHealthy: deps.isProxyHealthy, + getOllamaProxyToken: deps.getOllamaProxyToken, + persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, + localInference: deps.localInference, + providerOwnedInferenceProof: hostLocalRoute?.receipt.inference, + OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, + }, + ), ); if (outcome.done) { if (hostLocalRoute && hostLocalGatewayMutation && hostLocalSelection) { @@ -1125,7 +1230,15 @@ export function createSetupInference( /* An unreadable registry skips GPU release; it must not fail onboarding. */ } const result = await mutateGatewayRoute(); - releaseSupersededOllamaModel(previousSandbox, model, result, deps, revalidateSandboxIdentity); + releaseSupersededOllamaModel( + previousSandbox, + provider, + model, + endpointUrl, + result, + deps, + revalidateSandboxIdentity, + ); if (shouldLogSuccessfulRoute && "ok" in result) { deps.log(` ✓ Inference route set: ${provider} / ${model}`); } diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 4a760b5132e..8398480f592 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -75,6 +75,7 @@ import { hasUnsafeHostMountTerminalText } from "./registry/host-mount"; import { nemoclawStateRoot } from "./state-root"; export { normalizePersistedSandboxHostMounts } from "./registry/host-mount"; +export type { RetainedSandboxRecoveryRecord } from "./onboard-session/retained-sandbox-recovery"; export const SESSION_VERSION = 1; export const MACHINE_SNAPSHOT_VERSION = 1; diff --git a/src/lib/tunnel/services-gateway-ownership.test.ts b/src/lib/tunnel/services-gateway-ownership.test.ts index 42dd5aeee9c..e502f7ee28f 100644 --- a/src/lib/tunnel/services-gateway-ownership.test.ts +++ b/src/lib/tunnel/services-gateway-ownership.test.ts @@ -14,6 +14,8 @@ import * as gatewayStop from "./gateway-stop"; import * as sandboxGatewayStop from "./sandbox-gateway-stop"; import { stopAll } from "./services"; +const neutralOllamaCleanup = () => undefined; + vi.mock("../adapters/docker", () => ({ dockerCapture: vi.fn(), dockerForceRm: vi.fn(), @@ -247,7 +249,12 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, sandboxName: "alpha", releaseGatewayPort: true }); + stopAll({ + pidDir, + sandboxName: "alpha", + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -282,7 +289,7 @@ describe("stopAll gateway-stop wiring", () => { vi.spyOn(sandboxGatewayStop, "stopSandboxChannels").mockImplementation(() => {}); try { - stopAll({ pidDir, sandboxName: "alpha" }); + stopAll({ pidDir, sandboxName: "alpha", unloadOllamaModels: neutralOllamaCleanup }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -304,7 +311,11 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, releaseGatewayPort: true }); + stopAll({ + pidDir, + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -328,7 +339,11 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, releaseGatewayPort: true }); + stopAll({ + pidDir, + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } @@ -349,7 +364,11 @@ describe("stopAll gateway-stop wiring", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); try { - stopAll({ pidDir, releaseGatewayPort: true }); + stopAll({ + pidDir, + releaseGatewayPort: true, + unloadOllamaModels: neutralOllamaCleanup, + }); } finally { rmSync(pidDir, { recursive: true, force: true }); } diff --git a/src/lib/tunnel/services-sandbox.test.ts b/src/lib/tunnel/services-sandbox.test.ts index 87fe52cf782..4c9c34ef78c 100644 --- a/src/lib/tunnel/services-sandbox.test.ts +++ b/src/lib/tunnel/services-sandbox.test.ts @@ -21,6 +21,10 @@ function restoreSandboxEnv(saved: Record<(typeof SANDBOX_ENV_NAMES)[number], str } } +function stopAllWithoutOllama(opts: Parameters[0] = {}) { + return stopAll({ ...opts, cleanupOllamaModels: false }); +} + describe("stopAll with sandbox channels", () => { let pidDir: string; let stopSandboxChannels: ReturnType; @@ -46,7 +50,7 @@ describe("stopAll with sandbox channels", () => { it("stops in-sandbox channels when sandboxName is provided", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir, sandboxName: "test-sb" }); + stopAllWithoutOllama({ pidDir, sandboxName: "test-sb" }); expect(stopSandboxChannels).toHaveBeenCalledWith("test-sb", { info: expect.any(Function), @@ -58,7 +62,7 @@ describe("stopAll with sandbox channels", () => { it("warns when no sandbox name is available", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).not.toHaveBeenCalled(); const output = logSpy.mock.calls.map((call) => call[0]).join("\n"); @@ -69,7 +73,7 @@ describe("stopAll with sandbox channels", () => { it("still stops cloudflared when in-sandbox shutdown cannot stop a process", () => { writeFileSync(join(pidDir, "cloudflared.pid"), "999999999"); - stopAll({ pidDir, sandboxName: "test-sb" }); + stopAllWithoutOllama({ pidDir, sandboxName: "test-sb" }); expect(stopSandboxChannels).toHaveBeenCalledTimes(1); expect(existsSync(join(pidDir, "cloudflared.pid"))).toBe(false); @@ -78,7 +82,7 @@ describe("stopAll with sandbox channels", () => { it("reads sandbox name from NEMOCLAW_SANDBOX env when not in opts", () => { process.env.NEMOCLAW_SANDBOX = "env-sandbox"; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("env-sandbox", expect.any(Object)); }); @@ -86,7 +90,7 @@ describe("stopAll with sandbox channels", () => { it("reads sandbox name from NEMOCLAW_SANDBOX_NAME when NEMOCLAW_SANDBOX is unset", () => { process.env.NEMOCLAW_SANDBOX_NAME = "named-sandbox"; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("named-sandbox", expect.any(Object)); }); @@ -95,7 +99,7 @@ describe("stopAll with sandbox channels", () => { process.env.NEMOCLAW_SANDBOX_NAME = "name-sandbox"; process.env.NEMOCLAW_SANDBOX = "other-sandbox"; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("name-sandbox", expect.any(Object)); }); @@ -112,7 +116,7 @@ describe("stopAll with sandbox channels", () => { process.env.NEMOCLAW_SANDBOX = "other-sandbox"; try { - stopAll({ pidDir: effectivePidDir }); + stopAllWithoutOllama({ pidDir: effectivePidDir }); expect(stopSandboxChannels).toHaveBeenCalledWith("name-sandbox", expect.any(Object)); expect(existsSync(join(effectivePidDir, "cloudflared.pid"))).toBe(false); @@ -129,7 +133,7 @@ describe("stopAll with sandbox channels", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); process.env.NEMOCLAW_SANDBOX_NAME = invalidName; - stopAll({ pidDir }); + stopAllWithoutOllama({ pidDir }); expect(stopSandboxChannels).not.toHaveBeenCalled(); expect(logSpy.mock.calls.map((call) => call[0]).join("\n")).toContain("Invalid sandbox name"); @@ -146,7 +150,7 @@ describe("stopAll with sandbox channels", () => { writeFileSync(join(pidDir, "cloudflared.pid"), "999999999"); expect(() => - stopAll({ pidDir, sandboxName: "bad name", releaseGatewayPort: true }), + stopAllWithoutOllama({ pidDir, sandboxName: "bad name", releaseGatewayPort: true }), ).not.toThrow(); expect(stopSandboxChannels).not.toHaveBeenCalled(); @@ -161,7 +165,7 @@ describe("stopAll with sandbox channels", () => { it("does not stop default cloudflared for a malformed sandbox name without an explicit pidDir", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - expect(() => stopAll({ sandboxName: "bad name" })).not.toThrow(); + expect(() => stopAllWithoutOllama({ sandboxName: "bad name" })).not.toThrow(); expect(stopSandboxChannels).not.toHaveBeenCalled(); const output = logSpy.mock.calls.map((call) => call[0]).join("\n"); diff --git a/src/lib/tunnel/services.test.ts b/src/lib/tunnel/services.test.ts index a5b9101cb2d..913e04f68e8 100644 --- a/src/lib/tunnel/services.test.ts +++ b/src/lib/tunnel/services.test.ts @@ -596,7 +596,7 @@ describe("stopAll", () => { it("logs stop messages", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir }); + stopAll({ pidDir, unloadOllamaModels: () => undefined }); const output = logSpy.mock.calls.map((c) => c[0]).join("\n"); expect(output).toContain("All services stopped"); logSpy.mockRestore(); @@ -604,8 +604,14 @@ describe("stopAll", () => { it("runs injected Ollama cleanup before reporting services stopped", () => { const cleanup = vi.fn(); + const clearPendingOllamaModelCleanup = vi.fn(); const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - stopAll({ pidDir, unloadOllamaModels: cleanup }); + stopAll({ + pidDir, + sandboxName: "test-box", + unloadOllamaModels: cleanup, + clearPendingOllamaModelCleanup, + }); const stoppedCallIndex = logSpy.mock.calls.findIndex(([message]) => String(message).includes("All services stopped"), ); @@ -613,8 +619,75 @@ describe("stopAll", () => { logSpy.mockRestore(); expect(cleanup).toHaveBeenCalledOnce(); + expect(clearPendingOllamaModelCleanup).toHaveBeenCalledWith("test-box"); expect(cleanup.mock.invocationCallOrder[0]).toBeLessThan(stoppedCallOrder ?? 0); }); + + it("skips Ollama cleanup when the scoped caller proves no model ownership", () => { + const cleanup = vi.fn(); + const clearPendingOllamaModelCleanup = vi.fn(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + stopAll({ + pidDir, + sandboxName: "test-box", + cleanupOllamaModels: false, + unloadOllamaModels: cleanup, + clearPendingOllamaModelCleanup, + }); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(cleanup).not.toHaveBeenCalled(); + expect(clearPendingOllamaModelCleanup).not.toHaveBeenCalled(); + expect(output).toContain("All services stopped"); + }); + + it("reports Ollama cleanup failure and retains its recovery route", () => { + const failure = { + ok: false as const, + outcome: "discovery-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: [], + discoveries: [], + requests: [], + message: "could not connect", + }; + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + stopAll({ pidDir, unloadOllamaModels: () => failure }); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(output).toContain("Ollama model cleanup failed at http://host.docker.internal:11434"); + expect(output).toContain("saved local route was retained"); + expect(output).toContain("restore access to http://host.docker.internal:11434"); + expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); + expect(output).not.toContain("All services stopped"); + }); + + it("propagates an unexpected Ollama cleanup failure after stopping services (#10553)", () => { + const clearPendingOllamaModelCleanup = vi.fn(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(() => + stopAll({ + pidDir, + sandboxName: "test-box", + unloadOllamaModels: () => { + throw new Error("transport failed\nwith unbounded detail"); + }, + clearPendingOllamaModelCleanup, + }), + ).toThrow("Ollama model cleanup failed unexpectedly: transport failed with unbounded detail"); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + logSpy.mockRestore(); + + expect(output).toContain("restore access to the saved local Ollama endpoint"); + expect(output).toContain("Host services stopped; Ollama model cleanup remains incomplete"); + expect(output).not.toContain("All services stopped"); + expect(clearPendingOllamaModelCleanup).not.toHaveBeenCalled(); + }); }); // #6212: after cloudflared yields a public URL, startAll must register that diff --git a/src/lib/tunnel/services.ts b/src/lib/tunnel/services.ts index 1f9e902b7f0..61db429830c 100644 --- a/src/lib/tunnel/services.ts +++ b/src/lib/tunnel/services.ts @@ -19,7 +19,11 @@ import { renderBox } from "../cli/banner"; import { AGENT_PRODUCT_NAME, CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding"; import { isObjectRecord } from "../core/json-types"; import { DASHBOARD_PORT } from "../core/ports"; -import { unloadOllamaModels as unloadDefaultOllamaModels } from "../inference/ollama/proxy"; +import { + clearPendingOllamaModelCleanup as clearDefaultPendingOllamaModelCleanup, + unloadOllamaModels as unloadDefaultOllamaModels, + type OllamaUnloadResult, +} from "../inference/ollama/proxy"; import { buildSubprocessEnv } from "../subprocess-env"; import * as agentForwardStop from "./agent-forward-stop"; import { registerTunnelOrigin } from "./allowed-origins"; @@ -45,7 +49,11 @@ export interface ServiceOptions { /** Injectable process operations (identity + signalling) for tests. */ processControl?: ProcessControl; /** Injectable Ollama model cleanup for tests. */ - unloadOllamaModels?: () => void; + unloadOllamaModels?: () => OllamaUnloadResult | void; + /** Whether this scoped stop owns Ollama models that require cleanup. Defaults to true. */ + cleanupOllamaModels?: boolean; + /** Clears pending Ollama cleanup recovery after this sandbox's models unload. */ + clearPendingOllamaModelCleanup?: (sandboxName: string) => void; /** Cloudflare named tunnel token. Falls back to CLOUDFLARE_TUNNEL_TOKEN. */ cloudflareTunnelToken?: string; /** Also release the managed host gateway port (legacy full-stop only). */ @@ -491,7 +499,7 @@ export function showStatus(opts: ServiceOptions = {}): void { } } -export function stopAll(opts: ServiceOptions = {}): void { +export function stopAll(opts: ServiceOptions = {}): OllamaUnloadResult | void { // Resolve the target sandbox once and reuse it for in-sandbox and host-side cleanup. const rawSandboxName = opts.sandboxName ?? @@ -522,12 +530,47 @@ export function stopAll(opts: ServiceOptions = {}): void { warn("Hint: run 'nemoclaw stop' with a registered sandbox or set NEMOCLAW_SANDBOX_NAME."); } - try { - const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; - unloadOllamaModels(); - } catch { - /* best-effort */ + let ollamaCleanupIncomplete = false; + let ollamaCleanup: OllamaUnloadResult | undefined; + let ollamaCleanupError: Error | undefined; + if (opts.cleanupOllamaModels !== false) { + try { + const unloadOllamaModels = opts.unloadOllamaModels ?? unloadDefaultOllamaModels; + const cleanup = unloadOllamaModels(); + if (cleanup) ollamaCleanup = cleanup; + if (cleanup && !cleanup.ok) { + ollamaCleanupIncomplete = true; + warn( + `Ollama model cleanup failed at ${cleanup.endpoint} (${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was retained; ${ + cleanup.outcome === "discovery-failed" + ? `restore access to ${cleanup.endpoint}` + : cleanup.outcome === "still-resident" + ? `stop the recorded model at ${cleanup.endpoint}` + : `allow the model unload request at ${cleanup.endpoint}` + }, then retry this command.`, + ); + } else if (sandboxName) { + (opts.clearPendingOllamaModelCleanup ?? clearDefaultPendingOllamaModelCleanup)(sandboxName); + } + } catch (error) { + ollamaCleanupIncomplete = true; + const detail = (error instanceof Error ? error.message : String(error)) + .replace(/\s+/g, " ") + .trim() + .slice(0, 300); + ollamaCleanupError = new Error( + `Ollama model cleanup failed unexpectedly: ${detail || "unknown error"}. ` + + "The saved local route was retained; restore access to the saved local Ollama " + + "endpoint, then retry this command.", + { cause: error }, + ); + warn(ollamaCleanupError.message); + } } + const finishOllamaCleanup = (): OllamaUnloadResult | void => { + if (ollamaCleanupError) throw ollamaCleanupError; + return ollamaCleanup; + }; // Stop host-side services only when their state directory is explicit or // derived from a trusted sandbox name. An invalid requested sandbox must not @@ -560,15 +603,20 @@ export function stopAll(opts: ServiceOptions = {}): void { "Hint: rerun with NEMOCLAW_GATEWAY_PORT= to release that gateway, or 'openshell gateway list' to find it.", ); info("Host services stopped; managed gateway not released."); - return; + return finishOllamaCleanup(); } if (gatewayOutcome === "unconfirmed") { info("Host services stopped; managed gateway release was not confirmed."); - return; + return finishOllamaCleanup(); } - info("All services stopped."); + if (ollamaCleanupIncomplete) { + info("Host services stopped; Ollama model cleanup remains incomplete."); + } else { + info("All services stopped."); + } + return finishOllamaCleanup(); } /** diff --git a/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts b/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts index b928fe8ef3e..5d72eca699e 100644 --- a/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts +++ b/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts @@ -20,6 +20,7 @@ describe("PR review advisor security boundaries", () => { const credentialEnv = "PR_REVIEW_ADVISOR_TEST_API_KEY"; vi.stubEnv(credentialEnv, "test-secret"); const configDir = fs.mkdtempSync(path.join(ROOT, ".tmp-pr-advisor-config-")); + vi.spyOn(ModelRegistry.prototype, "find").mockReturnValue(undefined); try { await expect( diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index f00767858f1..9bd89e2a6dc 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -452,7 +452,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and "--rm", "--add-host", "host.openshell.internal:host-gateway", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-s", "-o", "/dev/null", @@ -480,7 +480,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and "--rm", "--add-host", "host.openshell.internal:host-gateway", - "curlimages/curl:8.10.1", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", "-sf", "--connect-timeout", "3", diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 6c553196f23..327701fb38c 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -6,6 +6,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { + getOllamaApiCommand, + OLLAMA_HOST_DOCKER_INTERNAL, +} from "../../../src/lib/inference/local.ts"; import { assertExitZero, type CommandRunner, @@ -77,6 +81,21 @@ class FakeRunner implements CommandRunner { } describe("E2E fixture clients", () => { + it("uses a digest-pinned image for the live Windows-host transport", () => { + const command = getOllamaApiCommand( + ["-sf", "http://host.docker.internal:11434/api/tags"], + OLLAMA_HOST_DOCKER_INTERNAL, + ); + + expect(command.slice(0, 4)).toEqual([ + "docker", + "run", + "--rm", + expect.stringMatching(/^docker\.io\/curlimages\/curl@sha256:[a-f0-9]{64}$/u), + ]); + expect(command).not.toContain("curlimages/curl:8.10.1"); + }); + it.each([ "a2345678901234567890", "e2e--sandbox", diff --git a/test/inference/ollama/ollama-gpu-cleanup.test.ts b/test/inference/ollama/ollama-gpu-cleanup.test.ts index b2641a3fbb6..e684bb4ec97 100644 --- a/test/inference/ollama/ollama-gpu-cleanup.test.ts +++ b/test/inference/ollama/ollama-gpu-cleanup.test.ts @@ -2,11 +2,24 @@ // SPDX-License-Identifier: Apache-2.0 import type { SpawnSyncReturns } from "node:child_process"; -import { describe, expect, it } from "vitest"; - +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +import { + OLLAMA_HOST_DOCKER_INTERNAL, + persistResolvedOllamaHost, + prepareOllamaApiExecution, + resetOllamaHostCache, +} from "../../../src/lib/inference/local.js"; import { unloadOllamaModels as unloadOllamaModelsImpl } from "../../../src/lib/inference/ollama/proxy.js"; -type SpawnCall = { command: string; args: readonly string[] }; +type SpawnCall = { + command: string; + args: readonly string[]; + options?: { env?: NodeJS.ProcessEnv }; +}; type SpawnSync = (typeof import("node:child_process"))["spawnSync"]; type OllamaModules = { unloadOllamaModels: (typeof import("../../../src/lib/inference/ollama/proxy.ts"))["unloadOllamaModels"]; @@ -40,8 +53,12 @@ function withMockedSpawnSync( ollamaHost = "127.0.0.1", ): T | Promise { const calls: SpawnCall[] = []; - const spawnSync = ((command: string, args: readonly string[]) => { - const call = { command, args }; + const spawnSync = (( + command: string, + args: readonly string[], + options?: { env?: NodeJS.ProcessEnv }, + ) => { + const call = { command, args, options }; calls.push(call); return responder(call); }) as SpawnSync; @@ -80,6 +97,47 @@ function unloadOf(model: string) { } describe("Ollama GPU cleanup", () => { + it("restores the persisted Windows-host transport after the process cache is cleared", () => { + const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-cleanup-route-")); + const calls: SpawnCall[] = []; + const respond = respondWithLoadedModels("llama3.2:1b"); + const spawnSync = ((command: string, args: readonly string[]) => { + const call = { command, args }; + calls.push(call); + return respond(call); + }) as SpawnSync; + + try { + persistResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL, stateRoot); + resetOllamaHostCache(); + const result = unloadOllamaModelsImpl(["llama3.2:1b"], { + ollamaHostStateRoot: stateRoot, + sleep: () => {}, + spawnSync, + }); + + expect(result).toMatchObject({ + ok: true, + outcome: "released", + endpoint: "http://host.docker.internal:11434", + }); + expect(calls).toHaveLength(3); + calls.forEach(({ command, args }) => { + expect(command).toBe("docker"); + expect(args).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + ]), + ); + }); + } finally { + resetOllamaHostCache(); + rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("uses the resolved local Ollama host for discovery, release, and verification (#10074)", async () => { await withMockedSpawnSync( respondWithLoadedModels("llama3.2:1b"), @@ -91,18 +149,93 @@ describe("Ollama GPU cleanup", () => { outcome: "released", endpoint: "http://host.docker.internal:11434", }); - expect( - calls.filter(({ command }) => command === "curl").map(({ args }) => args.at(-1)), - ).toEqual([ + const dockerCalls = calls.filter(({ command }) => command === "docker"); + expect(dockerCalls).toHaveLength(3); + expect(dockerCalls.map(({ args }) => args.at(-1))).toEqual([ "http://host.docker.internal:11434/api/ps", "http://host.docker.internal:11434/api/generate", "http://host.docker.internal:11434/api/ps", ]); + dockerCalls.forEach(({ args }) => { + expect(args).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + ]), + ); + }); }, "host.docker.internal", ); }); + it("retains cleanup recovery when no local Ollama endpoint is reachable", () => { + const result = unloadOllamaModelsImpl(["llama3.2:1b"], { + getResolvedOllamaHost: () => null, + }); + + expect(result).toMatchObject({ + ok: false, + outcome: "discovery-failed", + endpoint: "http://127.0.0.1:11434", + selectedModels: ["llama3.2:1b"], + discoveries: [], + requests: [], + message: "No reachable local Ollama endpoint was found for cleanup", + }); + }); + + it("isolates Docker credentials for discovery, release, and verification", () => { + const calls: SpawnCall[] = []; + const cleanup = vi.fn(() => ({ ok: true as const })); + const respond = respondWithLoadedModels("llama3.2:1b"); + const spawnSync = (( + command: string, + args: readonly string[], + options?: { env?: NodeJS.ProcessEnv }, + ) => { + const call = { command, args, options }; + calls.push(call); + return options?.env?.DOCKER_CONFIG === "/tmp/credential-free-docker" + ? respond(call) + : fail("ambient Docker config used"); + }) as SpawnSync; + + const result = unloadOllamaModelsImpl(["llama3.2:1b"], { + getResolvedOllamaHost: () => OLLAMA_HOST_DOCKER_INTERNAL, + sleep: () => {}, + spawnSync, + prepareOllamaApiExecution: ( + command: Parameters[0], + host: Parameters[1], + options: NonNullable[2]>, + ) => + prepareOllamaApiExecution(command, host, { + ...options, + prepareDockerEnvironment: () => ({ + env: { DOCKER_CONFIG: "/tmp/credential-free-docker" }, + isolatedCredentialConfig: true, + cleanup, + }), + }), + }); + + expect(result).toMatchObject({ ok: true, outcome: "released" }); + expect(calls).toHaveLength(3); + expect(calls.map(({ options }) => options?.env?.DOCKER_CONFIG)).toEqual([ + "/tmp/credential-free-docker", + "/tmp/credential-free-docker", + "/tmp/credential-free-docker", + ]); + expect(calls.map(({ args }) => args.at(-1))).toEqual([ + "http://host.docker.internal:11434/api/ps", + "http://host.docker.internal:11434/api/generate", + "http://host.docker.internal:11434/api/ps", + ]); + expect(cleanup).toHaveBeenCalledTimes(3); + }); + it("unloads every running model through the Ollama API", async () => { await withMockedSpawnSync( respondWithLoadedModels("llama3.1:8b", "qwen:7b"), diff --git a/test/inference/ollama/ollama-pull-timeout.test.ts b/test/inference/ollama/ollama-pull-timeout.test.ts index a61f95a556a..389753077af 100644 --- a/test/inference/ollama/ollama-pull-timeout.test.ts +++ b/test/inference/ollama/ollama-pull-timeout.test.ts @@ -100,7 +100,14 @@ pullOllamaModel("qwen3.5:9b") expect(result.status, result.stderr).toBe(0); const payload = JSON.parse(result.stdout.trim()); expect(payload.ok).toBe(true); - expect(payload.captured.cmd).toBe("curl"); + expect(payload.captured.cmd).toBe("docker"); + expect(payload.captured.args).toEqual( + expect.arrayContaining([ + "run", + "--rm", + "docker.io/curlimages/curl@sha256:d9b4541e214bcd85196d6e92e2753ac6d0ea699f0af5741f8c6cccbfcf00ef4b", + ]), + ); const maxTimeIndex = payload.captured.args.indexOf("--max-time"); expect(maxTimeIndex).toBeGreaterThanOrEqual(0); expect(payload.captured.args[maxTimeIndex + 1]).toBe("0.5"); diff --git a/test/onboarding/onboard-host-local-inference-routing.test.ts b/test/onboarding/onboard-host-local-inference-routing.test.ts index a8a3196671d..5e8f1a7ea0b 100644 --- a/test/onboarding/onboard-host-local-inference-routing.test.ts +++ b/test/onboarding/onboard-host-local-inference-routing.test.ts @@ -471,7 +471,6 @@ describe("onboard host-local inference routing", () => { const route = fixture(application, "ollama"); const legacyRun = vi.fn(); const legacyValidate = vi.fn(); - const legacyWarmup = vi.fn(); const legacyOllamaProof = vi.fn(); const verify = vi.fn(() => { route.events.push("gateway-route-verify"); @@ -495,10 +494,11 @@ describe("onboard host-local inference routing", () => { applyLocalInferenceRoute: undefined, run: legacyRun, validateLocalProvider: legacyValidate, - getOllamaWarmupCommand: legacyWarmup, localInference: { validateOllamaModelWithToolsOverride: legacyOllamaProof, validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, + persistResolvedOllamaHost: () => () => {}, }, verifyInferenceRoute: verify, verifyOnboardInferenceSmoke: smoke, @@ -579,7 +579,6 @@ describe("onboard host-local inference routing", () => { ); expect(legacyRun).not.toHaveBeenCalled(); expect(legacyValidate).not.toHaveBeenCalled(); - expect(legacyWarmup).not.toHaveBeenCalled(); expect(legacyOllamaProof).not.toHaveBeenCalled(); expect(route.gatewayRollback).not.toHaveBeenCalled(); }, diff --git a/test/onboarding/onboard-inference-reconciliation.test.ts b/test/onboarding/onboard-inference-reconciliation.test.ts index 87c74264391..9d25955a331 100644 --- a/test/onboarding/onboard-inference-reconciliation.test.ts +++ b/test/onboarding/onboard-inference-reconciliation.test.ts @@ -9,7 +9,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createLocalInferenceRouteApplier } from "../../src/lib/onboard/local-inference-route.js"; -import type { SetupInference } from "../../src/lib/onboard/setup-inference.js"; +import type { SetupInference, SetupInferenceDeps } from "../../src/lib/onboard/setup-inference.js"; import { writeOkOpenshell } from "../helpers/onboard-openshell-fixture"; import { bedrockRuntimeOnboard, @@ -1011,13 +1011,31 @@ console.log(JSON.stringify({ }); describe("re-onboard Ollama GPU release (#9110)", () => { - const priorEntry = { name: "test-box", provider: "ollama-local", model: "llama3" }; + type ReleaseEntry = { + name: string; + provider: string; + model: string; + endpointUrl?: string | null; + }; + const priorEntry: ReleaseEntry = { + name: "test-box", + provider: "ollama-local", + model: "llama3", + }; function releaseHarness(options: { - getSandbox: () => typeof priorEntry | null; - sandboxes: (typeof priorEntry)[]; - unloadOllamaModels: (onlyModels: readonly string[]) => void; + getSandbox: () => ReleaseEntry | null; + sandboxes: ReleaseEntry[] | (() => ReleaseEntry[]); + unloadOllamaModels: NonNullable; applyLocalInferenceRoute?: () => Promise; + loadPersistedOllamaHost?: () => "127.0.0.1" | "host.docker.internal" | null; + clearPersistedOllamaHostIfUnused?: SetupInferenceDeps["localInference"]["clearPersistedOllamaHostIfUnused"]; + loadPendingOllamaModelCleanup?: (sandboxName: string) => readonly string[]; + persistPendingOllamaModelCleanup?: (sandboxName: string, models: readonly string[]) => void; + clearPendingOllamaModelCleanup?: ( + sandboxName: string, + releasedModels?: readonly string[], + ) => void; }) { return createDirectSetupInferenceHarness({ runOpenshell: (args) => @@ -1032,8 +1050,23 @@ describe("re-onboard Ollama GPU release (#9110)", () => { persistAndProbeOllamaProxy: async () => {}, applyLocalInferenceRoute: options.applyLocalInferenceRoute, getSandbox: options.getSandbox, - listSandboxes: () => ({ sandboxes: options.sandboxes, defaultSandbox: null }), + listSandboxes: () => ({ + sandboxes: + typeof options.sandboxes === "function" ? options.sandboxes() : options.sandboxes, + defaultSandbox: null, + }), unloadOllamaModels: options.unloadOllamaModels, + localInference: { + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + validateSandboxFacingOllamaModel: () => ({ ok: true }), + runOllamaWarmup: () => {}, + persistResolvedOllamaHost: () => () => {}, + loadPersistedOllamaHost: options.loadPersistedOllamaHost, + clearPersistedOllamaHostIfUnused: options.clearPersistedOllamaHostIfUnused, + loadPendingOllamaModelCleanup: options.loadPendingOllamaModelCleanup ?? (() => []), + persistPendingOllamaModelCleanup: options.persistPendingOllamaModelCleanup ?? (() => {}), + clearPendingOllamaModelCleanup: options.clearPendingOllamaModelCleanup ?? (() => {}), + }, }, }); } @@ -1054,6 +1087,24 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).toHaveBeenCalledWith(["llama3"]); }); + it("retires the final Ollama route receipt after switching providers", async () => { + const clearPersistedOllamaHostIfUnused = vi.fn(() => true); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }], + unloadOllamaModels: vi.fn(), + clearPersistedOllamaHostIfUnused, + }); + + await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual({ + ok: true, + }); + + expect(clearPersistedOllamaHostIfUnused).toHaveBeenCalledWith([ + { ...priorEntry, provider: "vllm-local", model: "vllm-model" }, + ]); + }); + it("keeps the successful route when the superseded model unload fails (#9110)", async () => { const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(() => { throw new Error("synthetic unload failure"); @@ -1067,6 +1118,8 @@ describe("re-onboard Ollama GPU release (#9110)", () => { let result: Awaited>; try { result = await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("synthetic unload failure")); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("retry only the recorded models")); } finally { warn.mockRestore(); } @@ -1074,6 +1127,122 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).toHaveBeenCalledWith(["llama3"]); }); + it("reports structured cleanup failure after a successful provider switch", async () => { + const unloadOllamaModels = vi.fn(() => ({ + ok: false as const, + outcome: "unload-request-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: ["llama3"], + discoveries: [], + requests: [], + message: "connection refused", + })); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [priorEntry], + unloadOllamaModels, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + let result: Awaited>; + try { + result = await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("http://host.docker.internal:11434"), + ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("unload-request-failed")); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("Allow the model unload request")); + const warning = warn.mock.calls.map(([message]) => String(message)).join("\n"); + expect(warning).toContain("Re-run onboarding or destroy 'test-box'"); + expect(warning).not.toContain("stop, or destroy"); + } finally { + warn.mockRestore(); + } + expect(result).toEqual({ ok: true }); + }); + + it("persists a failed superseded cleanup and retries only that model on re-onboard", async () => { + let current = priorEntry; + let pending: readonly string[] = []; + const persistPendingOllamaModelCleanup = vi.fn((_sandboxName, models: readonly string[]) => { + pending = models; + }); + const clearPendingOllamaModelCleanup = vi.fn( + (_sandboxName, releasedModels?: readonly string[]) => { + pending = releasedModels ? pending.filter((model) => !releasedModels.includes(model)) : []; + }, + ); + const unloadOllamaModels = vi + .fn>() + .mockReturnValueOnce({ + ok: false, + outcome: "unload-request-failed", + endpoint: "http://host.docker.internal:11434", + selectedModels: ["llama3"], + discoveries: [], + requests: [], + message: "connection refused", + }) + .mockReturnValueOnce(undefined); + const harness = releaseHarness({ + getSandbox: () => current, + sandboxes: () => [current], + unloadOllamaModels, + loadPendingOllamaModelCleanup: () => pending, + persistPendingOllamaModelCleanup, + clearPendingOllamaModelCleanup, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + expect(pending).toEqual(["llama3"]); + current = { ...priorEntry, model: "qwen3.5:9b" }; + + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + } finally { + warn.mockRestore(); + } + + expect(unloadOllamaModels).toHaveBeenNthCalledWith(1, ["llama3"]); + expect(unloadOllamaModels).toHaveBeenNthCalledWith(2, ["llama3"]); + expect(clearPendingOllamaModelCleanup).toHaveBeenCalledWith("test-box", ["llama3"]); + expect(pending).toEqual([]); + }); + + it("names manual cleanup when a superseded-model retry record cannot be written", async () => { + const persistPendingOllamaModelCleanup = vi.fn(() => { + throw new Error("state directory is unavailable"); + }); + const unloadOllamaModels = vi.fn(() => ({ + ok: false as const, + outcome: "unload-request-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: ["llama3"], + discoveries: [], + requests: [], + message: "connection refused", + })); + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [priorEntry], + unloadOllamaModels, + persistPendingOllamaModelCleanup, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + const warning = warn.mock.calls.map(([message]) => String(message)).join("\n"); + expect(warning).toContain("Manually release only llama3"); + expect(warning).toContain("http://host.docker.internal:11434"); + expect(warning).not.toContain("Re-run onboarding, stop, or destroy"); + } finally { + warn.mockRestore(); + } + + expect(persistPendingOllamaModelCleanup.mock.invocationCallOrder[0]).toBeLessThan( + unloadOllamaModels.mock.invocationCallOrder[0] ?? 0, + ); + }); + it("keeps the model when the re-onboard selects the same one (#9110)", async () => { const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); const prior = { ...priorEntry, model: "qwen3.5:9b" }; @@ -1108,6 +1277,31 @@ describe("re-onboard Ollama GPU release (#9110)", () => { expect(unloadOllamaModels).not.toHaveBeenCalled(); }); + it("keeps the route and shared model for a compatible local Ollama peer", async () => { + const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(); + const clearPersistedOllamaHostIfUnused = vi.fn(() => true); + const peer: ReleaseEntry = { + name: "peer", + provider: "compatible-endpoint", + model: "llama3:latest", + endpointUrl: "http://127.0.0.1:11434/v1", + }; + const harness = releaseHarness({ + getSandbox: () => priorEntry, + sandboxes: [{ ...priorEntry, provider: "vllm-local", model: "vllm-model" }, peer], + unloadOllamaModels, + loadPersistedOllamaHost: () => "127.0.0.1", + clearPersistedOllamaHostIfUnused, + }); + + await expect(harness.setupInference("test-box", "vllm-model", "vllm-local")).resolves.toEqual({ + ok: true, + }); + + expect(unloadOllamaModels).not.toHaveBeenCalled(); + expect(clearPersistedOllamaHostIfUnused).not.toHaveBeenCalled(); + }); + it("reads the prior route and releases the model inside the sandbox mutation lock (#9110)", async () => { const events: string[] = []; const unloadOllamaModels = vi.fn<(onlyModels: readonly string[]) => void>(() => { diff --git a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts index 2bdb5ab6e5c..9eda677a291 100644 --- a/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts +++ b/test/runtime/sandbox/destroy-cleanup-sandbox-services.test.ts @@ -2,51 +2,81 @@ // SPDX-License-Identifier: Apache-2.0 // // Regression guard for #2717: cleanupSandboxServices must invoke -// `unloadOllamaModels()` exactly once across both branches of the destroy -// flow — never zero (orphans GPU memory) and never twice (the original -// duplicate-call bug). Mirrors the structural argument captured in the -// inline comments in `src/lib/actions/sandbox/destroy.ts`. +// `unloadOllamaModels()` exactly once across both branches when the sandbox +// owns Ollama cleanup work, and never for an unrelated provider. This avoids +// both orphaned GPU memory and the original duplicate-call bug. import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { CleanupSandboxServicesDeps } from "../../../src/lib/actions/sandbox/destroy.js"; import { cleanupSandboxServices } from "../../../src/lib/actions/sandbox/destroy.js"; +import { ollamaModelRefsMatch } from "../../../src/lib/inference/ollama/model-discovery.js"; +import type { OllamaUnloadResult } from "../../../src/lib/inference/ollama/proxy.js"; import { SANDBOX_PROVIDER_SUFFIXES } from "../../../src/lib/onboard/sandbox-provider-cleanup.js"; -type SandboxLike = { provider?: string | null } | null; +type SandboxLike = { name?: string; model?: string | null; provider?: string | null } | null; +type StopAllOptions = { + sandboxName: string; + cleanupOllamaModels?: boolean; + unloadOllamaModels?: () => OllamaUnloadResult | void; +}; -function buildDeps(sandbox: SandboxLike): { +function buildDeps( + sandbox: SandboxLike, + peers: Exclude[] = [], +): { deps: Required< Pick< CleanupSandboxServicesDeps, | "getSandbox" + | "listSandboxes" | "stopAll" | "unloadOllamaModels" + | "loadPendingOllamaModelCleanup" + | "clearPendingOllamaModelCleanup" + | "withOllamaModelOwnershipLock" + | "ollamaModelRefsMatch" | "runOpenshell" | "rmSync" | "stopGooglechatWebhookTunnel" | "googlechatWebhookTunnelPidDir" > >; - stopAllCalls: Array<{ sandboxName: string }>; + stopAllCalls: StopAllOptions[]; unloadCalls: number; + unloadArgs: Array; } { - const stopAllCalls: Array<{ sandboxName: string }> = []; + const stopAllCalls: StopAllOptions[] = []; + const target = sandbox + ? { name: "regression-2717", model: "target-model:latest", ...sandbox } + : null; let unloadCalls = 0; + const unloadArgs: Array = []; return { stopAllCalls, + unloadArgs, get unloadCalls() { return unloadCalls; }, deps: { - getSandbox: vi.fn(() => sandbox as never), - stopAll: vi.fn((opts: { sandboxName: string }) => { + getSandbox: vi.fn(() => target as never), + listSandboxes: vi.fn(() => ({ + sandboxes: [...(target ? [target] : []), ...peers] as never, + defaultSandbox: null, + })), + stopAll: vi.fn((opts: StopAllOptions) => { stopAllCalls.push(opts); + return opts.cleanupOllamaModels === false ? undefined : opts.unloadOllamaModels?.(); }), - unloadOllamaModels: vi.fn(() => { + unloadOllamaModels: vi.fn((onlyModels?: readonly string[]) => { unloadCalls += 1; + unloadArgs.push(onlyModels); }), + loadPendingOllamaModelCleanup: vi.fn(() => []), + clearPendingOllamaModelCleanup: vi.fn(), + withOllamaModelOwnershipLock: (operation) => operation(), + ollamaModelRefsMatch, runOpenshell: vi.fn(() => ({ status: 0 })), rmSync: vi.fn(), stopGooglechatWebhookTunnel: vi.fn(() => "/tmp/nemoclaw-services-regression-2717-googlechat"), @@ -56,17 +86,84 @@ function buildDeps(sandbox: SandboxLike): { } describe("cleanupSandboxServices Ollama unload (#2717)", () => { + const cleanupFailure = { + ok: false as const, + outcome: "discovery-failed" as const, + endpoint: "http://host.docker.internal:11434", + selectedModels: [], + discoveries: [], + requests: [], + message: "could not connect", + }; + it("delegates GPU unload to stopAll() exactly once when stopHostServices=true", () => { const harness = buildDeps({ provider: "ollama-local" }); cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); expect(harness.deps.stopAll).toHaveBeenCalledTimes(1); - expect(harness.stopAllCalls[0]).toEqual({ sandboxName: "regression-2717" }); - // stopAll() invokes unloadOllamaModels() internally — see services.ts. - // cleanupSandboxServices itself must not call it again. + expect(harness.stopAllCalls[0]).toEqual( + expect.objectContaining({ + sandboxName: "regression-2717", + cleanupOllamaModels: true, + unloadOllamaModels: expect.any(Function), + }), + ); + expect(harness.deps.unloadOllamaModels).toHaveBeenCalledOnce(); + expect(harness.unloadCalls).toBe(1); + }); + + it("skips host-wide Ollama discovery for a final sandbox with no Ollama ownership", () => { + const harness = buildDeps({ provider: "nvidia-prod" }); + + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + + expect(harness.stopAllCalls).toEqual([ + expect.objectContaining({ + sandboxName: "regression-2717", + cleanupOllamaModels: false, + unloadOllamaModels: expect.any(Function), + }), + ]); expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); - expect(harness.unloadCalls).toBe(0); + }); + + it("keeps host-wide Ollama cleanup enabled for retained model recovery", () => { + const harness = buildDeps({ provider: "nvidia-prod" }); + vi.mocked(harness.deps.loadPendingOllamaModelCleanup).mockReturnValue(["old-model"]); + + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + + expect(harness.stopAllCalls).toEqual([ + expect.objectContaining({ + sandboxName: "regression-2717", + cleanupOllamaModels: true, + unloadOllamaModels: expect.any(Function), + }), + ]); + expect(harness.deps.unloadOllamaModels).toHaveBeenCalledOnce(); + }); + + it("holds model ownership while final host-wide cleanup runs", () => { + const harness = buildDeps({ provider: "ollama-local" }); + let ownershipHeld = false; + harness.deps.withOllamaModelOwnershipLock = vi.fn((operation) => { + ownershipHeld = true; + try { + return operation(); + } finally { + ownershipHeld = false; + } + }); + vi.mocked(harness.deps.unloadOllamaModels).mockImplementation(() => { + expect(ownershipHeld).toBe(true); + }); + + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + + expect(harness.deps.stopAll).toHaveBeenCalledOnce(); + expect(harness.deps.unloadOllamaModels).toHaveBeenCalledOnce(); + expect(ownershipHeld).toBe(false); }); it("calls unloadOllamaModels() exactly once for an Ollama sandbox when stopHostServices=false", () => { @@ -76,9 +173,98 @@ describe("cleanupSandboxServices Ollama unload (#2717)", () => { expect(harness.deps.stopAll).not.toHaveBeenCalled(); expect(harness.deps.unloadOllamaModels).toHaveBeenCalledTimes(1); + expect(harness.unloadArgs).toEqual([["target-model:latest"]]); expect(harness.unloadCalls).toBe(1); }); + it("releases only the destroyed sandbox model when another Ollama sandbox uses a different model", () => { + const harness = buildDeps({ provider: "ollama-local", model: "target-model:latest" }, [ + { name: "peer", provider: "ollama-local", model: "peer-model:latest" }, + ]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.unloadArgs).toEqual([["target-model:latest"]]); + }); + + it("keeps a model that another Ollama sandbox shares", () => { + const harness = buildDeps({ provider: "ollama-local", model: "shared-model" }, [ + { name: "peer", provider: "ollama-local", model: "shared-model:latest" }, + ]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); + }); + + it("retries a pending superseded model after the sandbox route changes", () => { + const harness = buildDeps({ provider: "nvidia-prod", model: "new-model" }); + vi.mocked(harness.deps.loadPendingOllamaModelCleanup).mockReturnValue(["old-model"]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.unloadArgs).toEqual([["old-model"]]); + expect(harness.deps.clearPendingOllamaModelCleanup).toHaveBeenCalledWith("regression-2717", [ + "old-model", + ]); + }); + + it("keeps a pending superseded model that an Ollama peer shares", () => { + const harness = buildDeps({ provider: "nvidia-prod", model: "new-model" }, [ + { name: "peer", provider: "ollama-local", model: "old-model:latest" }, + ]); + vi.mocked(harness.deps.loadPendingOllamaModelCleanup).mockReturnValue(["old-model"]); + + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps); + + expect(harness.deps.unloadOllamaModels).not.toHaveBeenCalled(); + expect(harness.deps.clearPendingOllamaModelCleanup).not.toHaveBeenCalled(); + }); + + it("preserves destroy recovery state when stopAll cannot release Ollama", () => { + const harness = buildDeps({ provider: "ollama-local" }); + vi.mocked(harness.deps.stopAll).mockReturnValue(cleanupFailure); + + expect(() => + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps), + ).toThrow(/saved route were retained.*retry destroy/); + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + }); + + it("preserves destroy recovery state when stopAll throws unexpectedly (#10553)", () => { + const harness = buildDeps({ provider: "ollama-local" }); + const stopError = new Error(`unexpected cleanup failure ${"detail ".repeat(100)}`); + vi.mocked(harness.deps.stopAll).mockImplementation(() => { + throw stopError; + }); + + let thrown: unknown; + try { + cleanupSandboxServices("regression-2717", { stopHostServices: true }, harness.deps); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(Error); + expect(thrown).toMatchObject({ cause: stopError }); + expect((thrown as Error).message).toMatch( + /sandbox 'regression-2717' was deleted.*local registry and cleanup state.*nemoclaw regression-2717 destroy/, + ); + expect((thrown as Error).message.length).toBeLessThan(700); + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + expect(harness.deps.runOpenshell).not.toHaveBeenCalled(); + }); + + it("preserves destroy recovery state when scoped Ollama release fails", () => { + const harness = buildDeps({ provider: "ollama-local" }); + vi.mocked(harness.deps.unloadOllamaModels).mockReturnValue(cleanupFailure); + + expect(() => + cleanupSandboxServices("regression-2717", { stopHostServices: false }, harness.deps), + ).toThrow(/saved route were retained.*retry destroy/); + expect(harness.deps.rmSync).not.toHaveBeenCalled(); + }); + it("skips unloadOllamaModels() entirely for non-Ollama providers", () => { const harness = buildDeps({ provider: "nvidia-prod" });