Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
59 commits
Select commit Hold shift + click to select a range
d591a90
fix(inference): keep Windows Ollama requests in Docker
ericksoa Sep 1, 2026
97eaa30
Merge branch 'main' into fix/wsl-ollama-model-discovery-10553
ericksoa Sep 1, 2026
a2e4956
fix(inference): complete Windows Ollama transport
ericksoa Sep 1, 2026
a426ab8
fix(inference): persist Windows Ollama cleanup route
ericksoa Sep 1, 2026
057ebc5
fix(inference): preserve destroy cleanup contract
ericksoa Sep 1, 2026
e4cc146
refactor(inference): finalize Ollama route ownership
ericksoa Sep 1, 2026
a0059be
fix(inference): restore accepted Ollama route
ericksoa Sep 1, 2026
4fc9f3d
fix(inference): complete Ollama route recovery
ericksoa Sep 1, 2026
abc65bd
fix(inference): finalize Ollama lifecycle recovery
ericksoa Sep 1, 2026
3dbf08b
fix(tunnel): scope Ollama cleanup completion
ericksoa Sep 1, 2026
71e39e6
fix(inference): complete Ollama route lifecycle
ericksoa Sep 1, 2026
e191f6a
test(inference): require exact Ollama probe URLs
ericksoa Sep 1, 2026
bed0a10
fix(destroy): preserve failed Ollama cleanup recovery
ericksoa Sep 1, 2026
b35166e
test(inference): focus Ollama transport contracts
ericksoa Sep 1, 2026
a15ea79
fix: clear Windows-host Ollama review gaps
ericksoa Sep 1, 2026
7125b6e
fix: isolate every Windows Ollama operation
ericksoa Sep 1, 2026
6ba00d7
test: require exact Windows Ollama URLs
ericksoa Sep 1, 2026
40c869a
fix: preserve peer Ollama model ownership
ericksoa Sep 1, 2026
1307316
refactor: retire Ollama routes through owner
ericksoa Sep 1, 2026
ed3c43a
fix: persist scoped Ollama cleanup retries
ericksoa Sep 1, 2026
9382e36
fix: propagate Ollama cleanup exceptions
ericksoa Sep 1, 2026
715c315
fix(inference): preserve Ollama cleanup recovery
prekshivyas Sep 1, 2026
38a6327
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
prekshivyas Sep 1, 2026
6eb158b
test: allow policy rollback under coverage
ericksoa Sep 1, 2026
aa89007
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
prekshivyas Sep 1, 2026
e7e8582
fix: preserve scoped Ollama cleanup recovery
ericksoa Sep 1, 2026
b2929f5
test: stabilize readiness deadline budget
ericksoa Sep 1, 2026
5686a33
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
prekshivyas Sep 1, 2026
11855ea
test: use readiness clock fix from base
ericksoa Sep 1, 2026
146fbe0
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
prekshivyas Sep 1, 2026
735203b
fix: retain Ollama cleanup when host is unavailable
ericksoa Sep 1, 2026
b6b3464
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
prekshivyas Sep 1, 2026
7e60520
fix: scope final Ollama cleanup to model owners
ericksoa Sep 1, 2026
c394f83
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
ericksoa Sep 1, 2026
7df5c02
fix: isolate Windows Ollama readiness transport
ericksoa Sep 1, 2026
b1c6ac3
refactor: centralize Windows Ollama model discovery
ericksoa Sep 1, 2026
9a48167
fix: keep Ollama lifecycle transitions atomic
ericksoa Sep 1, 2026
d2968fa
Merge remote-tracking branch 'origin/main' into fix/wsl-ollama-model-…
ericksoa Sep 1, 2026
7051d83
test: remove duplicate readiness clock
ericksoa Sep 1, 2026
edcb7f3
Merge remote-tracking branch 'origin/main' into fix/wsl-ollama-model-…
ericksoa Sep 1, 2026
f059cd0
test: prove Ollama unload runs under ownership lock
ericksoa Sep 1, 2026
fac042b
fix: preserve destroy recovery guidance
prekshivyas Sep 1, 2026
a101ffb
fix: complete Windows Ollama transport migration
ericksoa Sep 1, 2026
27be95f
merge: incorporate concurrent Windows Ollama migration
prekshivyas Sep 1, 2026
c3ac1ce
test: pair immutable probe image with fast E2E
ericksoa Sep 1, 2026
284f65a
merge: incorporate concurrent branch updates
prekshivyas Sep 1, 2026
6f3827e
test: expect immutable probe image guidance
ericksoa Sep 1, 2026
301d64b
test: isolate sandbox service tests from Ollama
ericksoa Sep 1, 2026
09b199f
fix: use multi-platform Ollama probe image
ericksoa Sep 2, 2026
afde82a
test: verify probe pin through transport
ericksoa Sep 2, 2026
5142e5c
fix: serialize Ollama route ownership transitions
ericksoa Sep 2, 2026
6032621
test: guard retired Windows sleep path
prekshivyas Sep 2, 2026
1cb5f9d
test: allow bounded Windows readiness waits
ericksoa Sep 2, 2026
e43ae80
test: focus Windows fallback on behavior
prekshivyas Sep 2, 2026
ebc31a2
test: reject subprocess Windows readiness sleeps
ericksoa Sep 2, 2026
f7975ea
test: isolate missing advisor model lookup
ericksoa Sep 2, 2026
2587eff
fix: protect local Ollama route ownership
prekshivyas Sep 2, 2026
da9fbfc
test: verify bounded Windows readiness delays
ericksoa Sep 2, 2026
2b5cd70
Merge remote-tracking branch 'origin/fix/wsl-ollama-model-discovery-1…
ericksoa Sep 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => {
expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe(
`http://host.docker.internal:${OLLAMA_PORT}/api/ps`,
);
expect(runCaptureImpl.mock.calls[0][0][0]).toBe("docker");
expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe(
`http://host.docker.internal:${OLLAMA_PORT}/api/generate`,
);
expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("docker");
expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({
model: "qwen3.6:35b",
stream: false,
Expand All @@ -89,9 +91,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => {
expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe(
`http://127.0.0.1:${OLLAMA_PORT}/api/ps`,
);
expect(runCaptureImpl.mock.calls[0][0][0]).toBe("curl");
expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe(
`http://127.0.0.1:${OLLAMA_PORT}/api/generate`,
);
expect(runCaptureExImpl.mock.calls[0][0][0]).toBe("curl");
});

it("falls back to an allowlisted host instead of probing an arbitrary registry URL", () => {
Expand Down Expand Up @@ -228,7 +232,7 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => {
endpoint: `http://host.docker.internal:${OLLAMA_PORT}`,
inventoryLabel: "llama3.2:1b",
});
expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", undefined);
expect(probeModelInventory).toHaveBeenCalledWith("host.docker.internal", expect.any(Function));
});

it("keeps the warm failure when the daemon does hold the model (#9455)", () => {
Expand Down
26 changes: 19 additions & 7 deletions src/lib/actions/sandbox/agent/ollama-restart-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args"
import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports";
import {
describeModelInventory,
getOllamaApiCommand,
getResolvedOllamaHost,
ollamaInventoryContainsModel,
OLLAMA_HOST_DOCKER_INTERNAL,
Expand All @@ -30,7 +31,7 @@ import {
type OllamaRuntimeRunCaptureFn,
probeOllamaRuntimeModelStatus,
} from "../../../inference/ollama-runtime-context";
import { runCaptureEx } from "../../../runner";
import { runCapture, runCaptureEx } from "../../../runner";

export interface OllamaRestartRecoveryRoute {
provider?: string | null;
Expand Down Expand Up @@ -150,9 +151,8 @@ function buildWarmCommand(model: string, hostname: string): string[] {
keep_alive: "15m",
options: { num_predict: 16 },
});
return [
"curl",
...buildValidatedCurlCommandArgs([
return getOllamaApiCommand(
buildValidatedCurlCommandArgs([
"-sS",
"--connect-timeout",
"3",
Expand All @@ -164,7 +164,18 @@ function buildWarmCommand(model: string, hostname: string): string[] {
body,
`http://${hostname}:${OLLAMA_PORT}/api/generate`,
]),
];
hostname,
);
}

function createRawOllamaCapture(
hostname: string,
capture: OllamaRuntimeRunCaptureFn,
): OllamaRuntimeRunCaptureFn {
return (command, options) => {
const [executable, ...args] = command;
return capture(executable === "curl" ? getOllamaApiCommand(args, hostname) : command, options);
};
}

function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" {
Expand Down Expand Up @@ -209,9 +220,10 @@ export function maybeWarmOllamaAfterDaemonRestart(
const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost;
const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost);
const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus;
const rawCapture = createRawOllamaCapture(rawHost, deps.runCaptureImpl ?? runCapture);
let status: OllamaRuntimeModelStatus;
try {
status = probe(model, () => rawHost, deps.runCaptureImpl);
status = probe(model, () => rawHost, rawCapture);
} catch {
return { kind: "skipped", reason: "unreachable" };
}
Expand Down Expand Up @@ -239,7 +251,7 @@ export function maybeWarmOllamaAfterDaemonRestart(
// an unreadable inventory keeps the original warm-failure reason.
if (response === "ollama-error") {
const probeInventory = deps.probeModelInventory ?? probeOllamaEndpointInventory;
const inventory = probeInventory(rawHost, deps.runCaptureImpl);
const inventory = probeInventory(rawHost, rawCapture);
if (inventory && !ollamaInventoryContainsModel(inventory, model)) {
return {
kind: "skipped",
Expand Down
31 changes: 23 additions & 8 deletions src/lib/actions/sandbox/destroy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
revokeHttpsPinRuntimeAdapterRoute,
} from "../../inference/https-pin-runtime-adapter";
import { prepareManagedLlamaCppRuntimeCleanupForSandbox } from "../../inference/local-model-profile/cleanup";
import type { OllamaUnloadResult } from "../../inference/ollama/proxy";
import {
CURRENT_RUNTIME_PROVIDER_BUNDLES,
normalizeRuntimeProviderIdentity,
Expand Down Expand Up @@ -165,8 +166,8 @@ type RunOpenshell = (args: string[], opts?: Record<string, unknown>) => { status

export type CleanupSandboxServicesDeps = {
getSandbox?: typeof registry.getSandbox;
stopAll?: (opts: { sandboxName: string }) => void;
unloadOllamaModels?: () => void;
stopAll?: (opts: { sandboxName: string }) => OllamaUnloadResult | void;
unloadOllamaModels?: () => OllamaUnloadResult | void;
runOpenshell?: RunOpenshell;
rmSync?: typeof fs.rmSync;
stopGooglechatWebhookTunnel?: (sandboxName: string) => string;
Expand Down Expand Up @@ -227,17 +228,17 @@ export function cleanupSandboxServices(
deps.stopAll ??
((opts: { sandboxName: string }) => {
const services = require("../../tunnel/services") as {
stopAll: (opts: { sandboxName: string }) => void;
stopAll: (opts: { sandboxName: string }) => OllamaUnloadResult | void;
};
services.stopAll(opts);
return services.stopAll(opts);
});
const unloadOllamaModels =
deps.unloadOllamaModels ??
(() => {
const { unloadOllamaModels: unload } = require("../../inference/ollama/proxy") as {
unloadOllamaModels: () => void;
unloadOllamaModels: () => OllamaUnloadResult;
};
unload();
return unload();
});
const runOpenshell =
deps.runOpenshell ??
Expand Down Expand Up @@ -289,14 +290,28 @@ export function cleanupSandboxServices(
if (stopHostServices) {
// `stopAll()` already runs `unloadOllamaModels()` unconditionally —
// see src/lib/tunnel/services.ts. Don't double-call here.
stopAll({ sandboxName: validatedSandboxName });
const cleanup = stopAll({ sandboxName: validatedSandboxName });
if (cleanup && !cleanup.ok) {
throw new Error(
`Sandbox host services stopped, but Ollama model cleanup failed at ${cleanup.endpoint} ` +
`(${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was ` +
"retained; repair Ollama and retry destroy.",
);
}
} else {
// No global stop, so `stopAll()` did not run; explicitly free Ollama
// models for this sandbox if its provider used Ollama. Without this
// branch a single-sandbox destroy would leave models loaded on the GPU.
const sb = getSandbox(validatedSandboxName);
if (sb?.provider?.includes("ollama")) {
unloadOllamaModels();
const cleanup = unloadOllamaModels();
if (cleanup && !cleanup.ok) {
throw new Error(
`Sandbox resources were removed, but Ollama model cleanup failed at ${cleanup.endpoint} ` +
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
`(${cleanup.outcome}: ${cleanup.message ?? "no detail"}). The saved local route was ` +
"retained; repair Ollama and retry destroy.",
);
}
}
}

Expand Down
214 changes: 214 additions & 0 deletions src/lib/inference/local-windows-ollama-transport.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";

import {
applyOllamaRuntimeContextWindow,
CONTAINER_REACHABILITY_IMAGE,
getLocalProviderHealthCheck,
getOllamaHostForCleanup,
getOllamaModelOptions,
getOllamaProbeCommand,
getOllamaWarmupRequestCommand,
OLLAMA_HOST_DOCKER_INTERNAL,
loadPersistedOllamaHost,
persistResolvedOllamaHost,
probeOllamaModelCapabilities,
resetOllamaHostCache,
resetOllamaRuntimeContextWindowAutoState,
setResolvedOllamaHost,
validateLocalProvider,
} from "./local";

describe("Windows-host Ollama transport", () => {
afterEach(() => {
resetOllamaHostCache();
resetOllamaRuntimeContextWindowAutoState();
});

it("restores the accepted route for cleanup in a fresh process", () => {
const stateRoot = mkdtempSync(join(tmpdir(), "nemoclaw-ollama-host-receipt-"));
try {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
persistResolvedOllamaHost(undefined, stateRoot);
resetOllamaHostCache();

expect(loadPersistedOllamaHost(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL);
expect(getOllamaHostForCleanup(stateRoot)).toBe(OLLAMA_HOST_DOCKER_INTERNAL);
} finally {
rmSync(stateRoot, { recursive: true, force: true });
}
});

it("reads the model inventory through Docker Desktop (#10553)", () => {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
const capture = vi.fn(() => JSON.stringify({ models: [{ name: "qwen3.5:9b" }] }));

expect(getOllamaModelOptions(capture)).toEqual(["qwen3.5:9b"]);
expect(capture).toHaveBeenCalledWith(
[
"docker",
"run",
"--rm",
CONTAINER_REACHABILITY_IMAGE,
"-sf",
"--connect-timeout",
"3",
"--max-time",
"5",
"http://host.docker.internal:11434/api/tags",
],
{ ignoreError: true },
);
});

it("builds warm-up and validation requests for Docker Desktop (#10553)", () => {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);

expect(getOllamaWarmupRequestCommand("qwen3.5:9b")).toEqual([
"docker",
"run",
"--rm",
CONTAINER_REACHABILITY_IMAGE,
"-s",
"--connect-timeout",
"10",
"--max-time",
"120",
"http://host.docker.internal:11434/api/generate",
"-H",
"Content-Type: application/json",
"-d",
expect.stringContaining('"model":"qwen3.5:9b"'),
]);

expect(getOllamaProbeCommand("qwen3.5:9b")).toEqual([
"docker",
"run",
"--rm",
CONTAINER_REACHABILITY_IMAGE,
"-sS",
"--max-time",
"120",
"http://host.docker.internal:11434/api/generate",
"-H",
"Content-Type: application/json",
"-d",
expect.stringContaining('"model":"qwen3.5:9b"'),
]);
});

it("validates health and container reachability through Docker Desktop (#10553)", () => {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
const capture = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] }));

expect(getLocalProviderHealthCheck("ollama-local")).toEqual([
"docker",
"run",
"--rm",
CONTAINER_REACHABILITY_IMAGE,
"-sf",
"http://host.docker.internal:11434/api/tags",
]);
expect(
validateLocalProvider(
"ollama-local",
capture,
() => {},
() => ({
env: {},
isolatedCredentialConfig: false,
cleanup: () => ({ ok: true }),
}),
),
).toEqual({ ok: true });

expect(capture).toHaveBeenCalledTimes(2);
expect(capture.mock.calls[0]?.[0]).toEqual(
expect.arrayContaining([
"docker",
"run",
"--rm",
CONTAINER_REACHABILITY_IMAGE,
"http://host.docker.internal:11434/api/tags",
]),
);
expect(capture.mock.calls[1]?.[0]).toEqual(
expect.arrayContaining([
"docker",
"run",
"--rm",
"--add-host",
"host.openshell.internal:host-gateway",
]),
);
});

it("checks the Hermes context window through Docker Desktop (#10553)", () => {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
const capture = vi.fn((command: readonly string[]) =>
String(command.at(-1)).endsWith("/api/ps")
? JSON.stringify({
models: [{ name: "qwen3.5:9b", context_length: 65_536, processor: "100% GPU" }],
})
: "",
);
const env: NodeJS.ProcessEnv = {};

expect(
applyOllamaRuntimeContextWindow("qwen3.5:9b", {
contextWindowFloor: 64_000,
env,
logger: { log: vi.fn(), warn: vi.fn() },
runCaptureImpl: capture,
}),
).toEqual({ ok: true });
expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536");
expect(capture).toHaveBeenCalledOnce();
expect(capture.mock.calls[0]?.[0]).toEqual(
expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]),
);
});

it("checks model capability metadata through Docker Desktop (#10553)", () => {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
const capture = vi.fn((_command: readonly string[]) =>
JSON.stringify({ capabilities: ["tools"] }),
);

expect(probeOllamaModelCapabilities("qwen3.5:9b", capture)).toMatchObject({
source: "api",
supportsTools: true,
});
expect(capture).toHaveBeenCalledOnce();
expect(capture.mock.calls[0]?.[0]).toEqual(
expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]),
);
});

it("keeps the Hermes context-window check fail-closed on an invalid Docker response (#10553)", () => {
setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
const capture = vi.fn((_command: readonly string[]) =>
JSON.stringify({ models: [{ name: "qwen3.5:9b", context_length: "invalid" }] }),
);

const result = applyOllamaRuntimeContextWindow("qwen3.5:9b", {
contextWindowFloor: 64_000,
env: {},
logger: { log: vi.fn(), warn: vi.fn() },
runCaptureImpl: capture,
});

expect(result).toMatchObject({
ok: false,
message: expect.stringContaining("cannot verify the required 64000-token window"),
});
expect(capture.mock.calls[0]?.[0]).toEqual(
expect.arrayContaining(["docker", "run", "--rm", CONTAINER_REACHABILITY_IMAGE]),
);
});
});
3 changes: 2 additions & 1 deletion src/lib/inference/local.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1212,7 +1212,8 @@ describe("local inference helpers", () => {
it("builds a background warmup command for ollama models", () => {
const command = getOllamaWarmupCommand("nemotron-3-nano:30b");
expect(command).toEqual(expect.arrayContaining(["bash", "-c"]));
expect(command[2]).toMatch(/^nohup curl -s http:\/\/127.0.0.1:11434\/api\/generate /);
expect(command[2]).toContain("'--connect-timeout' '10' '--max-time' '120'");
expect(command[2]).toContain("http://127.0.0.1:11434/api/generate");
expect(command[2]).toMatch(/"model":"nemotron-3-nano:30b"/);
expect(command[2]).toMatch(/"keep_alive":"15m"/);
});
Expand Down
Loading
Loading