fix(ollama): diagnose stale runner probe timeouts

src/lib/inference/local-windows-ollama-transport.test.ts

@@ -668,6 +668,21 @@ describe("Windows-host Ollama transport", () => {
     expect(captureEx).toHaveBeenCalledOnce();
   });
 
+  it("omits local systemd guidance for Windows-host Ollama timeouts", () => {
+    setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
+    const capture = respondsOnlyThroughDockerDesktop("/api/show", "");
+
+    const result = validateOllamaModel(
+      "nemotron-3-nano:30b",
+      capture,
+      () => false,
+      () => ({ stdout: "", exitCode: 28, timedOut: true }),
+    );
+
+    expect(result.message).toContain("did not answer the local probe in time");
+    expect(result.message).not.toContain("systemctl");
+  });
+
   it("validates health and container reachability through Docker Desktop (#10553)", () => {
     setResolvedOllamaHost(OLLAMA_HOST_DOCKER_INTERNAL);
     const capture = vi.fn((command: readonly string[]) => {
@@ -920,6 +935,7 @@ describe("Windows-host Ollama transport", () => {
         "docker",
         "run",
         "--rm",
+        "-d",
         CONTAINER_REACHABILITY_IMAGE,
         "http://host.docker.internal:11434/api/generate",
       ]),

src/lib/inference/local.test.ts

@@ -1197,7 +1197,7 @@ describe("local inference helpers", () => {
     const captureEx = () => ({ stdout: "", exitCode: 0, timedOut: false });
     const result = validateOllamaModel("nemotron-3-nano:30b", () => "", undefined, captureEx);
     expect(result.ok).toBe(false);
-    expect(result.message).toMatch(/did not answer the local probe in time/);
+    expect(result.message).toMatch(/failed the local probe without a response/);
   });
 
   it("fails ollama model validation when Ollama returns an error payload", () => {
@@ -1367,7 +1367,7 @@ describe("local inference helpers", () => {
     );
     expect(result.ok).toBe(false);
     expect(callCount).toBe(1);
-    expect(result.message).toMatch(/did not answer the local probe in time/);
+    expect(result.message).toMatch(/failed the local probe without a response/);
   });
 
   it("fails when both probe attempts return empty (model truly unhealthy or too slow)", () => {

src/lib/inference/local.ts

@@ -75,6 +75,7 @@ import {
   resetOllamaRuntimeContextWindowAutoState,
   resolveOllamaRuntimeContextWindow as resolveOllamaRuntimeContextWindowWithHost,
 } from "./ollama-runtime-context";
+import { hasOllamaSystemdUnit } from "./ollama-version";
 import {
   type RecoveredManagedClusterVllmEndpoint,
   recoverInstalledManagedClusterVllmEndpoint,
@@ -1278,7 +1279,7 @@ export function getLocalProviderHealthCheck(provider: string): string[] | null {
   }
   if (!endpoint) return null;
   const curlArgs = buildValidatedCurlCommandArgs(["-sf", endpoint]);
-  return provider === "ollama-local" ? getOllamaApiCommand(curlArgs) : ["curl", ...curlArgs];
+  return ["curl", ...curlArgs];
 }
 
 /**
@@ -2317,7 +2318,10 @@ export function runOllamaWarmup(
     return;
   }
   try {
-    runImpl(execution.command, {
+    const warmupCommand = windowsHost
+      ? [...execution.command.slice(0, 3), "-d", ...execution.command.slice(3)]
+      : execution.command;
+    runImpl(warmupCommand, {
       ignoreError: true,
       ...(execution.env === undefined ? {} : { env: execution.env }),
     });
@@ -2338,10 +2342,10 @@ export function getOllamaProbeCommand(
     keep_alive: keepAlive,
     options: { num_predict: 16 },
   });
-  const host = getResolvedOllamaHost();
-  const endpoint = `http://${host}:${OLLAMA_PORT}/api/generate`;
-  return getOllamaApiCommand(
-    buildValidatedCurlCommandArgs([
+  const endpoint = `http://${getResolvedOllamaHost()}:${OLLAMA_PORT}/api/generate`;
+  return [
+    "curl",
+    ...buildValidatedCurlCommandArgs([
       "-sS",
       "--max-time",
       String(timeoutSeconds),
@@ -2351,8 +2355,7 @@ export function getOllamaProbeCommand(
       payload,
       endpoint,
     ]),
-    host,
-  );
+  ];
 }
 
 export function validateOllamaModel(
@@ -2374,6 +2377,7 @@ export function validateOllamaModel(
   const probeCmd = getOllamaProbeCommand(model);
   const probeResult = captureEx(probeCmd);
   let output = probeResult.stdout;
+  let timedOut = probeResult.timedOut;
   // Cold-loading a large model from disk can routinely exceed the default 120 s
   // probe window — on DGX Spark unified-memory hosts (#3251) and also on
   // tight-VRAM dGPU hosts (e.g. NVIDIA L4 23 GB) where the runner spills GPU→CPU
@@ -2383,13 +2387,24 @@ export function validateOllamaModel(
   if (probeResult.timedOut) {
     const retryResult = captureEx(getOllamaProbeCommand(model, 300));
     output = retryResult.stdout;
+    timedOut = retryResult.timedOut;
   }
   if (!output) {
+    const localDaemon = getResolvedOllamaHost() === OLLAMA_LOCALHOST;
+    const staleRunnerRecovery =
+      timedOut && localDaemon
+        ? " Stale runner processes from a previous model may be holding GPU memory. " +
+          (hasOllamaSystemdUnit(capture)
+            ? "Run 'sudo systemctl restart ollama' and rerun onboarding."
+            : "Restart Ollama and rerun onboarding.")
+        : "";
+    const failure =
+      timedOut === true
+        ? `Selected Ollama model '${model}' did not answer the local probe in time. It may still be loading, too large for the host, or otherwise unhealthy.`
+        : `Selected Ollama model '${model}' failed the local probe without a response. Check that Ollama is running and the model is available.`;
     return {
       ok: false,
-      message:
-        `Selected Ollama model '${model}' did not answer the local probe in time. ` +
-        "It may still be loading, too large for the host, or otherwise unhealthy.",
+      message: failure + staleRunnerRecovery,
     };
   }
 

src/lib/inference/ollama-probe-timeout.test.ts

@@ -1,9 +1,19 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 
-import { validateOllamaModel } from "./local";
+import {
+  OLLAMA_HOST_DOCKER_INTERNAL,
+  OLLAMA_LOCALHOST,
+  setResolvedOllamaHost,
+  validateOllamaModel,
+} from "./local";
+
+afterEach(() => {
+  vi.restoreAllMocks();
+  setResolvedOllamaHost(OLLAMA_LOCALHOST);
+});
 
 describe("Ollama probe timeout retry", () => {
   it("retries with extended timeout on non-Spark hosts when first probe times out", () => {
@@ -27,4 +37,51 @@
     expect(captureExCallCount).toBe(2);
     expect(commands[1]).toMatch(/--max-time.*300|300.*--max-time/);
   });
+
+  it.each([
+    { platform: "linux" as const, systemdUnit: true, systemctl: true },
+    { platform: "darwin" as const, systemdUnit: true, systemctl: false },
+    { platform: "linux" as const, systemdUnit: false, systemctl: false },
+  ])(
+    "selects recovery for $platform with systemd unit $systemdUnit",
+    ({ platform, systemdUnit, systemctl }) => {
+      vi.spyOn(process, "platform", "get").mockReturnValue(platform);
+      const result = validateOllamaModel(
+        "nemotron-3-nano:30b",
+        (command) =>
+          systemdUnit && command.some((argument) => argument.includes("systemctl"))
+            ? "ollama.service enabled"
+            : "",
+        () => false,
+        () => ({ stdout: "", exitCode: 28, timedOut: true }),
+      );
+      const message = result.message ?? "";
+
+      expect({
+        generic: message.includes("Restart Ollama and rerun onboarding"),
+        ok: result.ok,
+        staleRunner: message.includes("Stale runner processes from a previous model"),
+        systemctl: message.includes("systemctl"),
+      }).toEqual({ generic: !systemctl, ok: false, staleRunner: true, systemctl });
+    },
+  );
+
+  it("reports a fast retry failure from the final probe result", () => {
+    let callCount = 0;
+    const result = validateOllamaModel(
+      "nemotron-3-nano:30b",
+      () => "",
+      () => false,
+      () => {
+        callCount += 1;
+        return callCount === 1
+          ? { stdout: "", exitCode: 28, timedOut: true }
+          : { stdout: "", exitCode: 7, timedOut: false };
+      },
+    );
+
+    expect(result.message).toContain("failed the local probe without a response");
+    expect(result.message).not.toContain("Stale runner processes");
+  });
+
 });

src/lib/inference/ollama-version.test.ts

@@ -1,16 +1,46 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 
 import {
   getInstalledOllamaVersion,
   getRunningOllamaDaemonVersion,
+  hasOllamaSystemdUnit,
   isOllamaVersionAtLeast,
   MIN_OLLAMA_VERSION,
 } from "./ollama-version";
 
+const OLLAMA_SYSTEMD_UNIT_PROBE = [
+  "sh",
+  "-c",
+  "command -v systemctl >/dev/null && [ -d /run/systemd/system ] && systemctl list-unit-files ollama.service --no-legend 2>/dev/null | head -n1",
+] as const;
+
 describe("Ollama version detection", () => {
+  it("skips systemd unit detection outside Linux", () => {
+    const capture = vi.fn(() => "ollama.service enabled");
+
+    expect({ detected: hasOllamaSystemdUnit(capture, "darwin"), calls: capture.mock.calls }).toEqual(
+      { detected: false, calls: [] },
+    );
+  });
+
+  it.each([
+    { output: "", detected: false },
+    { output: "ollama.service enabled", detected: true },
+  ])("detects a Linux systemd unit from %j", ({ output, detected }) => {
+    const capture = vi.fn(() => output);
+
+    expect({
+      detected: hasOllamaSystemdUnit(capture, "linux"),
+      calls: capture.mock.calls,
+    }).toEqual({
+      detected,
+      calls: [[OLLAMA_SYSTEMD_UNIT_PROBE, { ignoreError: true, timeout: 5_000 }]],
+    });
+  });
+
   it("parses 'ollama version is X.Y.Z' output", () => {
     const capture = () => "ollama version is 0.6.2";
     expect(getInstalledOllamaVersion(capture)).toBe("0.6.2");

src/lib/inference/ollama-version.ts

@@ -13,9 +13,29 @@ import { runCapture } from "../runner";
 
 export type OllamaVersionRunCapture = (
   cmd: readonly string[],
-  opts?: { ignoreError?: boolean },
+  opts?: { ignoreError?: boolean; timeout?: number },
 ) => string;
 
+const SYSTEMD_UNIT_PROBE_TIMEOUT_MS = 5_000;
+
+/** Return true only when this Linux host has an installed systemd Ollama unit. */
+export function hasOllamaSystemdUnit(
+  capture: OllamaVersionRunCapture,
+  platform: NodeJS.Platform = process.platform,
+): boolean {
+  if (platform !== "linux") return false;
+  return Boolean(
+    capture(
+      [
+        "sh",
+        "-c",
+        "command -v systemctl >/dev/null && [ -d /run/systemd/system ] && systemctl list-unit-files ollama.service --no-legend 2>/dev/null | head -n1",
+      ],
+      { ignoreError: true, timeout: SYSTEMD_UNIT_PROBE_TIMEOUT_MS },
+    ).trim(),
+  );
+}
+
 /**
  * Minimum Ollama version NemoClaw expects when reusing an existing host
  * Ollama. Older Ollama runners can fail to return structured tool calls for

src/lib/onboard/ollama-systemd.ts

@@ -7,6 +7,7 @@ import nodePath from "node:path";
 import { OLLAMA_PORT } from "../core/ports";
 import { sleepSeconds } from "../core/wait";
 import { resolveOllamaContextWindowFloor } from "../inference/ollama-runtime-context";
+import { hasOllamaSystemdUnit } from "../inference/ollama-version";
 import {
   proveOllamaSystemdServiceExecutable,
   type OllamaServiceExecutableProof,
@@ -207,17 +208,9 @@ export function ensureOllamaLoopbackSystemdOverride(
   const platform = (options.platformImpl ?? (() => process.platform))();
   if (platform !== "linux") return "not-applicable";
 
-  const hasOllamaSystemdUnit =
-    options.hasOllamaSystemdUnitImpl?.() ??
-    !!runCapture(
-      [
-        "sh",
-        "-c",
-        "command -v systemctl >/dev/null && [ -d /run/systemd/system ] && systemctl list-unit-files ollama.service --no-legend 2>/dev/null | head -n1",
-      ],
-      { ignoreError: true },
-    ).trim();
-  if (!hasOllamaSystemdUnit) return "not-applicable";
+  const systemdUnitPresent =
+    options.hasOllamaSystemdUnitImpl?.() ?? hasOllamaSystemdUnit(runCapture, platform);
+  if (!systemdUnitPresent) return "not-applicable";
 
   // #5716: detect missing non-interactive sudo before attempting any override
   // command. Continuing is safe only when runtime listener evidence proves