diff --git a/docs/inference/set-up-ollama.mdx b/docs/inference/set-up-ollama.mdx index 18c8c36f260..b43a9b714e7 100644 --- a/docs/inference/set-up-ollama.mdx +++ b/docs/inference/set-up-ollama.mdx @@ -51,7 +51,20 @@ Set `NEMOCLAW_OLLAMA_INSTALL_MODE=system` or `NEMOCLAW_OLLAMA_INSTALL_MODE=user` The user-local mode downloads and extracts the Ollama release to `${HOME}/.local`, then starts `${HOME}/.local/bin/ollama serve` once. It does not configure a systemd service, create the `ollama` system user, or install CUDA drivers. -Restart the daemon manually after a reboot. +For ordinary user-local installs, restart the daemon manually after a reboot. + + +For a sandbox created with the portable experimental profile, `$$nemoclaw connect --probe-only` and `$$nemoclaw recover` can restart the user-local daemon that NemoClaw installed. +Before it decides whether to start Ollama, recovery probes `http://127.0.0.1:11434/api/tags`. +If the API is unhealthy, recovery starts Ollama only when the sandbox records `ollama-local` and a valid ownership receipt names the fixed regular executable. +It refuses to launch a duplicate when another `ollama` process exists but the API is unhealthy. +After it launches the daemon, it waits up to 30 seconds for `/api/tags` to return valid JSON with a `models` array. +Recovery then requires a trusted reachable result from the sandbox `inference.local` route. +When route repair needs the local Ollama dependency check, it verifies or recovers the authenticated proxy before it retries `inference.local`. +If Ollama does not become healthy within 30 seconds, the command identifies the exact receipt-bound executable and its `serve` argument, then tells you to retry recovery. +The command exits non-zero with recovery guidance when startup or route validation fails. +It does not start, stop, or replace a system service or another user-managed Ollama daemon. + NemoClaw prints a `PATH` hint when `${HOME}/.local/bin` is missing from your `PATH`. Add `export PATH="${HOME}/.local/bin:$PATH"` to your shell profile if you want to invoke `ollama` directly. diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 1b6be097676..f09b1799e53 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -126,6 +126,23 @@ Other workflows, including onboarding, rebuild, and `doctor --fix`, can explicit `$$nemoclaw gateway restart` instead restarts only the supported agent gateway inside the named sandbox. + +### Recover Portable Local Ollama + +For a portable experimental-profile sandbox with the recorded `ollama-local` provider, `connect --probe-only` and `recover` also verify the host-side inference chain. +Before it decides whether to start Ollama, the command probes `http://127.0.0.1:11434/api/tags` and leaves a healthy daemon unchanged. +When that API is unhealthy, it starts the fixed user-local executable only if NemoClaw has a valid ownership receipt. +It refuses to launch a duplicate when another `ollama` process exists but the API remains unhealthy. +After a launch, recovery waits up to 30 seconds for `/api/tags` to return valid JSON with a `models` array. +It does not take over a system service or an unrelated user-managed daemon. +It refuses a symbolic link, non-regular file, or non-executable file at the receipt path. + +The command also requires a trusted reachable route probe at `https://inference.local/v1/models` before it reports success. +When route repair needs the local Ollama dependency check, it verifies or recovers the authenticated proxy on port `11435` before it retries the route. +If Ollama does not become healthy within 30 seconds, the command identifies the exact receipt-bound executable and its `serve` argument, then tells you to retry recovery. +An Ollama startup or route failure exits non-zero and prints the available recovery guidance. + + ## Rebuild While Preserving State diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 63471af5716..e7746fe4b0e 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1020,10 +1020,21 @@ Set `NEMOCLAW_NO_CONNECT_HINT=1` to suppress the hint in scripted workflows. If the sandbox is running an outdated agent version, a non-blocking warning prints before connecting with a `$$nemoclaw rebuild` hint. If another terminal is already connected to the sandbox, `connect` prints a note with the number of existing sessions before proceeding. Multiple concurrent sessions are allowed. -`connect` does not pull or serve a model itself, but it does inspect managed-vLLM install variables such as `NEMOCLAW_VLLM_MODEL` and `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` if you exported them in the same shell. +`connect` does not pull a model itself, but it does inspect managed-vLLM install variables such as `NEMOCLAW_VLLM_MODEL` and `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` if you exported them in the same shell. An unknown model slug, malformed extra-args JSON, or a gated model (for example `deepseek-r1-distill-70b`) with no `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN` exits non-zero with the same error the installer would emit, before any sandbox readiness probe or SSH attach. Unset the managed-vLLM variable, or fix the value, before retrying. + +For a portable experimental-profile sandbox with the recorded `ollama-local` provider, `connect --probe-only` probes `http://127.0.0.1:11434/api/tags` before it decides whether to start Ollama. +If the API is unhealthy and the ownership receipt is valid, it starts the fixed user-local Ollama executable with `serve`. +It refuses to launch a duplicate when another `ollama` process exists but the API remains unhealthy. +After a launch, it waits up to 30 seconds for valid `/api/tags` JSON with a `models` array. +It then requires the existing sandbox `inference.local` route probe to return a trusted reachable result before it exits successfully. +When route repair needs the local Ollama dependency check, it verifies or recovers the authenticated proxy before it retries `inference.local`. +If Ollama does not become healthy within 30 seconds, the command identifies the exact receipt-bound executable and its `serve` argument, then tells you to retry recovery. +The command does not take over a system service or an unrelated user-managed Ollama daemon. + + Before reading or changing the live OpenShell gateway inference route, `connect` verifies the shared provider and sandbox metadata. When the live route differs and the metadata is compatible, `connect` warns and re-points the route to the target sandbox's recorded provider and model. Refer to [Use Shared Gateway Routes](../inference/manage-inference/use-shared-gateway-routes) for provider-global identity, route drift, and hard-error recovery. @@ -1378,6 +1389,12 @@ $$nemoclaw my-assistant recover + + +For a portable experimental-profile sandbox with the recorded `ollama-local` provider, `recover` also runs the ownership-bound Ollama and `inference.local` route reachability checks described for [`connect --probe-only`](#$$nemoclaw-name-connect). + + + `recover` re-evaluates the documented Hermes secret boundary against `/sandbox/.hermes/.env` and the supervisor runtime environment on every run, including when the gateway is already healthy. diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index aa6ed221762..12bc308eafe 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -33,6 +33,7 @@ Share redacted diagnostics only. | `~/.nemoclaw/onboard-session.json` | Resume marker for an onboarding attempt that failed before completion. | Yes, when you intentionally want to discard the failed session and start over. Prefer `$$nemoclaw onboard --fresh` when available. | | `~/.nemoclaw/usage-notice.json` | Records the third-party software notice version in `acceptedVersion` and the acceptance time in `acceptedAt`. Install, onboarding, and rebuild flows consult this file and prompt again when its recorded version differs from the current notice or the file is absent. | Yes; deleting it makes the next applicable install, onboarding, or rebuild flow prompt for acceptance again. | | `~/.nemoclaw/ollama-proxy-token` | Local auth token used by the host-side Ollama auth proxy. | Yes, but re-run onboarding afterward so NemoClaw recreates and registers the proxy token. | +| `~/.nemoclaw/ollama/user-local-ownership.json` | Mode-`0600`, credential-free receipt containing a schema version and the fixed `~/.local/bin/ollama` path. Portable OpenClaw recovery requires this receipt before it starts the binary. A successful system Ollama install removes the receipt. | No while a portable OpenClaw sandbox uses the user-local daemon. Deleting it disables automatic daemon restart until you reinstall Ollama through onboarding. | | `~/.nemoclaw/dual-station-vllm-api-key` | Owner-only host-global bearer API key shared by authenticated NemoClaw-managed vLLM profiles. | No while any managed vLLM runtime uses it. Full uninstall removes it only after matching runtime cleanup succeeds and no sibling gateway remains. | | `~/.nemoclaw/host-local-vllm-runtime.json` | Owner-only, credential-free receipt for a catalog-selected single-host vLLM runtime. It binds the immutable catalog, preset, and recipe digests to the exact container ID and API-key fingerprint. | No while that runtime exists. Recovery and full uninstall fail closed if its profile labels do not match this receipt, and uninstall removes it only after exact container cleanup succeeds. | | `~/.nemoclaw/managed-llama-cpp/api-key` | Owner-only bearer API key mounted read-only into the managed llama.cpp container. | No while the managed llama.cpp runtime exists. A retry reuses the same key. Full uninstall removes it with the managed state only after runtime cleanup succeeds. | diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index e7e35865652..040e6a202a7 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -480,6 +480,27 @@ describe("connectSandbox flow", () => { ); }); + it("probe-only mode exits before reporting success when inference.local returns no trusted result (#8502)", async () => { + const harness = createConnectHarness({ + registryEntry: { + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + }, + inferenceGetOutput: "Provider: nvidia-prod\nModel: nvidia/nemotron-3-super-120b-a12b\n", + inferenceProbeResponses: ["route probe unavailable"], + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.logSpy.mock.calls.flat().join("\n")).not.toContain("Probe complete"); + expect(harness.errorSpy.mock.calls.flat().join("\n")).toContain( + "inference route is not known healthy", + ); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + it("probe-only mode reports an ordinary running gateway for an already-running completion (#7919)", async () => { const harness = createConnectHarness({ processCheck: { diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 863555a7b34..3b8bf8ac0ce 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -284,7 +284,7 @@ async function runSandboxConnectProbe(sandboxName: string): Promise { ); } if (processCheck.wasRunning) { - await ensureSandboxInferenceRoute(sandboxName, agent, { quiet: true }); + await ensureSandboxInferenceRouteOrExit(sandboxName, agent); // Defense-in-depth scope-upgrade approval on the probe-only / `recover` // path (#4504): the gateway is up, so deterministically clear any pending // allowlisted CLI/webchat scope upgrade. Best-effort; never throws. @@ -299,7 +299,7 @@ async function runSandboxConnectProbe(sandboxName: string): Promise { return; } if (processCheck.recovered) { - await ensureSandboxInferenceRoute(sandboxName, agent, { quiet: true }); + await ensureSandboxInferenceRouteOrExit(sandboxName, agent); // Same defense-in-depth approval after a recovery (#4504); best-effort. runConnectAutoPairApprovalPass(sandboxName); const managedControlCompletion = @@ -313,7 +313,7 @@ async function runSandboxConnectProbe(sandboxName: string): Promise { } return; } - await ensureSandboxInferenceRoute(sandboxName, agent, { quiet: true }); + await ensureSandboxInferenceRouteOrExit(sandboxName, agent); console.error( ` Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); @@ -1087,9 +1087,10 @@ async function runConnectEntryPreflight( console.error(` Error: ${error instanceof Error ? error.message : String(error)}`); process.exit(1); } - // probe-only / recover never install or serve a model, so skip the - // express-vLLM model preflight for them (it only steers the install path - // and would otherwise hard-exit a recovery on a stale NEMOCLAW_VLLM_MODEL). + // probe-only / recover can restart receipt-owned local inference, but they + // never select, install, or pull a model. Skip the express-vLLM model + // preflight because it only steers installation and can reject recovery on + // a stale NEMOCLAW_VLLM_MODEL. if (!probeOnly) preflightVllmModelEnvOrExit(); const live = await ensureLiveSandboxOrExit(sandboxName, { allowNonReadyPhase: true, diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index d28971d6702..5f46646d015 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -91,13 +91,13 @@ function gatewayScopedArgs(args: string[], gatewayName?: string): string[] { /** Recover a receipt-bound portable sandbox before the live lookup rejects a stopped container. */ export function recoverPortableDemoSandboxLifecycleForConnect( sandboxName: string, - sandbox: Pick | null, + sandbox: Pick | null, gatewayName: string, ): PortableDemoLifecycleRecoveryResult { if (!sandbox) return { kind: "not-installed" }; return recoverPortableDemoSandboxLifecycle( sandboxName, - { agent: sandbox.agent, gatewayName }, + { agent: sandbox.agent, gatewayName, provider: sandbox.provider }, { openshellBinary: getOpenshellBinary(), captureOpenshell: (args, timeoutMs) => { diff --git a/src/lib/onboard/experimental/ollama-user-local-runtime.test.ts b/src/lib/onboard/experimental/ollama-user-local-runtime.test.ts new file mode 100644 index 00000000000..c2d76a16c7c --- /dev/null +++ b/src/lib/onboard/experimental/ollama-user-local-runtime.test.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + loadUserLocalOllamaOwnership, + recordUserLocalOllamaOwnership, + removeUserLocalOllamaOwnership, + userLocalOllamaOwnershipInternals, +} from "./ollama-user-local-runtime"; + +const temporaryDirectories: string[] = []; + +function createFixture(): { homeDir: string; stateDir: string; binPath: string } { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-owner-")); + temporaryDirectories.push(homeDir); + return { + homeDir, + stateDir: path.join(homeDir, ".nemoclaw"), + binPath: path.join(homeDir, ".local", "bin", "ollama"), + }; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("user-local Ollama ownership receipt", () => { + it("records and reloads only the fixed NemoClaw user-local path (#8502)", () => { + const fixture = createFixture(); + + recordUserLocalOllamaOwnership(fixture.binPath, fixture); + + expect(loadUserLocalOllamaOwnership(fixture)).toBe(fixture.binPath); + const receipt = userLocalOllamaOwnershipInternals.receiptPath(fixture); + expect(fs.statSync(receipt).mode & 0o777).toBe(0o600); + }); + + it("refuses to record an Ollama path outside the fixed user-local install (#8502)", () => { + const fixture = createFixture(); + + expect(() => recordUserLocalOllamaOwnership("/usr/local/bin/ollama", fixture)).toThrow( + "unexpected user-local Ollama path", + ); + }); + + it("rejects a receipt that redirects recovery to another executable (#8502)", () => { + const fixture = createFixture(); + const receipt = userLocalOllamaOwnershipInternals.receiptPath(fixture); + fs.mkdirSync(path.dirname(receipt), { recursive: true }); + fs.writeFileSync( + receipt, + `${JSON.stringify({ schemaVersion: 1, binPath: "/tmp/unrelated" })}\n`, + { mode: 0o600 }, + ); + + expect(() => loadUserLocalOllamaOwnership(fixture)).toThrow("ownership receipt is invalid"); + }); + + it("removes obsolete ownership after a system installation (#8502)", () => { + const fixture = createFixture(); + recordUserLocalOllamaOwnership(fixture.binPath, fixture); + + removeUserLocalOllamaOwnership(fixture); + + expect(loadUserLocalOllamaOwnership(fixture)).toBeNull(); + }); +}); diff --git a/src/lib/onboard/experimental/ollama-user-local-runtime.ts b/src/lib/onboard/experimental/ollama-user-local-runtime.ts new file mode 100644 index 00000000000..46ce59fc8c6 --- /dev/null +++ b/src/lib/onboard/experimental/ollama-user-local-runtime.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { openRegularFileNoFollow } from "../../adapters/fs/regular-file"; +import { OLLAMA_PORT } from "../../core/ports"; +import { ensureConfigDir } from "../../state/config-io"; + +export { OLLAMA_PORT }; + +const RECEIPT_DIRECTORY = "ollama"; +const RECEIPT_FILE = "user-local-ownership.json"; +const MAX_RECEIPT_BYTES = 1024; + +interface UserLocalOllamaOwnershipReceipt { + schemaVersion: 1; + binPath: string; +} + +export interface UserLocalOllamaOwnershipDeps { + homeDir?: string; + stateDir?: string; +} + +function resolveHomeDir(deps: UserLocalOllamaOwnershipDeps): string { + return deps.homeDir ?? os.homedir(); +} + +function resolveStateDir(deps: UserLocalOllamaOwnershipDeps): string { + return deps.stateDir ?? path.join(resolveHomeDir(deps), ".nemoclaw"); +} + +function receiptPath(deps: UserLocalOllamaOwnershipDeps): string { + return path.join(resolveStateDir(deps), RECEIPT_DIRECTORY, RECEIPT_FILE); +} + +function expectedBinPath(deps: UserLocalOllamaOwnershipDeps): string { + return path.join(resolveHomeDir(deps), ".local", "bin", "ollama"); +} + +function parseReceipt( + value: unknown, + deps: UserLocalOllamaOwnershipDeps, +): UserLocalOllamaOwnershipReceipt { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("NemoClaw-managed user-local Ollama ownership receipt is malformed"); + } + const receipt = value as Record; + if ( + Object.keys(receipt).sort().join(",") !== "binPath,schemaVersion" || + receipt.schemaVersion !== 1 || + receipt.binPath !== expectedBinPath(deps) + ) { + throw new Error("NemoClaw-managed user-local Ollama ownership receipt is invalid"); + } + return receipt as unknown as UserLocalOllamaOwnershipReceipt; +} + +/** Record the fixed user-local binary only after NemoClaw starts it successfully. */ +export function recordUserLocalOllamaOwnership( + binPath: string, + deps: UserLocalOllamaOwnershipDeps = {}, +): void { + if (binPath !== expectedBinPath(deps)) { + throw new Error("NemoClaw refused to record an unexpected user-local Ollama path"); + } + const target = receiptPath(deps); + ensureConfigDir(path.dirname(target)); + let file; + try { + file = openRegularFileNoFollow(target, { writable: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + file = openRegularFileNoFollow(target, { create: true, mode: 0o600, writable: true }); + } + try { + const receipt: UserLocalOllamaOwnershipReceipt = { schemaVersion: 1, binPath }; + file.replaceUtf8(`${JSON.stringify(receipt, null, 2)}\n`, 0o600); + } finally { + file.close(); + } +} + +/** Load the exact receipt-bound user-local path, or null when no receipt exists. */ +export function loadUserLocalOllamaOwnership( + deps: UserLocalOllamaOwnershipDeps = {}, +): string | null { + let file; + try { + file = openRegularFileNoFollow(receiptPath(deps)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + try { + const receipt = parseReceipt(JSON.parse(file.readUtf8(MAX_RECEIPT_BYTES)), deps); + return receipt.binPath; + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error("NemoClaw-managed user-local Ollama ownership receipt is malformed"); + } + throw error; + } finally { + file.close(); + } +} + +/** Remove stale user-local ownership after a successful system installation. */ +export function removeUserLocalOllamaOwnership(deps: UserLocalOllamaOwnershipDeps = {}): void { + try { + fs.unlinkSync(receiptPath(deps)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +export const userLocalOllamaOwnershipInternals = { receiptPath }; diff --git a/src/lib/onboard/experimental/portable-demo-lifecycle.test.ts b/src/lib/onboard/experimental/portable-demo-lifecycle.test.ts index f0fd1a48a3c..a5743343144 100644 --- a/src/lib/onboard/experimental/portable-demo-lifecycle.test.ts +++ b/src/lib/onboard/experimental/portable-demo-lifecycle.test.ts @@ -115,6 +115,13 @@ function installReceipt(stateDir: string, podman: ReturnType, @@ -546,6 +553,230 @@ describe("portable demo sandbox lifecycle", () => { expect(launchOpenshell).not.toHaveBeenCalled(); }); + it("restarts receipt-owned user-local Ollama before reporting portable recovery complete (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const binPath = createManagedOllamaBinary(stateDir); + let ollamaStarted = false; + const captureHost = vi.fn((command: string) => { + switch (command) { + case "pgrep": + return { status: 1 }; + case "curl": + return ollamaStarted + ? { status: 0, stdout: JSON.stringify({ models: [] }) } + : { status: 7, stderr: "connection refused" }; + default: + throw new Error(`Unexpected host command: ${command}`); + } + }); + const launchHost = vi.fn(() => { + ollamaStarted = true; + }); + const log = vi.fn(); + + expect( + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + env: { HOME: stateDir }, + stateDir, + podman: runtime.podman, + captureOpenshell: (args) => + args.includes("curl") ? { status: 0, stdout: "200" } : { status: 0 }, + captureHost, + launchHost, + loadManagedOllama: () => binPath, + log, + }, + ), + ).toEqual({ kind: "already-running" }); + expect(launchHost).toHaveBeenCalledWith(binPath, ["serve"], { + HOME: stateDir, + OLLAMA_HOST: "127.0.0.1:11434", + }); + expect(log).toHaveBeenCalledWith( + " Portable demo lifecycle restarted NemoClaw-managed Ollama.", + ); + }); + + it("does not inspect ownership or launch Ollama when the local API is already healthy (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const loadManagedOllama = vi.fn(); + const launchHost = vi.fn(); + + expect( + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + stateDir, + podman: runtime.podman, + captureOpenshell: (args) => + args.includes("curl") ? { status: 0, stdout: "200" } : { status: 0 }, + captureHost: (command) => + command === "curl" + ? { status: 0, stdout: JSON.stringify({ models: [] }) } + : { status: 1 }, + launchHost, + loadManagedOllama, + }, + ), + ).toEqual({ kind: "already-running" }); + expect(loadManagedOllama).not.toHaveBeenCalled(); + expect(launchHost).not.toHaveBeenCalled(); + }); + + it("does not launch an unowned user-local Ollama binary (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const launchHost = vi.fn(); + + expect( + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + stateDir, + podman: runtime.podman, + captureOpenshell: (args) => + args.includes("curl") ? { status: 0, stdout: "200" } : { status: 0 }, + captureHost: () => ({ status: 7 }), + launchHost, + loadManagedOllama: () => null, + }, + ), + ).toEqual({ kind: "already-running" }); + expect(launchHost).not.toHaveBeenCalled(); + }); + + it("refuses to launch a duplicate when another Ollama process is unhealthy (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const binPath = createManagedOllamaBinary(stateDir); + const launchHost = vi.fn(); + + expect(() => + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + stateDir, + podman: runtime.podman, + captureOpenshell: () => ({ status: 0 }), + captureHost: (command) => ({ status: command === "pgrep" ? 0 : 7 }), + launchHost, + loadManagedOllama: () => binPath, + }, + ), + ).toThrow("refused to launch a duplicate"); + expect(launchHost).not.toHaveBeenCalled(); + }); + + it("refuses a receipt-owned Ollama binary that is not executable (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const binPath = createManagedOllamaBinary(stateDir); + fs.chmodSync(binPath, 0o600); + const launchHost = vi.fn(); + + expect(() => + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + stateDir, + podman: runtime.podman, + captureOpenshell: () => ({ status: 0 }), + captureHost: () => ({ status: 7 }), + launchHost, + loadManagedOllama: () => binPath, + }, + ), + ).toThrow("is not a regular executable"); + expect(launchHost).not.toHaveBeenCalled(); + }); + + it("refuses a receipt-owned Ollama path that is a symbolic link (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const targetPath = createManagedOllamaBinary(stateDir); + const binPath = path.join(stateDir, "ollama-link"); + fs.symlinkSync(targetPath, binPath); + const launchHost = vi.fn(); + + expect(() => + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + stateDir, + podman: runtime.podman, + captureOpenshell: () => ({ status: 0 }), + captureHost: () => ({ status: 7 }), + launchHost, + loadManagedOllama: () => binPath, + }, + ), + ).toThrow("is not a regular executable"); + expect(launchHost).not.toHaveBeenCalled(); + }); + + it("fails after the bounded timeout when restarted Ollama stays unhealthy (#8502)", () => { + const stateDir = temporaryStateDir(); + const runtime = createPodman(); + installReceipt(stateDir, runtime.podman); + const binPath = createManagedOllamaBinary(stateDir); + let now = 0; + const launchHost = vi.fn(); + + let failure: Error | undefined; + try { + recoverPortableDemoSandboxLifecycle( + "alpha", + { agent: "openclaw", gatewayName: "nemoclaw", provider: "ollama-local" }, + { + platform: "linux", + stateDir, + podman: runtime.podman, + captureOpenshell: () => ({ status: 0 }), + captureHost: (command) => + command === "pgrep" + ? { status: 1 } + : { status: 0, stdout: JSON.stringify({ status: "ok" }) }, + launchHost, + loadManagedOllama: () => binPath, + now: () => now, + sleep: (milliseconds) => { + now += milliseconds; + }, + }, + ); + } catch (error) { + failure = error as Error; + } + expect(failure?.message).toContain("did not become healthy"); + expect(failure?.message).toContain( + `start the receipt-bound executable at ${JSON.stringify(binPath)} with the 'serve' argument, then retry`, + ); + expect(now).toBe(30_000); + expect(launchHost).toHaveBeenCalledOnce(); + }); + it("restarts the managed startup process once when recovery upgrades a schema-1 receipt (#8441)", () => { const stateDir = temporaryStateDir(); const runtime = createPodman(); diff --git a/src/lib/onboard/experimental/portable-demo-lifecycle.ts b/src/lib/onboard/experimental/portable-demo-lifecycle.ts index e8c14b61304..11117b29d31 100644 --- a/src/lib/onboard/experimental/portable-demo-lifecycle.ts +++ b/src/lib/onboard/experimental/portable-demo-lifecycle.ts @@ -10,6 +10,7 @@ import path from "node:path"; import { openRegularFileNoFollow } from "../../adapters/fs/regular-file"; import { ensureConfigDir } from "../../state/config-io"; import { isPortableExperimentalProfile } from "../docker-driver-platform"; +import { loadUserLocalOllamaOwnership, OLLAMA_PORT } from "./ollama-user-local-runtime"; const RECEIPT_DIRECTORY = "portable-demo-lifecycle"; const MAX_RECEIPT_BYTES = 4096; @@ -18,6 +19,7 @@ const PROBE_TIMEOUT_MS = 5_000; const EXEC_READY_TIMEOUT_MS = 90_000; const STARTUP_STOP_TIMEOUT_MS = 30_000; const STARTUP_TIMEOUT_MS = 90_000; +const OLLAMA_STARTUP_TIMEOUT_MS = 30_000; const POLL_INTERVAL_MS = 1_000; const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/u; const SANDBOX_ID_PATTERN = /^[A-Za-z0-9._:-]{1,256}$/u; @@ -60,6 +62,9 @@ export interface PortableDemoLifecycleDeps { podman?: (args: readonly string[]) => CommandResult; captureOpenshell?: (args: readonly string[], timeoutMs: number) => CommandResult; launchOpenshell?: (args: readonly string[]) => void; + captureHost?: (command: string, args: readonly string[], timeoutMs: number) => CommandResult; + launchHost?: (command: string, args: readonly string[], env: NodeJS.ProcessEnv) => void; + loadManagedOllama?: () => string | null; sleep?: (milliseconds: number) => void; now?: () => number; log?: (message: string) => void; @@ -73,6 +78,7 @@ export type PortableDemoLifecycleRecoveryResult = export interface PortableDemoLifecycleContext { agent?: string | null; gatewayName: string; + provider?: string | null; } function defaultPodman(args: readonly string[], env: NodeJS.ProcessEnv): CommandResult { @@ -113,6 +119,32 @@ function defaultLaunchOpenshell( child.unref(); } +function defaultCaptureHost( + command: string, + args: readonly string[], + timeoutMs: number, + env: NodeJS.ProcessEnv, +): CommandResult { + return spawnSync(command, [...args], { + encoding: "utf-8", + env, + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutMs, + }); +} + +function defaultLaunchHost(command: string, args: readonly string[], env: NodeJS.ProcessEnv): void { + const child = spawn(command, [...args], { + detached: true, + env, + shell: false, + stdio: "ignore", + }); + child.once("error", () => undefined); + child.unref(); +} + function defaultSleep(milliseconds: number): void { if (milliseconds > 0) Atomics.wait(SLEEP_BUFFER, 0, 0, milliseconds); } @@ -391,6 +423,93 @@ function gatewayIsRunning( return result.status === 0 && /(?:^|\D)(?:200|401)\s*$/u.test(String(result.stdout ?? "")); } +function ollamaIsHealthy( + captureHost: NonNullable, + timeoutMs: number, +): boolean { + const result = captureHost( + "curl", + [ + "-fsS", + "--max-time", + String(Math.max(1, Math.ceil(Math.min(PROBE_TIMEOUT_MS, timeoutMs) / 1_000))), + `http://127.0.0.1:${String(OLLAMA_PORT)}/api/tags`, + ], + Math.min(PROBE_TIMEOUT_MS, timeoutMs), + ); + if (result.status !== 0 || result.error) return false; + try { + const response = JSON.parse(String(result.stdout ?? "")); + return isRecord(response) && Array.isArray(response.models); + } catch { + return false; + } +} + +function assertManagedOllamaBinary(binPath: string): void { + let stats: fs.Stats; + try { + stats = fs.lstatSync(binPath); + } catch { + throw new Error( + `NemoClaw-managed Ollama binary '${binPath}' is missing; reinstall Ollama through nemoclaw onboard`, + ); + } + if (stats.isSymbolicLink() || !stats.isFile() || (stats.mode & 0o111) === 0) { + throw new Error( + `NemoClaw-managed Ollama binary '${binPath}' is not a regular executable; reinstall Ollama through nemoclaw onboard`, + ); + } +} + +function recoverManagedOllama( + context: PortableDemoLifecycleContext, + commandEnv: NodeJS.ProcessEnv, + stateDir: string, + timing: Required>, + deps: PortableDemoLifecycleDeps, +): void { + if (context.provider !== "ollama-local") return; + const captureHost = + deps.captureHost ?? + ((command, args, timeoutMs) => defaultCaptureHost(command, args, timeoutMs, commandEnv)); + if (ollamaIsHealthy(captureHost, PROBE_TIMEOUT_MS)) return; + + const homeDir = commandEnv.HOME ?? os.homedir(); + const binPath = deps.loadManagedOllama + ? deps.loadManagedOllama() + : loadUserLocalOllamaOwnership({ homeDir, stateDir }); + if (!binPath) return; + assertManagedOllamaBinary(binPath); + + const processProbe = captureHost("pgrep", ["-x", "ollama"], PROBE_TIMEOUT_MS); + if (processProbe.status === 0 && !processProbe.error) { + throw new Error( + `An Ollama process already exists, but http://127.0.0.1:${String(OLLAMA_PORT)}/api/tags is unavailable; NemoClaw refused to launch a duplicate`, + ); + } + if (processProbe.status !== 1 || processProbe.error) { + throw new Error(`Ollama process state could not be determined: ${commandDetail(processProbe)}`); + } + + const launchHost = + deps.launchHost ?? ((command, args, env) => defaultLaunchHost(command, args, env)); + launchHost(binPath, ["serve"], { + ...commandEnv, + HOME: homeDir, + OLLAMA_HOST: `127.0.0.1:${String(OLLAMA_PORT)}`, + }); + const recovered = waitFor(OLLAMA_STARTUP_TIMEOUT_MS, timing, (remainingMs) => + ollamaIsHealthy(captureHost, remainingMs), + ); + if (!recovered) { + throw new Error( + `NemoClaw-managed Ollama did not become healthy at http://127.0.0.1:${String(OLLAMA_PORT)}/api/tags within 30 seconds; start the receipt-bound executable at ${JSON.stringify(binPath)} with the 'serve' argument, then retry`, + ); + } + (deps.log ?? console.log)(" Portable demo lifecycle restarted NemoClaw-managed Ollama."); +} + /** Configure the hidden portable profile for one exact container. */ export function installPortableDemoSandboxLifecycle( sandboxName: string, @@ -487,6 +606,7 @@ export function recoverPortableDemoSandboxLifecycle( if (!execReady) { throw new Error(`Portable sandbox '${sandboxName}' did not reconnect to the OpenShell gateway`); } + recoverManagedOllama(context, commandEnv, stateDir, timing, deps); const gatewayRunning = gatewayIsRunning(receipt, gatewayName, capture, PROBE_TIMEOUT_MS); const refreshStartup = receipt.schemaVersion < CURRENT_RECEIPT_SCHEMA_VERSION; if (!refreshStartup && gatewayRunning) { diff --git a/src/lib/onboard/install-ollama-linux-upgrade.test.ts b/src/lib/onboard/install-ollama-linux-upgrade.test.ts index 64492170207..17bc7f3bbd3 100644 --- a/src/lib/onboard/install-ollama-linux-upgrade.test.ts +++ b/src/lib/onboard/install-ollama-linux-upgrade.test.ts @@ -25,6 +25,8 @@ function makeOpts(overrides: Partial): InstallOllamaL ensureManagedOllamaLoopbackSystemdOverrideImpl: vi.fn().mockReturnValue("ready"), fileExistsImpl: vi.fn().mockReturnValue(false), readFileImpl: vi.fn().mockReturnValue(""), + recordUserLocalOllamaOwnershipImpl: vi.fn(), + removeUserLocalOllamaOwnershipImpl: vi.fn(), log: vi.fn(), errorLog: vi.fn(), ...overrides, diff --git a/src/lib/onboard/install-ollama-linux.test.ts b/src/lib/onboard/install-ollama-linux.test.ts index 4f31ad14a66..bcbaf4908f2 100644 --- a/src/lib/onboard/install-ollama-linux.test.ts +++ b/src/lib/onboard/install-ollama-linux.test.ts @@ -27,6 +27,8 @@ function makeOpts(overrides: Partial): InstallOllamaL ensureManagedOllamaLoopbackSystemdOverrideImpl: vi.fn().mockReturnValue("ready"), fileExistsImpl: vi.fn().mockReturnValue(false), readFileImpl: vi.fn().mockReturnValue(""), + recordUserLocalOllamaOwnershipImpl: vi.fn(), + removeUserLocalOllamaOwnershipImpl: vi.fn(), log: vi.fn(), errorLog: vi.fn(), ...overrides, @@ -146,12 +148,14 @@ describe("installOllamaOnLinux (user-local)", () => { .fn() .mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); const runCaptureExImpl = vi.fn().mockReturnValue({ stdout: "", exitCode: 0, timedOut: false }); + const recordOwnership = vi.fn(); const opts = makeOpts({ modeOverride: "user-local", arch: () => "arm64", runCaptureImpl, runCaptureExImpl, runShellImpl, + recordUserLocalOllamaOwnershipImpl: recordOwnership, }); const result = installOllamaOnLinux(opts); expect(result).toEqual({ @@ -172,6 +176,9 @@ describe("installOllamaOnLinux (user-local)", () => { expect(startCall).toContain(`OLLAMA_HOST=127.0.0.1:`); expect(startCall).not.toContain("OLLAMA_CONTEXT_LENGTH="); expect(startCall).toContain(" serve "); + expect(recordOwnership).toHaveBeenCalledWith("/home/test/.local/bin/ollama", { + homeDir: "/home/test", + }); }); it("starts user-local Ollama with the requested Hermes context floor", () => { @@ -338,6 +345,21 @@ describe("installOllamaOnLinux (user-local)", () => { else process.env.PATH = originalPath; } }); + + it("reports failure when user-local ownership cannot be recorded (#8502)", () => { + const errorLog = vi.fn(); + const opts = makeOpts({ + modeOverride: "user-local", + runCaptureImpl: vi.fn().mockReturnValue("/usr/bin/zstd"), + recordUserLocalOllamaOwnershipImpl: () => { + throw new Error("receipt write denied"); + }, + errorLog, + }); + + expect(installOllamaOnLinux(opts).ok).toBe(false); + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("receipt write denied")); + }); }); describe("installOllamaOnLinux (system)", () => { @@ -357,11 +379,13 @@ describe("installOllamaOnLinux (system)", () => { .fn() .mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); const ensureOverride = vi.fn().mockReturnValue("ready"); + const removeOwnership = vi.fn(); const opts = makeOpts({ modeOverride: "system", runCaptureImpl: vi.fn().mockReturnValue("/usr/bin/zstd"), runShellImpl, ensureManagedOllamaLoopbackSystemdOverrideImpl: ensureOverride, + removeUserLocalOllamaOwnershipImpl: removeOwnership, }); const result = installOllamaOnLinux(opts); expect(result).toEqual({ ok: true, mode: "system", binPath: "/usr/local/bin/ollama" }); @@ -369,6 +393,7 @@ describe("installOllamaOnLinux (system)", () => { expect(installCall).toBeDefined(); expect(installCall).toContain("curl -fsSL"); expect(ensureOverride).toHaveBeenCalled(); + expect(removeOwnership).toHaveBeenCalledWith({ homeDir: "/home/test" }); }); it("passes the requested Hermes context floor to the systemd override", () => { diff --git a/src/lib/onboard/install-ollama-linux.ts b/src/lib/onboard/install-ollama-linux.ts index bd5b51ca198..2cdcbcdfd7c 100644 --- a/src/lib/onboard/install-ollama-linux.ts +++ b/src/lib/onboard/install-ollama-linux.ts @@ -5,13 +5,17 @@ import fs from "node:fs"; import os from "node:os"; import nodePath from "node:path"; -import { OLLAMA_PORT } from "../core/ports"; import { sleepSeconds, waitForHttp } from "../core/wait"; import { MIN_AUTODETECTED_OLLAMA_CONTEXT_WINDOW, resolveOllamaContextWindowFloor, } from "../inference/ollama-runtime-context"; import { cliName } from "./branding"; +import { + OLLAMA_PORT, + recordUserLocalOllamaOwnership, + removeUserLocalOllamaOwnership, +} from "./experimental/ollama-user-local-runtime"; import { decideInstallOllamaLinuxMode, hostCommandExists, @@ -71,6 +75,10 @@ export type InstallOllamaLinuxOptions = InstallOllamaLinuxModeOptions & { readFileImpl?: (path: string) => string; /** Test seam: redirect log output. */ log?: (message: string) => void; + /** Test seam: override the durable user-local ownership write. */ + recordUserLocalOllamaOwnershipImpl?: typeof recordUserLocalOllamaOwnership; + /** Test seam: override removal after a successful system installation. */ + removeUserLocalOllamaOwnershipImpl?: typeof removeUserLocalOllamaOwnership; }; /** @@ -365,6 +373,24 @@ export function installOllamaOnLinux(opts: InstallOllamaLinuxOptions): InstallOl const result = mode === "user-local" ? installOllamaUserLocal(opts) : installOllamaSystem(opts); // Pin to local loopback so a cached `host.docker.internal` from an // earlier WSL probe cannot route validation/pull at the Windows host. - if (result.ok) setResolvedOllamaHost("127.0.0.1"); + if (result.ok) { + setResolvedOllamaHost("127.0.0.1"); + const ownershipDeps = { homeDir: (opts.homedir ?? (() => os.homedir()))() }; + try { + if (result.mode === "user-local") { + (opts.recordUserLocalOllamaOwnershipImpl ?? recordUserLocalOllamaOwnership)( + result.binPath, + ownershipDeps, + ); + } else { + (opts.removeUserLocalOllamaOwnershipImpl ?? removeUserLocalOllamaOwnership)(ownershipDeps); + } + } catch (error) { + (opts.errorLog ?? ((message: string) => console.error(message)))( + ` Ollama install could not update its NemoClaw ownership receipt: ${error instanceof Error ? error.message : String(error)}`, + ); + return { ...result, ok: false }; + } + } return result; }