diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index d8d95f88c5c..cb85e8f7ab1 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -10,7 +10,7 @@ "test/install-preflight.test.ts": 4396, "test/nemoclaw-start.test.ts": 5289, "test/onboard-messaging.test.ts": 2097, - "test/onboard-selection.test.ts": 6922, + "test/onboard-selection.test.ts": 6891, "test/onboard.test.ts": 4783, "test/policies.test.ts": 2763 } diff --git a/scripts/ollama-auth-proxy.js b/scripts/ollama-auth-proxy.js index 5bec08d1bbe..d2215fb738a 100755 --- a/scripts/ollama-auth-proxy.js +++ b/scripts/ollama-auth-proxy.js @@ -80,6 +80,20 @@ const server = http.createServer((clientReq, clientRes) => { clientReq.pipe(proxyReq); }); +// The proxy binds 0.0.0.0, so an unhandled listen error (most commonly +// EADDRINUSE when the port is already taken) would crash with an uncaught +// exception. Exit cleanly with a non-zero code instead; the host-side +// startOllamaAuthProxy() detects the missing process and reports the port +// owner with remediation. See #4820. +server.on("error", (/** @type {NodeJS.ErrnoException} */ err) => { + if (err && err.code === "EADDRINUSE") { + console.error(`Ollama auth proxy: port ${LISTEN_PORT} is already in use`); + } else { + console.error(`Ollama auth proxy failed to start: ${err && err.message ? err.message : err}`); + } + process.exit(1); +}); + server.listen(LISTEN_PORT, "0.0.0.0", () => { console.log(`Ollama auth proxy listening on 0.0.0.0:${LISTEN_PORT} -> 127.0.0.1:${BACKEND_PORT}`); }); diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 1f8d28177e1..81d9daf4254 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -9,7 +9,7 @@ import type { GpuInfo } from "../local"; const path = require("path"); const { spawn, spawnSync } = require("child_process"); -const { ROOT, SCRIPTS, run, runCapture, shellQuote } = require("../../runner"); +const { ROOT, SCRIPTS, redact, run, runCapture, shellQuote } = require("../../runner"); const { OLLAMA_PORT, OLLAMA_PROXY_PORT } = require("../../core/ports"); const { waitForPort } = require("../../core/wait"); const { @@ -173,44 +173,154 @@ function killStaleProxy(): void { } } +// ── Port-conflict diagnostics ──────────────────────────────────── + +// Inspect what currently listens on the proxy port, excluding our own +// auth-proxy processes. Returns the owning PIDs and a human-readable +// description (command line) for each so a port conflict can be reported +// with the exact owning process instead of telling the user to run lsof +// themselves (issue #4820). +// +// `family` scopes the lookup: +// "4" — IPv4 listeners only. The proxy binds IPv4 (0.0.0.0), so only an +// IPv4 (or IPv6 dual-stack-wildcard) listener can actually block it. +// An IPv6-only listener (e.g. ::1 with IPV6_V6ONLY) does NOT conflict, +// so the pre-start abort uses this scope to avoid a false conflict. +// "any" — all TCP listeners. Used only to diagnose an already-failed bind, +// where the proxy died from EADDRINUSE: the culprit may be an IPv6 +// dual-stack wildcard (`:::PORT`) that blocks IPv4 yet lsof reports +// as IPv6, so the broad scope still names the owner. +// Either way we restrict to TCP listeners (not outbound connections / UDP that +// merely involve the port number). +function inspectForeignProxyPortOwners(family: "4" | "any" = "any"): { + pids: number[]; + descriptions: string[]; +} { + const pids: number[] = []; + const descriptions: string[] = []; + const selector = family === "4" ? `-ti4TCP:${OLLAMA_PROXY_PORT}` : `-tiTCP:${OLLAMA_PROXY_PORT}`; + const pidOutput = runCapture(["lsof", selector, "-sTCP:LISTEN"], { + ignoreError: true, + }); + if (!pidOutput || !String(pidOutput).trim()) return { pids, descriptions }; + for (const raw of String(pidOutput).trim().split(/\s+/)) { + const pid = Number.parseInt(raw, 10); + if (!Number.isInteger(pid) || pid <= 0) continue; + // Our own auth proxy is not a conflict — killStaleProxy() reclaims it. + if (isOllamaProxyProcess(pid)) continue; + pids.push(pid); + // Redact the owner's command line before display: a foreign process may + // carry a secret in its argv (e.g. `--token=…`), and this string is printed + // to the console. Matches the codebase convention of redacting command + // output before surfacing it. + const args = String( + redact(runCapture(["ps", "-p", String(pid), "-o", "args="], { ignoreError: true }) || ""), + ).trim(); + descriptions.push(args ? `PID ${pid}: ${args}` : `PID ${pid}`); + } + return { pids, descriptions }; +} + +function printProxyPortConflict(owners: { pids: number[]; descriptions: string[] }): void { + console.error( + ` Error: Ollama auth proxy cannot start — port ${OLLAMA_PROXY_PORT} is already in use by another process.`, + ); + for (const description of owners.descriptions) { + console.error(` ${description}`); + } + console.error(" Resolve the conflict, then re-run onboarding:"); + console.error(` • Stop the process above (e.g. kill ${owners.pids.join(" ") || ""}), or`); + // Export (don't inline) the override: OLLAMA_PROXY_PORT is read from the + // environment on every NemoClaw command, so a one-shot `VAR=… nemoclaw + // onboard` would drift — a later `nemoclaw connect` without it would manage + // the proxy on the default port while the route points at the custom one. + console.error(" • Choose a free proxy port and export it so every NemoClaw command"); + console.error(" uses the same value (add it to your shell profile to persist):"); + console.error(" export NEMOCLAW_OLLAMA_PROXY_PORT="); + console.error(" Containers will not be able to reach Ollama without the proxy."); +} + // ── Public API ─────────────────────────────────────────────────── +// How long to wait for the detached proxy to bind the port. Slower hosts and +// the window right after the systemd loopback restart can need several seconds, +// so poll with backoff instead of the previous single 2s probe (issue #4820). +const PROXY_START_ATTEMPTS = 12; + function startOllamaAuthProxy(): boolean { const crypto = require("crypto"); killStaleProxy(); + // After clearing any stale NemoClaw proxy, a process still holding the port + // is a genuine conflict. Report the exact owner and remediation up front so + // the user does not have to run lsof and interpret it themselves. Scope to + // IPv4: an IPv6-only listener does not block our 0.0.0.0 bind, so aborting on + // it would be a false conflict (a dual-stack blocker is still caught below + // via the spawned proxy's EADDRINUSE). + const preOwners = inspectForeignProxyPortOwners("4"); + if (preOwners.pids.length > 0) { + printProxyPortConflict(preOwners); + return false; + } + const proxyToken = crypto.randomBytes(24).toString("hex"); ollamaProxyToken = proxyToken; // Don't persist yet — wait until provider is confirmed in setupInference. // If the user backs out to a different provider, the token stays in memory // only and is discarded. const pid = spawnOllamaAuthProxy(proxyToken); - if (!waitForPort(OLLAMA_PROXY_PORT, 2)) { - console.error( - ` Error: Ollama auth proxy did not become ready on :${OLLAMA_PROXY_PORT} within timeout.`, - ); - return false; - } - if (!isOllamaProxyProcess(pid)) { - console.error(` Error: Ollama auth proxy failed to start on :${OLLAMA_PROXY_PORT}`); - console.error(` Containers will not be able to reach Ollama without the proxy.`); - console.error( - ` Check if port ${OLLAMA_PROXY_PORT} is already in use: lsof -ti :${OLLAMA_PROXY_PORT}`, - ); + + // Poll for readiness with backoff. Three terminal outcomes: + // • proxy alive and listening → success + // • proxy gone, a foreign process now owns the port → conflict (lost the + // EADDRINUSE race after the pre-check) + // • proxy gone, port free → it exited during startup (spawn failure) + for (let attempt = 0; attempt < PROXY_START_ATTEMPTS; attempt++) { + if (isOllamaProxyProcess(pid)) { + // waitForPort is a cheap TCP gate; proxyOwnsPortWithToken then proves the + // listener is our proxy (not a foreign service that grabbed the port) + // before we treat startup as successful. + if (waitForPort(OLLAMA_PROXY_PORT, 1) && proxyOwnsPortWithToken(proxyToken)) { + return true; + } + sleep(1); // alive but not yet bound — give a slow host more time + continue; + } + // The spawned proxy is gone. If it lost an EADDRINUSE race the blocker may + // be an IPv6 dual-stack listener, so use the broad scope to name the owner. + const owners = inspectForeignProxyPortOwners("any"); + if (owners.pids.length > 0) { + printProxyPortConflict(owners); + } else { + console.error(` Error: Ollama auth proxy exited during startup on :${OLLAMA_PROXY_PORT}.`); + console.error(" Containers will not be able to reach Ollama without the proxy."); + console.error(` Check the proxy port owner: lsof -ti :${OLLAMA_PROXY_PORT}`); + } return false; } - return true; + + console.error( + ` Error: Ollama auth proxy did not become ready on :${OLLAMA_PROXY_PORT} within ${PROXY_START_ATTEMPTS}s.`, + ); + console.error(" Containers will not be able to reach Ollama without the proxy."); + console.error(` Check the proxy port owner: lsof -ti :${OLLAMA_PROXY_PORT}`); + return false; } /** * Probe the running proxy to confirm it accepts the given token. * The proxy validates auth before forwarding to Ollama. A backend error like * 502 still proves the token was accepted, while 401 means token mismatch. + * + * Targets 127.0.0.1 (not `localhost`): the proxy binds IPv4 0.0.0.0, and + * `localhost` can resolve to ::1 first — on a host where an unrelated IPv6-only + * service holds the port, that would probe the wrong listener. This matches the + * other proxy probes (isProxyHealthy, probeOllamaAuthProxyHealth). See #4820. */ function probeProxyToken(token: string): "accepted" | "rejected" | "unreachable" { const result = runCurlWithAuthConfig( ["-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "3"], - `http://localhost:${OLLAMA_PROXY_PORT}/v1/models`, + `http://127.0.0.1:${OLLAMA_PROXY_PORT}/v1/models`, token, ); if (result.status !== 0) return "unreachable"; @@ -221,6 +331,22 @@ function probeProxyToken(token: string): "accepted" | "rejected" | "unreachable" return "unreachable"; } +// Confirm the listener on the proxy port is actually our auth proxy holding +// THIS token — not a foreign service that merely answers on the port. Our +// proxy is the only listener that BOTH rejects an unauthenticated request with +// 401 AND accepts the current token (200 from Ollama, or 502 when the backend +// is down — both non-401). A foreign HTTP service that ignores Authorization +// (answers 200/404 to everything) fails the unauthenticated-401 half, and a +// raw socket fails both. Requiring both halves is what makes this a +// proxy-specific readiness proof: without it we could persist a token for a +// process that never bound (lsof unavailable, or a dual-stack listener the +// IPv4 precheck missed, losing the EADDRINUSE race). The probes target +// 127.0.0.1, so they confirm our IPv4 proxy even when an unrelated IPv6-only +// listener shares the port number. See #4820. +function proxyOwnsPortWithToken(token: string): boolean { + return probeProxyToken(token) === "accepted" && probeProxyToken("") === "rejected"; +} + /** * Ensure the auth proxy is running with the correct persisted token. * Called on sandbox connect to recover from host reboots where the diff --git a/test/ollama-proxy-startup.test.ts b/test/ollama-proxy-startup.test.ts new file mode 100644 index 00000000000..a66c4087b29 --- /dev/null +++ b/test/ollama-proxy-startup.test.ts @@ -0,0 +1,342 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, it } from "vitest"; + +/** Parse JSON from the last stdout line, stripping any non-JSON prefix. */ +function parseStdoutJson(stdout: string): T { + const line = stdout.trim().split("\n").pop(); + if (!line) { + throw new Error("Expected JSON payload on the last stdout line"); + } + return JSON.parse(line); +} + +interface StartupResult { + returned: boolean; + spawnCount: number; + ncCalls: number; + authedProbes: number; + unauthProbes: number; + killCommands: string[][]; +} + +/** + * Run a child process that mocks the runner/child_process boundary and calls + * startOllamaAuthProxy() against the compiled proxy module. `setup` is inlined + * verbatim into the child and defines the runCapture / spawnSync behavior for + * the scenario under test. + */ +function runStartupScenario(setup: string): { + status: number | null; + stderr: string; + payload: StartupResult; +} { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ollama-proxy-startup-")); + const scriptPath = path.join(tmpDir, "startup-check.js"); + const proxyPath = JSON.stringify( + path.join(repoRoot, "dist", "lib", "inference", "ollama", "proxy.js"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + + const script = String.raw` +const childProcess = require("child_process"); +const runner = require(${runnerPath}); + +let spawnCount = 0; +let ncCalls = 0; +let authedProbes = 0; +let unauthProbes = 0; +const killCommands = []; + +${setup} + +// Default runner.run is a no-op success (used for kill in killStaleProxy). +if (!runner.run.__mocked) { + runner.run = () => ({ status: 0, stdout: "", stderr: "" }); +} + +const proxy = require(${proxyPath}); +const returned = proxy.startOllamaAuthProxy(); +console.log(JSON.stringify({ returned, spawnCount, ncCalls, authedProbes, unauthProbes, killCommands })); +`; + fs.writeFileSync(scriptPath, script); + + // The mocks and assertions hard-code the default ports (proxy :11435, + // backend :11434), so strip any inherited overrides to keep the child + // deterministic regardless of the caller's environment. + const childEnv: NodeJS.ProcessEnv = { ...process.env, HOME: tmpDir }; + delete childEnv.NEMOCLAW_OLLAMA_PROXY_PORT; + delete childEnv.NEMOCLAW_OLLAMA_PORT; + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: childEnv, + }); + + return { + status: result.status, + stderr: result.stderr, + payload: parseStdoutJson(result.stdout), + }; +} + +describe("startOllamaAuthProxy", () => { + it("reports the owning process and remediation when a foreign process holds the port", () => { + const { payload, stderr } = runStartupScenario(String.raw` +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("lsof") && text.includes("11435")) return "2222"; + if (text.includes("ps -p 2222")) return "/usr/bin/python3 -m http.server 11435"; + return ""; +}; +childProcess.spawn = () => { spawnCount += 1; return { pid: 7777, unref() {} }; }; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + if (args[0] === "nc") { ncCalls += 1; return { error: null, status: 0, stdout: "", stderr: "" }; } + if (args[0] === "curl") { + // proxyOwnsPortWithToken: authenticated probe (--config) → 200 (accepted); + // unauthenticated probe → 401 (rejected). Together they mark the listener + // as our auth proxy holding the token. Count each so tests can assert BOTH + // halves of the readiness proof actually ran. + const argv = Array.isArray(args[1]) ? args[1] : []; + const authed = argv.includes("--config"); + if (authed) { authedProbes += 1; } else { unauthProbes += 1; } + return { status: 0, stdout: authed ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; +`); + + assert.equal(payload.returned, false); + // Conflict is detected before any proxy is spawned. + assert.equal(payload.spawnCount, 0); + assert.match(stderr, /port 11435 is already in use/); + assert.match(stderr, /PID 2222: \/usr\/bin\/python3 -m http\.server 11435/); + assert.match(stderr, /kill 2222/); + assert.match(stderr, /NEMOCLAW_OLLAMA_PROXY_PORT=/); + }); + + it("starts the proxy when the port is free and the process binds it", () => { + const { payload } = runStartupScenario(String.raw` +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("lsof") && text.includes("11435")) return ""; + if (text.includes("ps -p 7777")) return "node /repo/scripts/ollama-auth-proxy.js"; + return ""; +}; +childProcess.spawn = () => { spawnCount += 1; return { pid: 7777, unref() {} }; }; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + if (args[0] === "nc") { ncCalls += 1; return { error: null, status: 0, stdout: "", stderr: "" }; } + if (args[0] === "curl") { + // proxyOwnsPortWithToken: authenticated probe (--config) → 200 (accepted); + // unauthenticated probe → 401 (rejected). Together they mark the listener + // as our auth proxy holding the token. Count each so tests can assert BOTH + // halves of the readiness proof actually ran. + const argv = Array.isArray(args[1]) ? args[1] : []; + const authed = argv.includes("--config"); + if (authed) { authedProbes += 1; } else { unauthProbes += 1; } + return { status: 0, stdout: authed ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; +`); + + assert.equal(payload.returned, true); + assert.equal(payload.spawnCount, 1); + // The readiness proof must run BOTH probes: a regression that accepted only + // the authenticated 200 (dropping the unauthenticated-401 check) would fail. + assert.ok(payload.authedProbes >= 1, "expected an authenticated token probe"); + assert.ok(payload.unauthProbes >= 1, "expected an unauthenticated 401 probe"); + }); + + it("starts despite an IPv6-only listener the IPv4-scoped preflight ignores", () => { + // Pins the address-family contract: the pre-start conflict check must use an + // IPv4-scoped lsof (-ti4TCP), since the proxy binds IPv4 0.0.0.0. An IPv6-only + // listener does not block that bind, so startup must still succeed. The stub + // returns a foreign owner ONLY for a broad (-tiTCP) query — if the preflight + // regressed to the broad probe it would see the owner and falsely abort, + // failing this test. + const { payload } = runStartupScenario(String.raw` +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("lsof") && text.includes("11435")) { + // IPv4-scoped query: the IPv6-only listener is invisible → no conflict. + if (text.includes("-ti4TCP") || text.includes("-i4")) return ""; + // Broad query would surface the IPv6-only owner (PID 9999). + return "9999"; + } + if (text.includes("ps -p 9999")) return "/usr/sbin/foreign-ipv6-service --listen [::1]:11435"; + if (text.includes("ps -p 7777")) return "node /repo/scripts/ollama-auth-proxy.js"; + return ""; +}; +childProcess.spawn = () => { spawnCount += 1; return { pid: 7777, unref() {} }; }; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + if (args[0] === "nc") { ncCalls += 1; return { error: null, status: 0, stdout: "", stderr: "" }; } + if (args[0] === "curl") { + const argv = Array.isArray(args[1]) ? args[1] : []; + const authed = argv.includes("--config"); + if (authed) { authedProbes += 1; } else { unauthProbes += 1; } + return { status: 0, stdout: authed ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; +`); + + assert.equal( + payload.returned, + true, + "IPv6-only listener must not abort the IPv4 proxy startup", + ); + assert.equal(payload.spawnCount, 1); + assert.ok(payload.authedProbes >= 1 && payload.unauthProbes >= 1); + }); + + it("recovers when a slow host binds the port only after a retry", () => { + const { payload } = runStartupScenario(String.raw` +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("lsof") && text.includes("11435")) return ""; + if (text.includes("ps -p 7777")) return "node /repo/scripts/ollama-auth-proxy.js"; + return ""; +}; +childProcess.spawn = () => { spawnCount += 1; return { pid: 7777, unref() {} }; }; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + if (args[0] === "nc") { + ncCalls += 1; + // Not listening for the first attempt's polling window, then ready. + return { error: null, status: ncCalls < 8 ? 1 : 0, stdout: "", stderr: "" }; + } + if (args[0] === "curl") { + // proxyOwnsPortWithToken: authenticated probe (--config) → 200 (accepted); + // unauthenticated probe → 401 (rejected). Together they mark the listener + // as our auth proxy holding the token. Count each so tests can assert BOTH + // halves of the readiness proof actually ran. + const argv = Array.isArray(args[1]) ? args[1] : []; + const authed = argv.includes("--config"); + if (authed) { authedProbes += 1; } else { unauthProbes += 1; } + return { status: 0, stdout: authed ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; +`); + + assert.equal(payload.returned, true); + assert.equal(payload.spawnCount, 1); + // Proves the outer retry loop crossed at least one full waitForPort window. + assert.ok(payload.ncCalls >= 6, "expected the proxy port to be polled across retries"); + // Both halves of the readiness proof still run on the successful retry. + assert.ok(payload.authedProbes >= 1, "expected an authenticated token probe"); + assert.ok(payload.unauthProbes >= 1, "expected an unauthenticated 401 probe"); + }); + + it("reports a spawn failure distinctly from a port conflict", () => { + const { payload, stderr } = runStartupScenario(String.raw` +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + // Port stays free: the spawned proxy exited without anyone owning the port. + if (text.includes("lsof") && text.includes("11435")) return ""; + if (text.includes("ps -p 8888")) return ""; + return ""; +}; +childProcess.spawn = () => { spawnCount += 1; return { pid: 8888, unref() {} }; }; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") return { status: 0, stdout: "", stderr: "" }; + if (args[0] === "nc") { ncCalls += 1; return { error: null, status: 0, stdout: "", stderr: "" }; } + if (args[0] === "curl") { + // proxyOwnsPortWithToken: authenticated probe (--config) → 200 (accepted); + // unauthenticated probe → 401 (rejected). Together they mark the listener + // as our auth proxy holding the token. Count each so tests can assert BOTH + // halves of the readiness proof actually ran. + const argv = Array.isArray(args[1]) ? args[1] : []; + const authed = argv.includes("--config"); + if (authed) { authedProbes += 1; } else { unauthProbes += 1; } + return { status: 0, stdout: authed ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; +`); + + assert.equal(payload.returned, false); + assert.equal(payload.spawnCount, 1); + assert.match(stderr, /exited during startup/); + assert.doesNotMatch(stderr, /already in use/); + }); + + it("reclaims a prior NemoClaw proxy on the port instead of reporting a conflict", () => { + const { payload, stderr } = runStartupScenario(String.raw` +// Reclaim is driven by the ACTUAL kill of pid 4242: the port/process only frees +// up once killStaleProxy issues \`kill 4242\`. A regression that skips reclaiming +// it leaves reclaimed=false, so lsof keeps reporting 4242 and startup cannot +// succeed — the killCommands assertion below then fails. +let reclaimed = false; +runner.run = (command) => { + killCommands.push(command); + if (Array.isArray(command) && command[0] === "kill" && command[1] === "4242") { + reclaimed = true; + } + return { status: 0, stdout: "", stderr: "" }; +}; +runner.run.__mocked = true; +// The persisted pid 4242 is a live NemoClaw proxy that currently owns the port. +// After killStaleProxy reclaims it, the freshly spawned pid 7777 binds the port. +const fs2 = require("node:fs"); +const path2 = require("node:path"); +const stateDir = path2.join(process.env.HOME, ".nemoclaw"); +fs2.mkdirSync(stateDir, { recursive: true }); +fs2.writeFileSync(path2.join(stateDir, "ollama-auth-proxy.pid"), "4242\n", { mode: 0o600 }); + +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : command; + if (text.includes("lsof") && text.includes("11435")) return reclaimed ? "" : "4242"; + if (text.includes("ps -p 4242")) return reclaimed ? "" : "node /repo/scripts/ollama-auth-proxy.js"; + if (text.includes("ps -p 7777")) return "node /repo/scripts/ollama-auth-proxy.js"; + return ""; +}; +childProcess.spawn = () => { spawnCount += 1; return { pid: 7777, unref() {} }; }; +const origSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (...args) => { + if (args[0] === "sleep") { return { status: 0, stdout: "", stderr: "" }; } + if (args[0] === "nc") { ncCalls += 1; return { error: null, status: 0, stdout: "", stderr: "" }; } + if (args[0] === "curl") { + // proxyOwnsPortWithToken: authenticated probe (--config) → 200 (accepted); + // unauthenticated probe → 401 (rejected). Together they mark the listener + // as our auth proxy holding the token. Count each so tests can assert BOTH + // halves of the readiness proof actually ran. + const argv = Array.isArray(args[1]) ? args[1] : []; + const authed = argv.includes("--config"); + if (authed) { authedProbes += 1; } else { unauthProbes += 1; } + return { status: 0, stdout: authed ? "200" : "401", stderr: "" }; + } + return origSpawnSync(...args); +}; +`); + + assert.equal(payload.returned, true); + assert.equal(payload.spawnCount, 1); + assert.doesNotMatch(stderr, /already in use/); + // The stale proxy must actually be reclaimed: assert `kill 4242` was issued. + assert.ok( + payload.killCommands.some( + (cmd) => Array.isArray(cmd) && cmd[0] === "kill" && cmd[1] === "4242", + ), + "expected killStaleProxy to issue `kill 4242`", + ); + }); +}); diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index e5afbbeff40..f23fd1ff4ce 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -18,30 +18,6 @@ const OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE = '{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"type":"function","function":{"name":"emit_ok","arguments":"{\\"ok\\":true}"}}]}}]}'; const PROVIDER_SELECTION_TEST_TIMEOUT_MS = testTimeout(60_000); -function writeOllamaToolCallingCurl(fakeBin: string) { - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='${OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE}' -status="200" -outfile="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - *) shift ;; - esac -done -if [ -n "$outfile" ]; then - printf '%s' "$body" > "$outfile" - printf '%s' "$status" -else - printf '%s' "$body" -fi -`, - { mode: 0o755 }, - ); -} - function writeOpenAiStyleAuthRetryCurl(fakeBin: string, goodToken: string, models = ["gpt-5.4"]) { fs.writeFileSync( path.join(fakeBin, "curl"), @@ -146,12 +122,22 @@ function writeAlwaysOkCurl(fakeBin: string, body = '{"id":"resp_123"}') { body='${body}' status="200" outfile="" +url="" +has_config=0 while [ "$#" -gt 0 ]; do case "$1" in -o) outfile="$2"; shift 2 ;; + --config) has_config=1; shift 2 ;; + http://*|https://*) url="$1"; shift ;; *) shift ;; esac done +# Model the real auth proxy: an unauthenticated request to :11435 gets 401, +# so startOllamaAuthProxy's readiness proof (unauth 401 + authenticated non-401) +# recognises this as our proxy. Harmless to non-proxy probes. +if [ "$has_config" -eq 0 ] && [[ "$url" == *:11435/* ]]; then + status="401" +fi if [ -n "$outfile" ]; then printf '%s' "$body" > "$outfile" fi @@ -940,24 +926,7 @@ const { setupNim } = require(${onboardPath}); const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='${OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE}' -status="200" -outfile="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - *) url="$1"; shift ;; - esac -done -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath}); @@ -1176,7 +1145,7 @@ console.log(JSON.stringify(result)); const waitPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "core", "wait.js")); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath}); @@ -1298,7 +1267,7 @@ const { setupNim } = require(${onboardPath}); const platformPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "platform.js")); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const runner = require(${runnerPath}); @@ -1409,7 +1378,7 @@ const { setupNim } = require(${onboardPath}); const platformPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "platform.js")); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const fs = require("fs"); @@ -1753,7 +1722,7 @@ ensureOllamaLoopbackSystemdOverride({ isNonInteractive: () => true }); const platformPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "platform.js")); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const runner = require(${runnerPath}); @@ -2090,7 +2059,7 @@ const { setupNim } = require(${onboardPath}); const pullLog = path.join(tmpDir, "pulls.log"); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); fs.writeFileSync( path.join(fakeBin, "ollama"), `#!/usr/bin/env bash @@ -2188,7 +2157,7 @@ const { setupNim } = require(${onboardPath}); const pullLog = path.join(tmpDir, "pulls.log"); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); fs.writeFileSync( path.join(fakeBin, "ollama"), `#!/usr/bin/env bash @@ -2294,7 +2263,7 @@ const { setupNim } = require(${onboardPath}); const pullLog = path.join(tmpDir, "pulls.log"); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); fs.writeFileSync( path.join(fakeBin, "ollama"), `#!/usr/bin/env bash @@ -2397,7 +2366,7 @@ const { setupNim } = require(${onboardPath}); const pullLog = path.join(tmpDir, "pulls.log"); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); fs.writeFileSync( path.join(fakeBin, "ollama"), `#!/usr/bin/env bash @@ -5038,7 +5007,7 @@ const { setupNim } = require(${onboardPath}); // Fake curl binary that returns a successful response — needed because // runCurlProbe and validateOllamaModel spawn real curl via child_process. fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); // Simulate: no Ollama installed, no Ollama running, no vLLM on native // Linux, so cloud + install-ollama should appear. @@ -5367,7 +5336,7 @@ const { setupNim } = require(${onboardPath}); const waitPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "core", "wait.js")); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath}); @@ -5531,7 +5500,7 @@ const { setupNim } = require(${onboardPath}); // Fake curl + zstd binaries on PATH. The install module uses curl to // probe the release tarball (HEAD) and zstd to decompress; both must // exist on PATH for the user-local path to choose the .tar.zst asset. - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); fs.writeFileSync(path.join(fakeBin, "zstd"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); const script = String.raw` @@ -5694,7 +5663,7 @@ const { setupNim } = require(${onboardPath}); const waitPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "core", "wait.js")); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); // Fake passwordless sudo so the upgrade gate doesn't short-circuit // before the official installer runs in this non-interactive scenario. fs.writeFileSync(path.join(fakeBin, "sudo"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); @@ -5856,7 +5825,7 @@ const { setupNim } = require(${onboardPath}); ); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath}); @@ -6309,7 +6278,7 @@ const { setupNim } = require(${onboardPath}); ); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath}); @@ -6450,7 +6419,7 @@ const { setupNim } = require(${onboardPath}); ); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath}); @@ -6580,7 +6549,7 @@ const { setupNim } = require(${onboardPath}); ); fs.mkdirSync(fakeBin, { recursive: true }); - writeOllamaToolCallingCurl(fakeBin); + writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); const script = String.raw` const credentials = require(${credentialsPath});