From 9a93975c7a5351a719197ba01b961f040002d478 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 15 Jul 2026 16:55:03 +0000 Subject: [PATCH 1/3] refactor(ollama): migrate auth proxy to .mts Convert scripts/ollama-auth-proxy.js to a typed ESM .mts entrypoint running under Node native type stripping without tsx. Preserve the fail-closed Bearer-token check, byte-length gate before timingSafeEqual, authorization header stripping, loopback backend, and EADDRINUSE exit. Trim the process needle to ollama-auth-proxy so upgrade and recovery still detect the old .js process next to the new .mts. Signed-off-by: Tinson Lai --- scripts/ollama-auth-proxy.js | 99 --------------------- scripts/ollama-auth-proxy.mts | 101 ++++++++++++++++++++++ src/lib/actions/uninstall/run-plan.ts | 2 +- src/lib/inference/ollama/proxy.ts | 6 +- test/e2e/live/gpu-e2e-helpers.ts | 2 +- test/e2e/live/ollama-auth-proxy.test.ts | 2 +- test/ollama-auth-proxy-handler-helpers.ts | 2 +- test/ollama-auth-proxy-handler.test.ts | 2 +- test/ollama-proxy-recovery.test.ts | 4 +- 9 files changed, 111 insertions(+), 109 deletions(-) delete mode 100755 scripts/ollama-auth-proxy.js create mode 100755 scripts/ollama-auth-proxy.mts diff --git a/scripts/ollama-auth-proxy.js b/scripts/ollama-auth-proxy.js deleted file mode 100755 index d2215fb738a..00000000000 --- a/scripts/ollama-auth-proxy.js +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env node -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * Authenticated reverse proxy for Ollama. - * - * Ollama has no built-in authentication. This proxy sits in front of it, - * validating a Bearer token before forwarding requests. Ollama binds to - * 127.0.0.1 (localhost only) while the proxy listens on 0.0.0.0 so the - * OpenShell gateway (running in a container) can reach it. - * - * Env: - * OLLAMA_PROXY_TOKEN — required, the Bearer token to validate - * OLLAMA_PROXY_PORT — listen port (default: 11435) - * OLLAMA_BACKEND_PORT — Ollama port on localhost (default: 11434) - */ - -const crypto = require("crypto"); -const http = require("http"); - -const TOKEN = process.env.OLLAMA_PROXY_TOKEN; -if (!TOKEN) { - console.error("OLLAMA_PROXY_TOKEN required"); - process.exit(1); -} - -const LISTEN_PORT = parseInt(process.env.OLLAMA_PROXY_PORT || "11435", 10); -const BACKEND_PORT = parseInt(process.env.OLLAMA_BACKEND_PORT || "11434", 10); - -const server = http.createServer((clientReq, clientRes) => { - // Every request must present a valid Bearer token. The proxy binds 0.0.0.0 - // so the OpenShell sandbox container can reach it via the docker bridge — - // which also means anything else with network reach to the host could, - // so unauthenticated requests are uniformly rejected (no health-check - // bypass for /api/tags). DevTest T5987914: "calls without - // Authorization: Bearer TOKEN should NOT return 200." See #3338. - // Compare buffers, not JS strings: a non-ASCII Authorization header - // can have the same .length as the expected string but a different byte - // length, which would make crypto.timingSafeEqual throw and crash the - // proxy (it binds 0.0.0.0). Build buffers first, gate timingSafeEqual on - // matching byte length. - const auth = clientReq.headers.authorization; - const expectedBuf = Buffer.from(`Bearer ${TOKEN}`); - const authBuf = typeof auth === "string" ? Buffer.from(auth) : null; - const tokenMatch = - authBuf !== null && - authBuf.length === expectedBuf.length && - crypto.timingSafeEqual(authBuf, expectedBuf); - if (!tokenMatch) { - clientRes.writeHead(401, { "Content-Type": "text/plain" }); - clientRes.end("Unauthorized"); - return; - } - - // Strip the auth header before forwarding to Ollama - const headers = { ...clientReq.headers }; - delete headers.authorization; - delete headers.host; - - const proxyReq = http.request( - { - hostname: "127.0.0.1", - port: BACKEND_PORT, - path: clientReq.url, - method: clientReq.method, - headers, - }, - (proxyRes) => { - clientRes.writeHead(proxyRes.statusCode, proxyRes.headers); - proxyRes.pipe(clientRes); - }, - ); - - proxyReq.on("error", (err) => { - clientRes.writeHead(502, { "Content-Type": "text/plain" }); - clientRes.end(`Ollama backend error: ${err.message}`); - }); - - clientReq.pipe(proxyReq); -}); - -// The proxy binds 0.0.0.0, so an unhandled listen error (most commonly -// EADDRINUSE when the port is already taken) would crash with an uncaught -// exception. Exit cleanly with a non-zero code instead; the host-side -// startOllamaAuthProxy() detects the missing process and reports the port -// owner with remediation. See #4820. -server.on("error", (/** @type {NodeJS.ErrnoException} */ err) => { - if (err && err.code === "EADDRINUSE") { - console.error(`Ollama auth proxy: port ${LISTEN_PORT} is already in use`); - } else { - console.error(`Ollama auth proxy failed to start: ${err && err.message ? err.message : err}`); - } - process.exit(1); -}); - -server.listen(LISTEN_PORT, "0.0.0.0", () => { - console.log(`Ollama auth proxy listening on 0.0.0.0:${LISTEN_PORT} -> 127.0.0.1:${BACKEND_PORT}`); -}); diff --git a/scripts/ollama-auth-proxy.mts b/scripts/ollama-auth-proxy.mts new file mode 100755 index 00000000000..983e3675b3c --- /dev/null +++ b/scripts/ollama-auth-proxy.mts @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Authenticated reverse proxy for Ollama. + * + * Ollama has no built-in authentication. This proxy sits in front of it, + * validating a Bearer token before forwarding requests. Ollama binds to + * 127.0.0.1 (localhost only) while the proxy listens on 0.0.0.0 so the + * OpenShell gateway (running in a container) can reach it. + * + * Env: + * OLLAMA_PROXY_TOKEN — required, the Bearer token to validate + * OLLAMA_PROXY_PORT — listen port (default: 11435) + * OLLAMA_BACKEND_PORT — Ollama port on localhost (default: 11434) + */ + +import crypto from "node:crypto"; +import http from "node:http"; + +const TOKEN = process.env.OLLAMA_PROXY_TOKEN; +if (!TOKEN) { + console.error("OLLAMA_PROXY_TOKEN required"); + process.exit(1); +} + +const LISTEN_PORT = parseInt(process.env.OLLAMA_PROXY_PORT || "11435", 10); +const BACKEND_PORT = parseInt(process.env.OLLAMA_BACKEND_PORT || "11434", 10); + +const server = http.createServer( + (clientReq: http.IncomingMessage, clientRes: http.ServerResponse) => { + // Every request must present a valid Bearer token. The proxy binds 0.0.0.0 + // so the OpenShell sandbox container can reach it via the docker bridge — + // which also means anything else with network reach to the host could, + // so unauthenticated requests are uniformly rejected (no health-check + // bypass for /api/tags). DevTest T5987914: "calls without + // Authorization: Bearer TOKEN should NOT return 200." See #3338. + // Compare buffers, not JS strings: a non-ASCII Authorization header + // can have the same .length as the expected string but a different byte + // length, which would make crypto.timingSafeEqual throw and crash the + // proxy (it binds 0.0.0.0). Build buffers first, gate timingSafeEqual on + // matching byte length. + const auth = clientReq.headers.authorization; + const expectedBuf = Buffer.from(`Bearer ${TOKEN}`); + const authBuf = typeof auth === "string" ? Buffer.from(auth) : null; + const tokenMatch = + authBuf !== null && + authBuf.length === expectedBuf.length && + crypto.timingSafeEqual(authBuf, expectedBuf); + if (!tokenMatch) { + clientRes.writeHead(401, { "Content-Type": "text/plain" }); + clientRes.end("Unauthorized"); + return; + } + + // Strip the auth header before forwarding to Ollama + const headers = { ...clientReq.headers }; + delete headers.authorization; + delete headers.host; + + const proxyReq = http.request( + { + hostname: "127.0.0.1", + port: BACKEND_PORT, + path: clientReq.url, + method: clientReq.method, + headers, + }, + (proxyRes: http.IncomingMessage) => { + clientRes.writeHead(proxyRes.statusCode ?? 502, proxyRes.headers); + proxyRes.pipe(clientRes); + }, + ); + + proxyReq.on("error", (err: Error) => { + clientRes.writeHead(502, { "Content-Type": "text/plain" }); + clientRes.end(`Ollama backend error: ${err.message}`); + }); + + clientReq.pipe(proxyReq); + }, +); + +// The proxy binds 0.0.0.0, so an unhandled listen error (most commonly +// EADDRINUSE when the port is already taken) would crash with an uncaught +// exception. Exit cleanly with a non-zero code instead; the host-side +// startOllamaAuthProxy() detects the missing process and reports the port +// owner with remediation. See #4820. +server.on("error", (err: NodeJS.ErrnoException) => { + if (err && err.code === "EADDRINUSE") { + console.error(`Ollama auth proxy: port ${LISTEN_PORT} is already in use`); + } else { + console.error(`Ollama auth proxy failed to start: ${err && err.message ? err.message : err}`); + } + process.exit(1); +}); + +server.listen(LISTEN_PORT, "0.0.0.0", () => { + console.log(`Ollama auth proxy listening on 0.0.0.0:${LISTEN_PORT} -> 127.0.0.1:${BACKEND_PORT}`); +}); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 282b60394a3..b4ffbf7e4e9 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -377,7 +377,7 @@ function stopMatchingPids(pattern: string, runtime: UninstallRuntime, label: str // to confirm a candidate PID is the Ollama auth proxy and not another node // process that happens to be on the same port. Mirrors the // `isOllamaProxyProcess` check in `src/lib/onboard-ollama-proxy.ts`. -const OLLAMA_AUTH_PROXY_CMDLINE_MARK = "ollama-auth-proxy.js"; +const OLLAMA_AUTH_PROXY_CMDLINE_MARK = "ollama-auth-proxy"; // Resolve the proxy port from runtime.env (rather than `process.env` at // module-load time) so a user who onboarded with NEMOCLAW_OLLAMA_PROXY_PORT diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 421f917d284..1ed5629562e 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -135,12 +135,12 @@ function loadPersistedProxyPid(): number | null { // ── Process management ─────────────────────────────────────────── function isOllamaProxyProcess(pid: number | null | undefined): boolean { - return isLocalAdapterProcess(pid, "ollama-auth-proxy.js", runCapture); + return isLocalAdapterProcess(pid, "ollama-auth-proxy", runCapture); } function spawnOllamaAuthProxy(token: string): number | null { const child = spawnDetachedNodeAdapter({ - scriptPath: path.join(SCRIPTS, "ollama-auth-proxy.js"), + scriptPath: path.join(SCRIPTS, "ollama-auth-proxy.mts"), env: { OLLAMA_PROXY_TOKEN: token, OLLAMA_PROXY_PORT: String(OLLAMA_PROXY_PORT), @@ -156,7 +156,7 @@ function killStaleProxy(): void { try { killLocalAdapterPid({ pidPath: PROXY_PID_PATH, - processNeedle: "ollama-auth-proxy.js", + processNeedle: "ollama-auth-proxy", run, runCapture, }); diff --git a/test/e2e/live/gpu-e2e-helpers.ts b/test/e2e/live/gpu-e2e-helpers.ts index 9daf570680f..0507d1a7822 100644 --- a/test/e2e/live/gpu-e2e-helpers.ts +++ b/test/e2e/live/gpu-e2e-helpers.ts @@ -242,7 +242,7 @@ sleep 2 curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" "http://127.0.0.1:$1/api/tags"`, "restart-proxy", PROXY_PORT, - path.join(REPO_ROOT, "scripts", "ollama-auth-proxy.js"), + path.join(REPO_ROOT, "scripts", "ollama-auth-proxy.mts"), ], { artifactName: "proxy-restart-from-token", diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index 4cc43263132..20e7243ba4a 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -21,7 +21,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -const PROXY_SCRIPT = path.join(REPO_ROOT, "scripts", "ollama-auth-proxy.js"); +const PROXY_SCRIPT = path.join(REPO_ROOT, "scripts", "ollama-auth-proxy.mts"); const OLLAMA_PORT = parsePort("NEMOCLAW_E2E_OLLAMA_PORT", 11434); const PROXY_PORT = parsePort("NEMOCLAW_E2E_OLLAMA_PROXY_PORT", 11435); const MODEL = process.env.NEMOCLAW_E2E_OLLAMA_PROXY_MODEL ?? "qwen2.5:0.5b"; diff --git a/test/ollama-auth-proxy-handler-helpers.ts b/test/ollama-auth-proxy-handler-helpers.ts index 28a70d2711b..a510bbd5f6c 100644 --- a/test/ollama-auth-proxy-handler-helpers.ts +++ b/test/ollama-auth-proxy-handler-helpers.ts @@ -17,7 +17,7 @@ export const PROXY_SCRIPT = path.resolve( import.meta.dirname, "..", "scripts", - "ollama-auth-proxy.js", + "ollama-auth-proxy.mts", ); const proxyOwners = new WeakMap(); diff --git a/test/ollama-auth-proxy-handler.test.ts b/test/ollama-auth-proxy-handler.test.ts index 2b1ba7e70db..4ad56016d94 100644 --- a/test/ollama-auth-proxy-handler.test.ts +++ b/test/ollama-auth-proxy-handler.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // // Mocked unit coverage for the Bearer-token enforcement and header-stripping -// contract of scripts/ollama-auth-proxy.js. The live E2E target +// contract of scripts/ollama-auth-proxy.mts. The live E2E target // (test/e2e/live/ollama-auth-proxy.test.ts) exercises the same boundary but // needs a real Ollama install plus a model pull; this pins the security- // critical request-handler behavior hermetically. diff --git a/test/ollama-proxy-recovery.test.ts b/test/ollama-proxy-recovery.test.ts index da9621496fe..d864560c9d0 100644 --- a/test/ollama-proxy-recovery.test.ts +++ b/test/ollama-proxy-recovery.test.ts @@ -105,7 +105,7 @@ console.log(JSON.stringify({ assert.equal(payload.proxySpawns.length, 1); assert.equal(payload.pid, "4242"); assert.equal(payload.proxySpawns[0].cmd, process.execPath); - assert.ok(payload.proxySpawns[0].args[0].endsWith("scripts/ollama-auth-proxy.js")); + assert.ok(payload.proxySpawns[0].args[0].endsWith("scripts/ollama-auth-proxy.mts")); assert.equal(payload.proxySpawns[0].detached, true); assert.equal(payload.proxySpawns[0].stdio, "ignore"); assert.equal(payload.proxySpawns[0].env.OLLAMA_PROXY_TOKEN, "persisted-token"); @@ -378,7 +378,7 @@ console.log(JSON.stringify({ assert.equal(payload.pid, "5000"); assert.deepEqual(payload.runCommands[0], ["kill", "4242"]); assert.equal(payload.proxySpawns[0].cmd, process.execPath); - assert.ok(payload.proxySpawns[0].args[0].endsWith("scripts/ollama-auth-proxy.js")); + assert.ok(payload.proxySpawns[0].args[0].endsWith("scripts/ollama-auth-proxy.mts")); assert.equal(payload.proxySpawns[0].env.OLLAMA_PROXY_TOKEN, "persisted-token"); assert.equal(payload.proxySpawns[0].env.OLLAMA_PROXY_PORT, "11435"); assert.equal(payload.proxySpawns[0].env.OLLAMA_BACKEND_PORT, "11434"); From cd973475256a70419d063da43575076a27c049b3 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 15 Jul 2026 14:04:10 -0700 Subject: [PATCH 2/3] fix(ollama): tighten auth proxy process matching Signed-off-by: Prekshi Vyas --- src/lib/actions/uninstall/run-plan.test.ts | 13 ++++--- src/lib/actions/uninstall/run-plan.ts | 9 ++--- src/lib/inference/bedrock-runtime-adapter.ts | 2 +- .../inference/local-adapter-lifecycle.test.ts | 36 +++++++++++++++---- src/lib/inference/local-adapter-lifecycle.ts | 13 ++++--- src/lib/inference/ollama/process.ts | 8 +++++ src/lib/inference/ollama/proxy.ts | 5 +-- .../openrouter-runtime-adapter-lifecycle.ts | 2 +- 8 files changed, 63 insertions(+), 25 deletions(-) create mode 100644 src/lib/inference/ollama/process.ts diff --git a/src/lib/actions/uninstall/run-plan.test.ts b/src/lib/actions/uninstall/run-plan.test.ts index bc9885e7dc2..c5f218ef50d 100644 --- a/src/lib/actions/uninstall/run-plan.test.ts +++ b/src/lib/actions/uninstall/run-plan.test.ts @@ -433,7 +433,10 @@ describe("uninstall run plan", () => { const logs: string[] = []; const killed: number[] = []; const exited = new Set(); - const stub = psStub("55678", { exited }); + const stub = psStub("55678", { + exited, + cmdline: "/usr/bin/node /opt/nemoclaw/scripts/ollama-auth-proxy.mts\n", + }); const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, { @@ -563,13 +566,15 @@ describe("uninstall run plan", () => { expect(logs).toContain("Stopped Ollama auth proxy 33333"); }); - it("never kills a process on :11435 whose cmdline is not the auth proxy", () => { + it.each([ + "ollama-auth-proxy-helper.mjs", + "ollama-auth-proxy.mts.backup", + ])("never kills the near-named %s process on :11435", (scriptName) => { const logs: string[] = []; const killed: number[] = []; - // Same owner, different cmdline — exercises the cmdline gate specifically. const stub = psStub("99999", { exited: new Set(), - cmdline: "/usr/sbin/nginx -g daemon off;\n", + cmdline: `/usr/bin/node /opt/nemoclaw/scripts/${scriptName}\n`, }); const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index b4ffbf7e4e9..72c6a73c6f4 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -23,6 +23,7 @@ import { providerDeleteSkipMessage, } from "../../domain/uninstall/messaging"; import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; +import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; import { stopHostGatewayProcesses } from "../../onboard/host-gateway-process"; import { isModelRouterCommandLineForPort } from "../../onboard/model-router-process"; import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup"; @@ -373,12 +374,6 @@ function stopMatchingPids(pattern: string, runtime: UninstallRuntime, label: str } } -// Identifier we look for in `/proc//cmdline` (via `ps -p -o args=`) -// to confirm a candidate PID is the Ollama auth proxy and not another node -// process that happens to be on the same port. Mirrors the -// `isOllamaProxyProcess` check in `src/lib/onboard-ollama-proxy.ts`. -const OLLAMA_AUTH_PROXY_CMDLINE_MARK = "ollama-auth-proxy"; - // Resolve the proxy port from runtime.env (rather than `process.env` at // module-load time) so a user who onboarded with NEMOCLAW_OLLAMA_PROXY_PORT // set to a custom value sees uninstall scan that same port. Mirrors the @@ -399,7 +394,7 @@ function resolveOllamaProxyPort(runtime: UninstallRuntime): number { function isOllamaAuthProxyPid(pid: number, runtime: UninstallRuntime): boolean { if (!Number.isInteger(pid) || pid <= 0) return false; const result = runtime.run("ps", ["-p", String(pid), "-o", "args="], { env: runtime.env }); - return result.status === 0 && result.stdout.includes(OLLAMA_AUTH_PROXY_CMDLINE_MARK); + return result.status === 0 && isOllamaAuthProxyCommandLine(result.stdout); } // `ps -p ` is preferred over `kill(pid, 0)` for existence probing here: diff --git a/src/lib/inference/bedrock-runtime-adapter.ts b/src/lib/inference/bedrock-runtime-adapter.ts index 83b8b91993b..93b86ae77da 100644 --- a/src/lib/inference/bedrock-runtime-adapter.ts +++ b/src/lib/inference/bedrock-runtime-adapter.ts @@ -323,7 +323,7 @@ function isAdapterProcess(pid: number | null | undefined): boolean { function killStaleAdapter(): void { killLocalAdapterPid({ pidPath: PID_PATH, - processNeedle: "bedrock-runtime-adapter.js", + processMatcher: "bedrock-runtime-adapter.js", run, runCapture, }); diff --git a/src/lib/inference/local-adapter-lifecycle.test.ts b/src/lib/inference/local-adapter-lifecycle.test.ts index 527aceb20ab..b4fde0ceb0f 100644 --- a/src/lib/inference/local-adapter-lifecycle.test.ts +++ b/src/lib/inference/local-adapter-lifecycle.test.ts @@ -22,6 +22,7 @@ import { writeLocalAdapterJsonFile, writeLocalAdapterSecretFile, } from "./local-adapter-lifecycle"; +import { isOllamaAuthProxyCommandLine } from "./ollama/process"; const tempDirs: string[] = []; const servers: http.Server[] = []; @@ -80,28 +81,51 @@ describe("local adapter lifecycle", () => { expect(fs.statSync(pidPath).mode & 0o777).toBe(0o600); }); - it("guards PID cleanup by process command line", () => { + it.each([ + "ollama-auth-proxy.js", + "ollama-auth-proxy.mts", + ])("guards PID cleanup for the supported %s script", (scriptName) => { const pidPath = path.join(tempDir(), "adapter.pid"); persistLocalAdapterPid(pidPath, 789); const killed: string[][] = []; + const commandLine = `node /opt/nemoclaw/scripts/${scriptName}`; - expect( - isLocalAdapterProcess(789, "ollama-auth-proxy.js", () => "node scripts/ollama-auth-proxy.js"), - ).toBe(true); + expect(isLocalAdapterProcess(789, isOllamaAuthProxyCommandLine, () => commandLine)).toBe(true); killLocalAdapterPid({ pidPath, - processNeedle: "ollama-auth-proxy.js", + processMatcher: isOllamaAuthProxyCommandLine, run: (args) => { killed.push(args); }, - runCapture: () => "node scripts/ollama-auth-proxy.js", + runCapture: () => commandLine, }); expect(killed).toEqual([["kill", "789"]]); expect(loadLocalAdapterPid(pidPath)).toBeNull(); }); + it.each([ + "ollama-auth-proxy-helper.mjs", + "ollama-auth-proxy.mts.backup", + ])("does not clean up the near-named %s process", (scriptName) => { + const pidPath = path.join(tempDir(), "adapter.pid"); + persistLocalAdapterPid(pidPath, 789); + const killed: string[][] = []; + + killLocalAdapterPid({ + pidPath, + processMatcher: isOllamaAuthProxyCommandLine, + run: (args) => { + killed.push(args); + }, + runCapture: () => `node /opt/nemoclaw/scripts/${scriptName}`, + }); + + expect(killed).toEqual([]); + expect(loadLocalAdapterPid(pidPath)).toBeNull(); + }); + it("probes adapter health with the expected token hash", async () => { const tokenHash = localAdapterTokenHash("secret-token"); const server = http.createServer((req, res) => { diff --git a/src/lib/inference/local-adapter-lifecycle.ts b/src/lib/inference/local-adapter-lifecycle.ts index 5c028dc1d0d..94adc8ac79b 100644 --- a/src/lib/inference/local-adapter-lifecycle.ts +++ b/src/lib/inference/local-adapter-lifecycle.ts @@ -22,6 +22,8 @@ export type RunFn = ( options?: { ignoreError?: boolean; suppressOutput?: boolean }, ) => unknown; +export type LocalAdapterProcessMatcher = string | ((commandLine: string) => boolean); + export const DEFAULT_LOCAL_ADAPTER_STATE_DIR = path.join(os.homedir(), ".nemoclaw"); export function ensureLocalAdapterStateDir(stateDir = DEFAULT_LOCAL_ADAPTER_STATE_DIR): void { @@ -105,22 +107,25 @@ export function loadLocalAdapterPid(filePath: string): number | null { export function isLocalAdapterProcess( pid: number | null | undefined, - processNeedle: string, + processMatcher: LocalAdapterProcessMatcher, runCapture: RunCaptureFn, ): boolean { if (!Number.isInteger(pid) || !pid || pid <= 0) return false; const cmdline = runCapture(["ps", "-p", String(pid), "-o", "args="], { ignoreError: true }); - return Boolean(String(cmdline || "").includes(processNeedle)); + const commandLine = String(cmdline || ""); + return typeof processMatcher === "function" + ? processMatcher(commandLine) + : commandLine.includes(processMatcher); } export function killLocalAdapterPid(options: { pidPath: string; - processNeedle: string; + processMatcher: LocalAdapterProcessMatcher; run: RunFn; runCapture: RunCaptureFn; }): void { const persistedPid = loadLocalAdapterPid(options.pidPath); - if (isLocalAdapterProcess(persistedPid, options.processNeedle, options.runCapture)) { + if (isLocalAdapterProcess(persistedPid, options.processMatcher, options.runCapture)) { options.run(["kill", String(persistedPid)], { ignoreError: true, suppressOutput: true }); } removeLocalAdapterFile(options.pidPath); diff --git a/src/lib/inference/ollama/process.ts b/src/lib/inference/ollama/process.ts new file mode 100644 index 00000000000..d77af510ba7 --- /dev/null +++ b/src/lib/inference/ollama/process.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const OLLAMA_AUTH_PROXY_SCRIPT_PATTERN = /(?:^|[\s/\\])ollama-auth-proxy\.(?:js|mts)(?=$|\s)/; + +export function isOllamaAuthProxyCommandLine(commandLine: string): boolean { + return OLLAMA_AUTH_PROXY_SCRIPT_PATTERN.test(commandLine); +} diff --git a/src/lib/inference/ollama/proxy.ts b/src/lib/inference/ollama/proxy.ts index 1ed5629562e..6a6a1ba5315 100644 --- a/src/lib/inference/ollama/proxy.ts +++ b/src/lib/inference/ollama/proxy.ts @@ -28,6 +28,7 @@ const { validateOllamaModel, } = require("../local"); const { anyRegistryModelFits, modelFitsAvailableMemory } = require("../ollama-model-registry"); +const { isOllamaAuthProxyCommandLine }: typeof import("./process") = require("./process"); const { buildSubprocessEnv } = require("../../subprocess-env"); const { prompt } = require("../../credentials/store"); const { promptManualModelId } = require("../model-prompts"); @@ -135,7 +136,7 @@ function loadPersistedProxyPid(): number | null { // ── Process management ─────────────────────────────────────────── function isOllamaProxyProcess(pid: number | null | undefined): boolean { - return isLocalAdapterProcess(pid, "ollama-auth-proxy", runCapture); + return isLocalAdapterProcess(pid, isOllamaAuthProxyCommandLine, runCapture); } function spawnOllamaAuthProxy(token: string): number | null { @@ -156,7 +157,7 @@ function killStaleProxy(): void { try { killLocalAdapterPid({ pidPath: PROXY_PID_PATH, - processNeedle: "ollama-auth-proxy", + processMatcher: isOllamaAuthProxyCommandLine, run, runCapture, }); diff --git a/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts b/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts index 166df0fe10e..c901ca2b4ed 100644 --- a/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts +++ b/src/lib/inference/openrouter-runtime-adapter-lifecycle.ts @@ -127,7 +127,7 @@ function isAdapterProcess(pid: number | null | undefined): boolean { function killStaleAdapter(): void { killLocalAdapterPid({ pidPath: PID_PATH, - processNeedle: PROCESS_NEEDLE, + processMatcher: PROCESS_NEEDLE, run, runCapture, }); From 3a4e4c646c3269311d3fa241fac45350d8ee5343 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 15 Jul 2026 14:16:38 -0700 Subject: [PATCH 3/3] test(ollama): cover occupied auth proxy port Signed-off-by: Prekshi Vyas --- src/lib/inference/ollama/process.ts | 3 +++ test/ollama-auth-proxy-handler.test.ts | 34 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/lib/inference/ollama/process.ts b/src/lib/inference/ollama/process.ts index d77af510ba7..e27a4a95a38 100644 --- a/src/lib/inference/ollama/process.ts +++ b/src/lib/inference/ollama/process.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// Keep `.js` detection for upgrade/uninstall cleanup of proxies launched by +// pre-migration releases. Remove it under #6926 once the minimum supported +// upgrade source is newer than the last release that launched the `.js` file. const OLLAMA_AUTH_PROXY_SCRIPT_PATTERN = /(?:^|[\s/\\])ollama-auth-proxy\.(?:js|mts)(?=$|\s)/; export function isOllamaAuthProxyCommandLine(commandLine: string): boolean { diff --git a/test/ollama-auth-proxy-handler.test.ts b/test/ollama-auth-proxy-handler.test.ts index 4ad56016d94..f15fdd9283c 100644 --- a/test/ollama-auth-proxy-handler.test.ts +++ b/test/ollama-auth-proxy-handler.test.ts @@ -23,7 +23,9 @@ import { afterEach, beforeEach, describe, expect, vi } from "vitest"; import { test as it } from "./helpers/owned-test-resources"; import { + forceKill, freePort, + PROXY_SCRIPT, request, startBackend, startProxy, @@ -127,6 +129,38 @@ describe("ollama-auth-proxy request handler", () => { }); describe("ollama-auth-proxy process ownership", () => { + it("reports EADDRINUSE and exits nonzero when the configured port is occupied", async ({ + onTestFinished, + resources, + }) => { + const portOwner = resources.ownServer(net.createServer()); + await new Promise((resolve, reject) => { + portOwner.once("error", reject); + portOwner.listen(0, "0.0.0.0", resolve); + }); + const occupiedPort = (portOwner.address() as AddressInfo).port; + const child = spawn(process.execPath, [PROXY_SCRIPT], { + env: { + ...process.env, + OLLAMA_PROXY_TOKEN: TOKEN, + OLLAMA_PROXY_PORT: String(occupiedPort), + OLLAMA_BACKEND_PORT: "1", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + onTestFinished(() => forceKill(child)); + const stderrChunks: Buffer[] = []; + child.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk))); + + const [exitCode, signal] = (await once(child, "close")) as [number | null, string | null]; + const stderr = Buffer.concat(stderrChunks).toString("utf8"); + + expect(signal).toBeNull(); + expect(exitCode).not.toBe(0); + expect(stderr).toContain(`Ollama auth proxy: port ${occupiedPort} is already in use`); + expect(stderr).not.toContain(TOKEN); + }); + it("reaps the proxy before reporting a readiness failure", async ({ onTestFinished, resources,