diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 288b96db43..1acd796483 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3625,7 +3625,8 @@ Passthrough commands do not consume flags intended for the downstream command as If a port value is not a valid integer or falls outside the allowed range, the CLI exits with an error. -`NEMOCLAW_GATEWAY_PORT` also cannot overlap configured service, vLLM, Ollama, Ollama proxy, OpenRouter runtime adapter, or HTTPS Pin Runtime adapter ports, and cannot use reserved auto-allocation ranges or the default inference/proxy ports `8000`, `11434`, `11435`, `11437`, and `11438`. +`NEMOCLAW_GATEWAY_PORT` also cannot overlap configured service, vLLM, Ollama, Ollama proxy, OpenRouter runtime adapter, or HTTPS Pin Runtime adapter ports, and cannot use reserved auto-allocation ranges or the default inference/proxy ports `8000`, `8081`, `11434`, `11435`, `11437`, and `11438`. +Port `8081` is reserved for authenticated llama.cpp existing-server attachment and cannot be assigned to any configurable NemoClaw service port. When you select OpenRouter, `NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT` must also be distinct from the gateway, vLLM, Ollama, Ollama proxy, and HTTPS Pin Runtime adapter ports. When you run multiple NemoClaw gateways with different `NEMOCLAW_GATEWAY_PORT` values, NemoClaw derives a separate gateway name, state directory, and compatibility container name from the port so one gateway does not tear down another. Only port `8080` uses a NemoClaw-managed Linux systemd user service or macOS Homebrew service. @@ -3690,7 +3691,7 @@ Set them before running `$$nemoclaw onboard`. | Variable | Format | Effect | |----------|--------|--------| -| `NEMOCLAW_PROVIDER` | provider key (e.g. `build`, `openrouter`, `openai`, `anthropic`, `anthropicCompatible`, `gemini`, `ollama`, `custom`, `vllm`, `nim-local`, `routed`, `hermes-provider`, `install-vllm`, `install-ollama`, `install-windows-ollama`, `start-windows-ollama`) | Selects the inference provider during onboarding. The wizard skips the provider menu in both interactive and non-interactive runs when this is set. Aliases: `cloud` → `build`, `open-router` / `openrouterai` → `openrouter`, `nim` → `nim-local`, `hermes` / `nous` / `nous-portal` → `hermes-provider`, `anthropiccompatible` → `anthropicCompatible`. Invalid values fail fast with the list of accepted keys. | +| `NEMOCLAW_PROVIDER` | provider key (e.g. `build`, `openrouter`, `openai`, `anthropic`, `anthropicCompatible`, `gemini`, `ollama`, `custom`, `vllm`, `nim-local`, `routed`, `hermes-provider`, `llama-cpp`, `install-vllm`, `install-ollama`, `install-windows-ollama`, `start-windows-ollama`) | Selects the inference provider during onboarding. The wizard skips the provider menu in both interactive and non-interactive runs when this is set. `llama-cpp` selects attachment of an authenticated, operator-managed llama.cpp server on loopback port `8081`. Set `NEMOCLAW_LLAMACPP_LOCAL_TOKEN`; set `NEMOCLAW_MODEL` to the served alias when the server exposes multiple models. If the server does not provide consistent native llama.cpp evidence, select `custom`. Aliases: `cloud` → `build`, `open-router` / `openrouterai` → `openrouter`, `nim` → `nim-local`, `hermes` / `nous` / `nous-portal` → `hermes-provider`, `anthropiccompatible` → `anthropicCompatible`. Invalid values fail fast with the list of accepted keys. | | `NEMOCLAW_TOOL_DISCLOSURE` | `progressive` or `direct` | Selects progressive tool discovery or the prior direct-exposure behavior. Defaults to `progressive`; `--tool-disclosure` takes precedence when both are set. | | `NEMOCLAW_ENDPOINT_URL` | URL | Custom endpoint URL. Used together with `NEMOCLAW_PROVIDER=custom` for OpenAI-compatible endpoints or `NEMOCLAW_PROVIDER=anthropicCompatible` for Anthropic-compatible endpoints. | | `NEMOCLAW_COMPATIBLE_AUTH_MODE` | `none` or unset | Explicitly selects no authentication for an HTTP OpenAI-compatible endpoint using `localhost`, `127.0.0.1`, or `[::1]` and port `8000`, `11434`, or `11435` during non-interactive onboarding. | diff --git a/install.sh b/install.sh index add4c49fb8..4f06875586 100755 --- a/install.sh +++ b/install.sh @@ -154,7 +154,7 @@ bootstrap_usage() { printf " Exact JSON array of pre-fingerprint managed sandbox names\n" printf " NEMOCLAW_PROVIDER build | openrouter | openai | anthropic | anthropicCompatible\n" printf " | gemini | ollama | custom | nim-local | vllm | routed\n" - printf " | hermes-provider\n" + printf " | hermes-provider | llama-cpp\n" printf " (aliases: cloud -> build, nim -> nim-local)\n" printf " NEMOCLAW_POLICY_MODE suggested | custom | skip\n" printf "\n" diff --git a/nemoclaw-blueprint/policies/presets/local-inference.yaml b/nemoclaw-blueprint/policies/presets/local-inference.yaml index 73a2222220..2689a73283 100644 --- a/nemoclaw-blueprint/policies/presets/local-inference.yaml +++ b/nemoclaw-blueprint/policies/presets/local-inference.yaml @@ -3,12 +3,23 @@ preset: name: local-inference - description: "Local inference access (Ollama, vLLM) via host gateway" + description: "Local inference access (Ollama, vLLM, llama.cpp) through the OpenShell gateway" network_policies: local_inference: name: local_inference endpoints: + - host: host.openshell.internal + port: 8081 + protocol: rest + enforcement: enforce + allowed_ips: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } - host: host.openshell.internal port: 11434 protocol: rest diff --git a/scripts/install.sh b/scripts/install.sh index a34dba8d1a..32b7074c10 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -227,7 +227,7 @@ resolve_nemoclaw_gateway_port() { error "NEMOCLAW_GATEWAY_PORT must not overlap the 18789-18799 dashboard port range." fi case "$port" in - 8000 | 11434 | 11435 | 11436 | 11437) + 8000 | 8081 | 11434 | 11435 | 11436 | 11437) error "NEMOCLAW_GATEWAY_PORT must not overlap a reserved inference or runtime-adapter port ($port)." ;; esac @@ -238,6 +238,7 @@ resolve_nemoclaw_gateway_port() { NEMOCLAW_OLLAMA_PROXY_PORT NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT + NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT ) local -a configured_ports=( "${NEMOCLAW_DASHBOARD_PORT:-18789}" @@ -246,12 +247,16 @@ resolve_nemoclaw_gateway_port() { "${NEMOCLAW_OLLAMA_PROXY_PORT:-11435}" "${NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT:-11436}" "${NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT:-11437}" + "${NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT:-11438}" ) local i configured_port for i in "${!configured_ports[@]}"; do configured_port="${configured_ports[$i]}" configured_port="${configured_port#"${configured_port%%[![:space:]]*}"}" configured_port="${configured_port%"${configured_port##*[![:space:]]}"}" + if [[ "$configured_port" =~ ^0*8081$ ]]; then + error "${configured_names[$i]} must not overlap the fixed llama.cpp inference port (8081)." + fi if [[ "$configured_port" =~ ^[0-9]+$ ]] && [ "$port" -eq "$configured_port" ]; then error "NEMOCLAW_GATEWAY_PORT conflicts with ${configured_names[$i]} ($configured_port)." fi @@ -817,7 +822,7 @@ usage() { printf " NEMOCLAW_INSTALL_REF Exact Git ref/SHA to install\n" printf " NEMOCLAW_PROVIDER build | openrouter | openai | anthropic | anthropicCompatible\n" printf " | gemini | ollama | custom | nim-local | vllm | routed\n" - printf " | hermes-provider\n" + printf " | hermes-provider | llama-cpp\n" printf " (aliases: cloud -> build, nim -> nim-local)\n" printf " NEMOCLAW_MODEL Inference model to configure\n" printf " NEMOCLAW_POLICY_MODE suggested | custom | skip\n" diff --git a/scripts/lib/runtime.sh b/scripts/lib/runtime.sh index b9447044e5..6d0d2d6006 100755 --- a/scripts/lib/runtime.sh +++ b/scripts/lib/runtime.sh @@ -243,6 +243,11 @@ _validate_port() { return 1 ;; esac + if [[ "$value" =~ ^0*8081$ ]]; then + printf 'Invalid %s=%s (conflicts with fixed llama.cpp inference port 8081)\n' \ + "$name" "$value" >&2 + return 1 + fi if ! { [ "$value" -ge 1024 ] && [ "$value" -le 65535 ]; }; then printf 'Invalid %s=%s (expected 1024-65535)\n' "$name" "$value" >&2 return 1 diff --git a/src/lib/actions/sandbox/rebuild-route-preflight.test.ts b/src/lib/actions/sandbox/rebuild-route-preflight.test.ts index 15779b6911..bf3763bfc2 100644 --- a/src/lib/actions/sandbox/rebuild-route-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-route-preflight.test.ts @@ -89,6 +89,15 @@ const remoteProviders = [ (provider): provider is typeof provider & { credentialEnv: string } => typeof provider.credentialEnv === "string" && provider.credentialEnv.length > 0, ); +const remoteProviderRouteOverrides = new Map>([ + [ + REMOTE_PROVIDER_CONFIG["llama-cpp"].providerName, + { + endpointUrl: REMOTE_PROVIDER_CONFIG["llama-cpp"].endpointUrl ?? null, + preferredInferenceApi: "openai-completions", + }, + ], +]); describe("commitRebuildRoutePreflight", () => { it("includes a credential-bearing provider in the migration matrix (#7798)", () => { @@ -98,10 +107,12 @@ describe("commitRebuildRoutePreflight", () => { it.each( remoteProviders, )("migrates missing shared-gateway credential identity for $providerName (#7798)", (providerConfig) => { + const routeOverrides = remoteProviderRouteOverrides.get(providerConfig.providerName) ?? {}; const target = sandbox("target", providerConfig.providerName, { + ...routeOverrides, credentialEnv: providerConfig.credentialEnv, }); - const peer = sandbox("peer", providerConfig.providerName); + const peer = sandbox("peer", providerConfig.providerName, routeOverrides); const state = transactionDependencies(registry(target, peer)); const result = commitRebuildRoutePreflight( diff --git a/src/lib/adapters/http/curl-args.test.ts b/src/lib/adapters/http/curl-args.test.ts index b1b595f428..8a0ece5ed1 100644 --- a/src/lib/adapters/http/curl-args.test.ts +++ b/src/lib/adapters/http/curl-args.test.ts @@ -4,9 +4,33 @@ import { describe, expect, it } from "vitest"; import { assertEndpointResolvesPublic } from "../../inference/endpoint-ssrf-preflight"; -import { validateCurlProbeArgs } from "./curl-args"; +import { buildBoundedCurlProbeSpawnArgs, validateCurlProbeArgs } from "./curl-args"; describe("validateCurlProbeArgs — credential-leak defence", () => { + it("allows a fixed response-byte cap for bounded llama.cpp probes (#8161)", () => { + expect( + validateCurlProbeArgs(["-sS", "--max-filesize", "262144", "http://127.0.0.1:8081/v1/models"]) + .args, + ).toEqual(["-sS", "--max-filesize", "262144"]); + }); + + it("rebuilds bounded llama.cpp probe argv from validated fields (#8161)", () => { + expect( + buildBoundedCurlProbeSpawnArgs( + ["-sS", "--max-filesize", "262144"], + "http://127.0.0.1:8081/v1/models", + "\n__NEMOCLAW_HTTP_STATUS_test__:", + ), + ).toEqual([ + "-sS", + "--max-filesize", + "262144", + "-w", + "\n__NEMOCLAW_HTTP_STATUS_test__:%{http_code}", + "http://127.0.0.1:8081/v1/models", + ]); + }); + it("rejects an inline Authorization header so credentials cannot reach argv", () => { expect(() => validateCurlProbeArgs([ diff --git a/src/lib/adapters/http/curl-args.ts b/src/lib/adapters/http/curl-args.ts index 786f36f471..1afd23f1a1 100644 --- a/src/lib/adapters/http/curl-args.ts +++ b/src/lib/adapters/http/curl-args.ts @@ -69,7 +69,13 @@ const CURL_SAFE_FLAG_OPTIONS = new Set([ // genuinely need to follow redirects from a fixed, hardcoded host (e.g. the // Ollama manifest probe) must opt in via CurlProbeArgOptions.allowRedirects. const CURL_REDIRECT_FLAG_OPTIONS = new Set(["-L", "-sfL", "--location"]); -const CURL_SAFE_VALUE_OPTIONS = new Set(["--connect-timeout", "--max-time", "-X", "--request"]); +const CURL_SAFE_VALUE_OPTIONS = new Set([ + "--connect-timeout", + "--max-time", + "--max-filesize", + "-X", + "--request", +]); const CURL_FORBIDDEN_MULTI_TRANSFER_OPTIONS = new Set(["--next"]); const CURL_SHORT_OPTIONS_WITH_VALUES = new Set(["-K", "-b", "-T", "-d", "-F", "-H", "-X"]); @@ -388,3 +394,12 @@ export function buildCurlProbeSpawnArgs( // lgtm[js/file-access-to-http] URL/argv are validated; file-backed config paths must be explicitly trusted. return [...args, ...outputArgs, ...statusArgs, url]; } + +export function buildBoundedCurlProbeSpawnArgs( + args: string[], + url: string, + statusMarker: string, +): string[] { + // lgtm[js/file-access-to-http] URL/argv are validated; the status marker is generated in-process. + return [...args, "-w", `${statusMarker}%{http_code}`, url]; +} diff --git a/src/lib/adapters/http/probe.test.ts b/src/lib/adapters/http/probe.test.ts index 39672e2a4c..b90ff7e695 100644 --- a/src/lib/adapters/http/probe.test.ts +++ b/src/lib/adapters/http/probe.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawn } from "node:child_process"; +import { once } from "node:events"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -104,6 +106,92 @@ describe("http-probe helpers", () => { expect(fs.existsSync(path.dirname(outputPath))).toBe(false); }); + it("captures a response at the process byte limit without a body temp file (#8161)", () => { + const body = "x".repeat(1024); + let maxBuffer: number | undefined; + const result = runCurlProbe(["-sS", "https://example.test/models"], { + maxResponseBytes: 1024, + spawnSyncImpl: (_command, args, options) => { + maxBuffer = options.maxBuffer; + expect(args).not.toContain("-o"); + const writeOut = args[args.indexOf("-w") + 1]; + const statusMarker = writeOut.slice(0, -"%{http_code}".length); + return { + pid: 1, + output: [], + stdout: `${body}${statusMarker}200`, + stderr: "", + status: 0, + signal: null, + }; + }, + }); + + expect(result).toMatchObject({ ok: true, httpStatus: 200, curlStatus: 0, body }); + expect(maxBuffer).toBeGreaterThan(1024); + expect(maxBuffer).toBeLessThan(1150); + }); + + it("maps a process buffer overflow to curl's oversized-response status (#8161)", () => { + const result = runCurlProbe(["-sS", "https://example.test/models"], { + maxResponseBytes: 1024, + spawnSyncImpl: () => ({ + pid: 1, + output: [], + stdout: "partial untrusted response", + stderr: "partial diagnostic", + status: null, + signal: "SIGTERM", + error: Object.assign(new Error("spawnSync curl ENOBUFS"), { code: "ENOBUFS" }), + }), + }); + + expect(result).toMatchObject({ + ok: false, + httpStatus: 0, + curlStatus: 63, + body: "", + stderr: "curl response exceeded the configured process byte limit", + }); + expect(result.message).not.toContain("partial untrusted response"); + expect(result.message).not.toContain("partial diagnostic"); + }); + + it("aborts an unknown-length chunked response at the process byte limit (#8161)", async () => { + const serverScript = String.raw` + const http = require("node:http"); + const server = http.createServer((_request, response) => { + response.writeHead(200, { + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + }); + response.write("x".repeat(128 * 1024)); + response.end("x".repeat(128 * 1024)); + }); + server.listen(0, "127.0.0.1", () => { + process.stdout.write(String(server.address().port) + "\n"); + }); + process.on("SIGTERM", () => server.close(() => process.exit(0))); + `; + const server = spawn(process.execPath, ["-e", serverScript], { + stdio: ["ignore", "pipe", "pipe"], + }); + const exit = once(server, "exit"); + const [portOutput] = await once(server.stdout, "data"); + + try { + const result = runCurlProbe( + ["-sS", "--max-time", "5", `http://127.0.0.1:${Number(String(portOutput).trim())}/`], + { maxResponseBytes: 1024, pinnedAddresses: [] }, + ); + + expect(result).toMatchObject({ ok: false, httpStatus: 0, curlStatus: 63, body: "" }); + } finally { + server.kill("SIGTERM"); + await exit; + } + }); + it("lets the process wrapper outlive curl --max-time", () => { let timeout: number | undefined; const result = runCurlProbe(["-sS", "--max-time", "60", "https://example.test/models"], { diff --git a/src/lib/adapters/http/probe.ts b/src/lib/adapters/http/probe.ts index 0529e1194e..f217625d4b 100644 --- a/src/lib/adapters/http/probe.ts +++ b/src/lib/adapters/http/probe.ts @@ -6,6 +6,7 @@ import { type SpawnSyncReturns, spawnSync, } from "node:child_process"; +import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -16,7 +17,11 @@ import type { ProbeResult } from "../../onboard/types"; import { buildScrubbedCurlProbeEnv, scrubCredentialEnv } from "../../security/credential-env"; import { ROOT } from "../../state/paths"; import { addTraceEvent, withTraceSpan } from "../../trace"; -import { buildCurlProbeSpawnArgs, validateCurlProbeArgs } from "./curl-args"; +import { + buildBoundedCurlProbeSpawnArgs, + buildCurlProbeSpawnArgs, + validateCurlProbeArgs, +} from "./curl-args"; export type CurlProbeResult = ProbeResult; @@ -25,6 +30,8 @@ export interface CurlProbeOptions { env?: NodeJS.ProcessEnv; replaceEnv?: boolean; timeoutMs?: number; + /** Maximum response-body bytes captured by the curl process. */ + maxResponseBytes?: number; /** Absolute or cwd-relative curl config files created by trusted NemoClaw callers. */ trustedConfigFiles?: readonly string[]; /** @@ -50,6 +57,8 @@ export interface StreamingProbeResult { const DEFAULT_CURL_PROCESS_TIMEOUT_MS = 30_000; const CURL_PROCESS_TIMEOUT_SLACK_MS = 5_000; +const CURL_HTTP_STATUS_MARKER_PREFIX = "\n__NEMOCLAW_HTTP_STATUS_"; +const CURL_OVERSIZED_RESPONSE_STATUS = 63; function resolveCurlProbeSpawnEnv( args: readonly string[], @@ -152,6 +161,31 @@ function normalizeSpawnErrorCode(error: unknown): number { return typeof rawErrorCode === "number" ? rawErrorCode : 1; } +function normalizeMaxResponseBytes(value: number | undefined): number | null { + if (value === undefined) return null; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error("curl probe maxResponseBytes must be a positive safe integer"); + } + return value; +} + +function isSpawnBufferOverflow(error: unknown): boolean { + return isErrnoException(error) && error.code === "ENOBUFS"; +} + +function splitBoundedCurlOutput( + stdout: string, + statusMarker: string, +): { body: string; status: number } { + const markerIndex = stdout.lastIndexOf(statusMarker); + if (markerIndex < 0) return { body: "", status: 0 }; + const status = Number(stdout.slice(markerIndex + statusMarker.length).trim()); + return { + body: stdout.slice(0, markerIndex), + status: Number.isFinite(status) ? status : 0, + }; +} + function sanitizeCurlUrl(value: string): string { try { const url = new URL(value); @@ -261,7 +295,16 @@ function runCurlProbeImpl(argv: string[], opts: CurlProbeOptions = {}): CurlProb const { args, url } = validateCurlProbeArgs(argv, opts); const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; const timeout = resolveCurlProcessTimeoutMs(argv, opts); - const curlArgs = buildCurlProbeSpawnArgs(args, url, bodyFile, "json"); + const maxResponseBytes = normalizeMaxResponseBytes(opts.maxResponseBytes); + const statusMarker = `${CURL_HTTP_STATUS_MARKER_PREFIX}${randomUUID()}__:`; + const curlArgs = + maxResponseBytes === null + ? buildCurlProbeSpawnArgs(args, url, bodyFile, "json") + : buildBoundedCurlProbeSpawnArgs(args, url, statusMarker); + const maxBuffer = + maxResponseBytes === null + ? undefined + : maxResponseBytes + Buffer.byteLength(`${statusMarker}999`); const result = spawnSyncImpl( "curl", // lgtm[js/file-access-to-http] curlArgs were validated and rebuilt from safe probe fields. @@ -271,26 +314,54 @@ function runCurlProbeImpl(argv: string[], opts: CurlProbeOptions = {}): CurlProb encoding: "utf8", timeout, env: resolveCurlProbeSpawnEnv(args, opts), + ...(maxBuffer === undefined ? {} : { maxBuffer }), }, ); - const body = fs.existsSync(bodyFile) ? fs.readFileSync(bodyFile, "utf8") : ""; + const boundedOutput = + maxResponseBytes === null + ? null + : splitBoundedCurlOutput(String(result.stdout || ""), statusMarker); + const body = + boundedOutput?.body ?? (fs.existsSync(bodyFile) ? fs.readFileSync(bodyFile, "utf8") : ""); if (result.error) { - const errorCode = normalizeSpawnErrorCode(result.error); - const errorMessage = compactText( - `${result.error.message || String(result.error)} ${String(result.stderr || "")}`, - ); + const overflow = isSpawnBufferOverflow(result.error); + const errorCode = overflow + ? CURL_OVERSIZED_RESPONSE_STATUS + : normalizeSpawnErrorCode(result.error); + const errorMessage = overflow + ? "curl response exceeded the configured process byte limit" + : compactText( + `${result.error.message || String(result.error)} ${String(result.stderr || "")}`, + ); const failure = { ok: false, httpStatus: 0, curlStatus: errorCode, - body, + body: overflow ? "" : body, stderr: errorMessage, - message: summarizeProbeFailure(body, 0, errorCode, errorMessage), + message: summarizeProbeFailure(overflow ? "" : body, 0, errorCode, errorMessage), }; emitCurlResultTraceEvent({ ok: false, http_status: 0, curl_status: errorCode }); return failure; } - const status = Number(String(result.stdout || "").trim()); + if (maxResponseBytes !== null && Buffer.byteLength(body) > maxResponseBytes) { + const errorMessage = "curl response exceeded the configured process byte limit"; + const failure = { + ok: false, + httpStatus: 0, + curlStatus: CURL_OVERSIZED_RESPONSE_STATUS, + body: "", + stderr: errorMessage, + message: summarizeCurlFailure(CURL_OVERSIZED_RESPONSE_STATUS, errorMessage), + }; + emitCurlResultTraceEvent({ + ok: false, + http_status: 0, + curl_status: CURL_OVERSIZED_RESPONSE_STATUS, + }); + return failure; + } + const status = boundedOutput?.status ?? Number(String(result.stdout || "").trim()); const probeResult = { ok: result.status === 0 && status >= 200 && status < 300, httpStatus: Number.isFinite(status) ? status : 0, diff --git a/src/lib/core/ports.test.ts b/src/lib/core/ports.test.ts index e141704054..878e62f3ad 100644 --- a/src/lib/core/ports.test.ts +++ b/src/lib/core/ports.test.ts @@ -7,6 +7,7 @@ import { parseGatewayPort, parsePort, validateHttpsPinRuntimeAdapterPort, + validateLlamaCppPortReservation, validateOpenRouterRuntimeAdapterPort, } from "./ports"; @@ -160,6 +161,26 @@ describe("parseGatewayPort", () => { }); }); +describe("validateLlamaCppPortReservation", () => { + it.each([ + "gatewayPort", + "dashboardPort", + "vllmPort", + "ollamaPort", + "ollamaProxyPort", + "bedrockRuntimeAdapterPort", + "openrouterRuntimeAdapterPort", + "httpsPinRuntimeAdapterPort", + ] as const)("rejects configured %s collision with fixed port 8081 (#8161)", (field) => { + expect(() => + validateLlamaCppPortReservation({ + ...GATEWAY_VALIDATION_OPTIONS, + [field]: 8081, + }), + ).toThrow(/fixed llama\.cpp inference port \(8081\)/); + }); +}); + describe("validateOpenRouterRuntimeAdapterPort", () => { const ENV_KEY = "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT"; diff --git a/src/lib/core/ports.ts b/src/lib/core/ports.ts index 31ddb4985f..e1ba0f4f88 100644 --- a/src/lib/core/ports.ts +++ b/src/lib/core/ports.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { LLAMA_CPP_PORT } from "../inference/llama-cpp/contract"; + /** * Central port configuration — override any port via environment variables. * TypeScript counterpart of bin/lib/ports.js. @@ -60,6 +62,8 @@ export const VLLM_PORT = parsePort("NEMOCLAW_VLLM_PORT", 8000); export const OLLAMA_PORT = parsePort("NEMOCLAW_OLLAMA_PORT", 11434); /** Ollama auth proxy port (default 11435, override via NEMOCLAW_OLLAMA_PROXY_PORT). */ export const OLLAMA_PROXY_PORT = parsePort("NEMOCLAW_OLLAMA_PROXY_PORT", 11435); +/** llama.cpp existing-server attachment port; fixed by the declarative serving contract. */ +export { LLAMA_CPP_PORT }; /** Hermes OpenAI-compatible API port (manifest `forward_ports[1]` / start.sh `PUBLIC_PORT`); reserved — never a valid dashboard port, for any agent. (#4984) */ export const HERMES_OPENAI_API_PORT = 8642; /** Bedrock Runtime adapter port (default 11436, override via NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT). */ @@ -90,6 +94,7 @@ export function validateGatewayPort( } const reservedDefaults = [ + { label: "llama.cpp inference", port: LLAMA_CPP_PORT }, { label: "vLLM / NIM inference", port: 8000 }, { label: "Ollama inference", port: 11434 }, { label: "Ollama auth proxy", port: 11435 }, @@ -152,6 +157,7 @@ export function validateOpenRouterRuntimeAdapterPort( } const reservedDefaults = [ + { label: "llama.cpp inference", port: LLAMA_CPP_PORT }, { label: "vLLM / NIM inference", port: 8000 }, { label: "Ollama inference", port: 11434 }, { label: "Ollama auth proxy", port: 11435 }, @@ -200,6 +206,7 @@ export function validateHttpsPinRuntimeAdapterPort( } const reservedDefaults = [ + { label: "llama.cpp inference", port: LLAMA_CPP_PORT }, { label: "vLLM / NIM inference", port: 8000 }, { label: "Ollama inference", port: 11434 }, { label: "Ollama auth proxy", port: 11435 }, @@ -250,3 +257,47 @@ export const GATEWAY_PORT = parseGatewayPort("NEMOCLAW_GATEWAY_PORT", DEFAULT_GA openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, }); + +/** Reject every configurable service collision with fixed llama.cpp attachment port 8081. */ +export function validateLlamaCppPortReservation( + options: RuntimeAdapterPortValidationOptions, +): void { + const configuredPorts = [ + { envVar: "NEMOCLAW_GATEWAY_PORT", port: options.gatewayPort }, + { envVar: "NEMOCLAW_DASHBOARD_PORT", port: options.dashboardPort }, + { envVar: "NEMOCLAW_VLLM_PORT", port: options.vllmPort }, + { envVar: "NEMOCLAW_OLLAMA_PORT", port: options.ollamaPort }, + { envVar: "NEMOCLAW_OLLAMA_PROXY_PORT", port: options.ollamaProxyPort }, + { + envVar: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT", + port: options.bedrockRuntimeAdapterPort, + }, + { + envVar: "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", + port: options.openrouterRuntimeAdapterPort, + }, + { + envVar: "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + port: options.httpsPinRuntimeAdapterPort, + }, + ]; + const conflict = configuredPorts.find(({ port }) => port === LLAMA_CPP_PORT); + if (conflict) { + throw new Error( + `Invalid port: ${conflict.envVar}="${LLAMA_CPP_PORT}" — conflicts with the fixed llama.cpp inference port (${LLAMA_CPP_PORT})`, + ); + } +} + +validateLlamaCppPortReservation({ + gatewayPort: GATEWAY_PORT, + dashboardPort: DASHBOARD_PORT, + dashboardRangeStart: DASHBOARD_PORT_RANGE_START, + dashboardRangeEnd: DASHBOARD_PORT_RANGE_END, + vllmPort: VLLM_PORT, + ollamaPort: OLLAMA_PORT, + ollamaProxyPort: OLLAMA_PROXY_PORT, + bedrockRuntimeAdapterPort: BEDROCK_RUNTIME_ADAPTER_PORT, + openrouterRuntimeAdapterPort: OPENROUTER_RUNTIME_ADAPTER_PORT, + httpsPinRuntimeAdapterPort: HTTPS_PIN_RUNTIME_ADAPTER_PORT, +}); diff --git a/src/lib/credentials/store.ts b/src/lib/credentials/store.ts index eb038859b8..374350bd92 100644 --- a/src/lib/credentials/store.ts +++ b/src/lib/credentials/store.ts @@ -43,6 +43,7 @@ export const KNOWN_CREDENTIAL_ENV_KEYS: readonly string[] = [ "GEMINI_API_KEY", "COMPATIBLE_API_KEY", "COMPATIBLE_ANTHROPIC_API_KEY", + "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", "BRAVE_API_KEY", "TAVILY_API_KEY", "GITHUB_TOKEN", diff --git a/src/lib/inference/config.test.ts b/src/lib/inference/config.test.ts index a3c784c086..4d23cad43c 100644 --- a/src/lib/inference/config.test.ts +++ b/src/lib/inference/config.test.ts @@ -19,6 +19,7 @@ import { getSandboxInferenceConfig, HERMES_PROVIDER_MODEL_OPTIONS, INFERENCE_ROUTE_URL, + LLAMA_CPP_LOCAL_CREDENTIAL_ENV, MANAGED_PROVIDER_ID, OLLAMA_LOCAL_CREDENTIAL_ENV, parseGatewayInference, @@ -160,6 +161,40 @@ describe("inference selection config", () => { expect(OLLAMA_LOCAL_CREDENTIAL_ENV).not.toBe(DEFAULT_ROUTE_CREDENTIAL_ENV); }); + it("maps llama.cpp attachment to inference.local with Chat Completions (#8161)", () => { + expect(getProviderSelectionConfig("llama-cpp-local", "team/model-alias")).toEqual({ + endpointType: "custom", + endpointUrl: INFERENCE_ROUTE_URL, + ncpPartner: null, + model: "team/model-alias", + profile: DEFAULT_ROUTE_PROFILE, + credentialEnv: LLAMA_CPP_LOCAL_CREDENTIAL_ENV, + provider: "llama-cpp-local", + providerLabel: "Local llama.cpp", + }); + expect( + getSandboxInferenceConfig("team/model-alias", "llama-cpp-local", "openai-responses"), + ).toEqual({ + providerKey: MANAGED_PROVIDER_ID, + primaryModelRef: `${MANAGED_PROVIDER_ID}/team/model-alias`, + inferenceBaseUrl: INFERENCE_ROUTE_URL, + inferenceApi: "openai-completions", + inferenceCompat: { supportsStore: false }, + }); + }); + + it.each([ + undefined, + "", + " ", + "/models/model.gguf", + "models/../secret", + "foo/./bar", + "a".repeat(257), + ])("refuses llama.cpp selection without a validated served alias: %s (#8161)", (model) => { + expect(getProviderSelectionConfig("llama-cpp-local", model)).toBeNull(); + }); + it("maps nvidia-nim to the sandbox inference route", () => { expect(getProviderSelectionConfig("nvidia-nim", "nvidia/nemotron-3-super-120b-a12b")).toEqual({ endpointType: "custom", diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index 5b6e0d90d0..b21fe40c2f 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -7,6 +7,7 @@ */ import { isSafeModelId, shouldSkipResponsesProbe } from "../validation"; +import { isSafeLlamaCppServedModelAlias, LLAMA_CPP_CREDENTIAL_ENV } from "./llama-cpp/contract"; import { DEFAULT_OLLAMA_MODEL } from "./local"; import { OPENROUTER_CREDENTIAL_ENV, OPENROUTER_PROVIDER_NAME } from "./openrouter"; @@ -65,6 +66,7 @@ export const DEFAULT_ROUTE_CREDENTIAL_ENV = "OPENAI_API_KEY"; // never read the user's host OpenAI key for local providers. See GH #2519. export const OLLAMA_LOCAL_CREDENTIAL_ENV = "NEMOCLAW_OLLAMA_PROXY_TOKEN"; export const VLLM_LOCAL_CREDENTIAL_ENV = "NEMOCLAW_VLLM_LOCAL_TOKEN"; +export const LLAMA_CPP_LOCAL_CREDENTIAL_ENV = LLAMA_CPP_CREDENTIAL_ENV; export const MANAGED_PROVIDER_ID = "inference"; export { DEFAULT_OLLAMA_MODEL }; @@ -215,6 +217,14 @@ export function getProviderSelectionConfig( credentialEnv: OLLAMA_LOCAL_CREDENTIAL_ENV, providerLabel: "Local Ollama", }; + case "llama-cpp-local": + if (!model || !isSafeLlamaCppServedModelAlias(model)) return null; + return { + ...base, + model, + credentialEnv: LLAMA_CPP_LOCAL_CREDENTIAL_ENV, + providerLabel: "Local llama.cpp", + }; default: return null; } @@ -276,6 +286,7 @@ export function getSandboxInferenceConfig( }; break; case "compatible-endpoint": + case "llama-cpp-local": providerKey = MANAGED_PROVIDER_ID; primaryModelRef = `${MANAGED_PROVIDER_ID}/${model}`; inferenceCompat = { diff --git a/src/lib/inference/gateway-route-compatibility.test.ts b/src/lib/inference/gateway-route-compatibility.test.ts index 3680db6404..7b53350a4a 100644 --- a/src/lib/inference/gateway-route-compatibility.test.ts +++ b/src/lib/inference/gateway-route-compatibility.test.ts @@ -105,6 +105,54 @@ describe("shared gateway inference route compatibility", () => { }); }); + it("preserves llama.cpp endpoint and completions API as durable route identity (#8161)", () => { + const result = discover( + discoveryRoute("llama-cpp-local", { + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + }), + [ + sandbox("llama-peer", { + provider: "llama-cpp-local", + model: "team/model-alias", + endpointUrl: "http://127.0.0.1:8081/v1", + preferredInferenceApi: "openai-completions", + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + }), + ], + ); + + expect(result).toEqual({ + ok: true, + requiredModel: "team/model-alias", + requiredEndpointUrl: "http://127.0.0.1:8081/v1", + requiredInferenceApi: "openai-completions", + }); + }); + + it("blocks a conflicting llama.cpp endpoint on a shared gateway (#8161)", () => { + const result = check( + route("llama-cpp-local", "team/model-alias", { + endpointUrl: "http://127.0.0.1:8081/v1", + preferredInferenceApi: "openai-completions", + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + }), + [ + sandbox("llama-peer", { + provider: "llama-cpp-local", + model: "team/model-alias", + endpointUrl: "http://localhost:8082/v1", + preferredInferenceApi: "openai-completions", + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + }), + ], + ); + + expect(result).toMatchObject({ + ok: false, + conflicts: [{ sandboxName: "llama-peer", reason: "custom-endpoint" }], + }); + }); + it("blocks conflicting or unprovable discovery before a provider probe (#6315)", () => { expect(discover(discoveryRoute("anthropic-prod"), [sandbox("stopped-peer")])).toMatchObject({ ok: false, diff --git a/src/lib/inference/gateway-route-compatibility.ts b/src/lib/inference/gateway-route-compatibility.ts index d44ab6c712..58c689a069 100644 --- a/src/lib/inference/gateway-route-compatibility.ts +++ b/src/lib/inference/gateway-route-compatibility.ts @@ -68,7 +68,11 @@ export type CurrentGatewayRouteDiscoveryPreflight = ( }, ) => GatewayRouteDiscoveryResult; -const CUSTOM_ROUTE_PROVIDERS = new Set(["compatible-endpoint", "compatible-anthropic-endpoint"]); +const CUSTOM_ROUTE_PROVIDERS = new Set([ + "compatible-endpoint", + "compatible-anthropic-endpoint", + "llama-cpp-local", +]); const SUPPORTED_INFERENCE_APIS = new Set([ "openai-completions", diff --git a/src/lib/inference/llama-cpp/contract.ts b/src/lib/inference/llama-cpp/contract.ts new file mode 100644 index 0000000000..5a8cd2aa2b --- /dev/null +++ b/src/lib/inference/llama-cpp/contract.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Stable contract for operator-run llama.cpp existing-server attachment. */ +export const LLAMA_CPP_SELECTION_KEY = "llama-cpp"; +export const LLAMA_CPP_PROVIDER_NAME = "llama-cpp-local"; +export const LLAMA_CPP_PROVIDER_LABEL = "Local llama.cpp"; +export const LLAMA_CPP_CREDENTIAL_ENV = "NEMOCLAW_LLAMACPP_LOCAL_TOKEN"; +export const LLAMA_CPP_PORT = 8081; +export const LLAMA_CPP_HOST_BASE_URL = `http://127.0.0.1:${LLAMA_CPP_PORT}`; +export const LLAMA_CPP_HOST_OPENAI_BASE_URL = `${LLAMA_CPP_HOST_BASE_URL}/v1`; +export const LLAMA_CPP_GATEWAY_BASE_URL = `http://host.openshell.internal:${LLAMA_CPP_PORT}/v1`; + +const MAX_LLAMA_CPP_SERVED_MODEL_ALIAS_BYTES = 256; + +/** A served alias may contain namespaces, but must not expose a model filesystem path. */ +export function isSafeLlamaCppServedModelAlias(value: string): boolean { + const alias = value.trim(); + if (!alias || alias !== value) return false; + if (Buffer.byteLength(alias, "utf8") > MAX_LLAMA_CPP_SERVED_MODEL_ALIAS_BYTES) { + return false; + } + if (!/^[A-Za-z0-9._:/-]+$/.test(alias)) return false; + if (/^(?:file:|[A-Za-z]:[\\/]|[./~]|\\\\)/i.test(alias)) return false; + if (alias.split("/").some((segment) => segment === "." || segment === "..")) { + return false; + } + if (alias.includes("\\") || /\.gguf$/i.test(alias)) return false; + return true; +} diff --git a/src/lib/inference/llama-cpp/index.test.ts b/src/lib/inference/llama-cpp/index.test.ts new file mode 100644 index 0000000000..b20cfbc103 --- /dev/null +++ b/src/lib/inference/llama-cpp/index.test.ts @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { describe, expect, it, vi } from "vitest"; +import { validateCurlProbeArgs } from "../../adapters/http/curl-args"; +import type { CurlProbeOptions, CurlProbeResult } from "../../adapters/http/probe"; +import { isSafeLlamaCppServedModelAlias, probeLlamaCppAttachment } from "./index"; + +function response(httpStatus: number, body: string): CurlProbeResult { + return { + ok: httpStatus >= 200 && httpStatus < 300, + httpStatus, + curlStatus: 0, + body, + stderr: "", + message: `HTTP ${httpStatus}`, + } as CurlProbeResult; +} + +function curlFailure(curlStatus: number): CurlProbeResult { + return { + ok: false, + httpStatus: 0, + curlStatus, + body: "", + stderr: "bounded probe failure", + message: "bounded probe failure", + }; +} + +function nativeModel(id = "team/model-alias") { + return { + id, + object: "model", + owned_by: "llamacpp", + meta: { n_vocab: 128000, n_ctx: 8192, n_ctx_train: 32768, n_embd: 4096 }, + }; +} + +function nativeResponses(model = "team/model-alias"): CurlProbeResult[] { + return [ + response(401, '{"error":"unauthorized"}'), + response(200, JSON.stringify({ data: [nativeModel(model)] })), + response(200, '{"status":"ok"}'), + response( + 200, + JSON.stringify({ + model_alias: model, + model_path: "/models/model.gguf", + total_slots: 2, + default_generation_settings: { params: {} }, + }), + ), + response(200, "# TYPE llamacpp:requests_processing gauge\nllamacpp:requests_processing 0\n"), + ]; +} + +function scriptedProbe(responses: CurlProbeResult[]) { + let index = 0; + return vi.fn((argv: string[], options?: CurlProbeOptions) => { + expect(() => validateCurlProbeArgs(argv, options)).not.toThrow(); + const current = responses[index]; + index += 1; + expect(current, `unexpected probe ${index}`).toBeDefined(); + return current!; + }); +} + +describe("isSafeLlamaCppServedModelAlias", () => { + it("accepts a served model alias at the 256-byte boundary (#8161)", () => { + expect(isSafeLlamaCppServedModelAlias("a".repeat(256))).toBe(true); + }); + + it("rejects a served model alias beyond the 256-byte boundary (#8161)", () => { + expect(isSafeLlamaCppServedModelAlias("a".repeat(257))).toBe(false); + }); +}); + +describe("probeLlamaCppAttachment", () => { + it("requires an operator-supplied native API key (#8161)", () => { + expect(probeLlamaCppAttachment(" ")).toMatchObject({ + ok: false, + reason: "authentication-required", + }); + }); + + it.each([ + "http://127.0.0.1:8082", + "http://192.0.2.10:8081", + ])("rejects attachment endpoint %s outside fixed loopback port 8081 (#8161)", (baseUrl) => { + const probe = vi.fn(); + expect( + probeLlamaCppAttachment("secret-token", { baseUrl, runCurlProbeImpl: probe }), + ).toMatchObject({ ok: false, reason: "invalid-endpoint" }); + expect(probe).not.toHaveBeenCalled(); + }); + + it("accepts a bounded authenticated native llama.cpp fingerprint (#8161)", () => { + const probe = scriptedProbe(nativeResponses()); + + expect(probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: probe })).toEqual({ + ok: true, + model: "team/model-alias", + }); + expect(probe).toHaveBeenCalledTimes(5); + for (const [argv, options] of probe.mock.calls) { + expect(argv).toEqual(expect.arrayContaining(["--max-time", "5", "--max-filesize", "262144"])); + expect(options).toEqual(expect.objectContaining({ maxResponseBytes: 262144 })); + } + }); + + it("accepts llama.cpp's native metrics-disabled response (#8161)", () => { + const responses = nativeResponses(); + responses[4] = response( + 501, + '{"error":{"code":501,"message":"metrics endpoint is disabled","type":"not_supported_error"}}', + ); + + expect( + probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: scriptedProbe(responses) }), + ).toMatchObject({ ok: true }); + }); + + it("accepts non-conflicting health metadata from llama.cpp (#8161)", () => { + const responses = nativeResponses(); + responses[2] = response(200, '{"status":"ok","slots_idle":2}'); + + expect( + probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: scriptedProbe(responses) }), + ).toMatchObject({ ok: true }); + }); + + it("requires an exact operator-supplied alias when multiple models are served (#8161)", () => { + const responses = nativeResponses("second/model"); + responses[1] = response( + 200, + JSON.stringify({ data: [nativeModel("first/model"), nativeModel("second/model")] }), + ); + + const result = probeLlamaCppAttachment("secret-token", { + requestedModel: "second/model", + runCurlProbeImpl: scriptedProbe(responses), + }); + + expect(result).toEqual({ ok: true, model: "second/model" }); + }); + + it("rejects mixed llama.cpp and vLLM model metadata when the requested entry is native llama.cpp (#8161)", () => { + const responses = nativeResponses(); + responses[1] = response( + 200, + JSON.stringify({ + data: [nativeModel(), { id: "other/model", object: "model", owned_by: "vllm" }], + }), + ); + const probe = scriptedProbe(responses); + + expect( + probeLlamaCppAttachment("secret-token", { + requestedModel: "team/model-alias", + runCurlProbeImpl: probe, + }), + ).toMatchObject({ ok: false, reason: "conflicting-fingerprint" }); + expect(probe).toHaveBeenCalledTimes(2); + }); + + it("rejects an ambiguous multi-model catalog instead of guessing (#8161)", () => { + const responses = nativeResponses(); + responses[1] = response( + 200, + JSON.stringify({ data: [nativeModel("first/model"), nativeModel("second/model")] }), + ); + + expect( + probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: scriptedProbe(responses) }), + ).toMatchObject({ ok: false, reason: "ambiguous-model" }); + }); + + it("rejects an unauthenticated OpenAI-compatible server (#8161)", () => { + const result = probeLlamaCppAttachment("secret-token", { + runCurlProbeImpl: scriptedProbe([response(200, JSON.stringify({ data: [nativeModel()] }))]), + }); + + expect(result).toMatchObject({ ok: false, reason: "authentication-required" }); + }); + + it("rejects a vLLM model catalog (#8161)", () => { + const result = probeLlamaCppAttachment("secret-token", { + runCurlProbeImpl: scriptedProbe([ + response(401, '{"error":"unauthorized"}'), + response( + 200, + JSON.stringify({ data: [{ id: "model", object: "model", owned_by: "vllm" }] }), + ), + ]), + }); + + expect(result).toMatchObject({ ok: false, reason: "not-llama-cpp" }); + }); + + it.each([ + 401, 403, + ])("rejects an authenticated model catalog response with HTTP %s (#8161)", (status) => { + const result = probeLlamaCppAttachment("secret-token", { + runCurlProbeImpl: scriptedProbe([ + response(401, '{"error":"unauthorized"}'), + response(status, '{"error":"unauthorized"}'), + ]), + }); + + expect(result).toMatchObject({ ok: false, reason: "authentication-rejected" }); + }); + + it("rejects an oversized fingerprint response (#8161)", () => { + const result = probeLlamaCppAttachment("secret-token", { + runCurlProbeImpl: scriptedProbe([response(401, '{"error":"unauthorized"}'), curlFailure(63)]), + }); + + expect(result).toMatchObject({ ok: false, reason: "oversized-response" }); + }); + + it("rejects a timed-out fingerprint probe (#8161)", () => { + const result = probeLlamaCppAttachment("secret-token", { + runCurlProbeImpl: scriptedProbe([curlFailure(28)]), + }); + + expect(result).toMatchObject({ ok: false, reason: "probe-timeout" }); + }); + + it("rejects a malformed authenticated fingerprint (#8161)", () => { + const result = probeLlamaCppAttachment("secret-token", { + runCurlProbeImpl: scriptedProbe([ + response(401, '{"error":"unauthorized"}'), + response(200, "not-json"), + ]), + }); + + expect(result).toMatchObject({ ok: false, reason: "malformed-fingerprint" }); + }); + + it("rejects a spoofed catalog without corroborating native endpoints (#8161)", () => { + const responses = nativeResponses(); + responses[3] = response(404, '{"error":"not found"}'); + + expect( + probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: scriptedProbe(responses) }), + ).toMatchObject({ ok: false, reason: "conflicting-fingerprint" }); + }); + + it("rejects conflicting model identity across native endpoints (#8161)", () => { + const responses = nativeResponses(); + responses[3] = response( + 200, + JSON.stringify({ + model_alias: "different/model", + model_path: "/models/model.gguf", + total_slots: 2, + default_generation_settings: { params: {} }, + }), + ); + + expect( + probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: scriptedProbe(responses) }), + ).toMatchObject({ ok: false, reason: "conflicting-fingerprint" }); + }); + + it.each([ + "/models/model.gguf", + "C:\\models\\model.gguf", + "../model.gguf", + "model.gguf", + "models/../secret", + "foo/./bar", + ])("rejects path-like served model alias %s (#8161)", (model) => { + const responses = nativeResponses(model); + expect( + probeLlamaCppAttachment("secret-token", { runCurlProbeImpl: scriptedProbe(responses) }), + ).toMatchObject({ ok: false, reason: "unsafe-model-alias" }); + }); + + it("keeps the credential out of curl arguments and returned diagnostics (#8161)", () => { + const token = "llama-secret-credential"; + const responses = nativeResponses(); + const configModes: number[] = []; + let index = 0; + const probe = vi.fn((argv: string[], options?: CurlProbeOptions) => { + for (const configPath of options?.trustedConfigFiles ?? []) { + configModes.push(fs.statSync(configPath).mode & 0o777); + } + const current = responses[index++]; + expect(current, `unexpected probe ${index}`).toBeDefined(); + return current!; + }); + + const result = probeLlamaCppAttachment(token, { runCurlProbeImpl: probe }); + + expect(JSON.stringify(result)).not.toContain(token); + for (const [argv, options] of probe.mock.calls) { + expect(JSON.stringify(argv)).not.toContain(token); + for (const configPath of options?.trustedConfigFiles ?? []) { + expect(fs.existsSync(configPath)).toBe(false); + } + } + expect(configModes).toEqual([0o600, 0o600, 0o600, 0o600]); + }); +}); diff --git a/src/lib/inference/llama-cpp/index.ts b/src/lib/inference/llama-cpp/index.ts new file mode 100644 index 0000000000..213e037448 --- /dev/null +++ b/src/lib/inference/llama-cpp/index.ts @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createBearerAuthConfig } from "../../adapters/http/auth-config"; +import { + type CurlProbeOptions, + type CurlProbeResult, + runCurlProbe, +} from "../../adapters/http/probe"; +import { + isSafeLlamaCppServedModelAlias, + LLAMA_CPP_HOST_BASE_URL, + LLAMA_CPP_PORT, +} from "./contract"; + +export * from "./contract"; + +type LlamaCppModelEntry = { + id?: unknown; + object?: unknown; + owned_by?: unknown; + meta?: unknown; +}; + +type LlamaCppModelsResponse = { data?: unknown }; + +export type LlamaCppAttachmentFailureReason = + | "unreachable" + | "authentication-required" + | "authentication-rejected" + | "credential-preparation" + | "invalid-endpoint" + | "oversized-response" + | "probe-timeout" + | "malformed-fingerprint" + | "ambiguous-model" + | "unsafe-model-alias" + | "not-llama-cpp" + | "conflicting-fingerprint"; + +export type LlamaCppAttachmentResult = + | { ok: true; model: string } + | { + ok: false; + reason: LlamaCppAttachmentFailureReason; + message: string; + }; + +export interface ProbeLlamaCppAttachmentOptions { + requestedModel?: string | null; + baseUrl?: string; + runCurlProbeImpl?: (argv: string[], options?: CurlProbeOptions) => CurlProbeResult; +} + +const LLAMA_CPP_META_NUMERIC_KEYS = [ + "n_vocab", + "n_ctx", + "n_ctx_train", + "n_embd", + "n_params", + "size", +] as const; + +const LLAMA_CPP_MAX_PROBE_RESPONSE_BYTES = 256 * 1024; + +function failure( + reason: LlamaCppAttachmentFailureReason, + message: string, +): LlamaCppAttachmentResult { + return { ok: false, reason, message }; +} + +function parseJsonObject(body: string): Record | null { + try { + const value: unknown = JSON.parse(body); + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; + } catch { + return null; + } +} + +function hasNativeLlamaCppModelMetadata(entry: LlamaCppModelEntry): boolean { + if (entry.object !== "model" || entry.owned_by !== "llamacpp") return false; + if (!entry.meta || typeof entry.meta !== "object" || Array.isArray(entry.meta)) return false; + const meta = entry.meta as Record; + return ( + LLAMA_CPP_META_NUMERIC_KEYS.filter( + (key) => typeof meta[key] === "number" && Number.isFinite(meta[key]), + ).length >= 4 + ); +} + +function selectModelEntry( + entries: LlamaCppModelEntry[], + requestedModel: string | null, +): LlamaCppModelEntry | null { + if (requestedModel) { + return entries.find((entry) => entry.id === requestedModel) ?? null; + } + return entries.length === 1 ? entries[0] : null; +} + +function parseModelEntries(response: LlamaCppModelsResponse): LlamaCppModelEntry[] | null { + if (!Array.isArray(response.data)) return null; + const entries = response.data.filter( + (entry): entry is LlamaCppModelEntry => + entry !== null && typeof entry === "object" && !Array.isArray(entry), + ); + return entries.length === response.data.length ? entries : null; +} + +function hasHealthyNativeResponse(result: CurlProbeResult): boolean { + const body = parseJsonObject(result.body); + return result.ok && body?.status === "ok"; +} + +function hasMatchingNativeProps(result: CurlProbeResult, model: string): boolean { + if (!result.ok) return false; + const body = parseJsonObject(result.body); + if (!body || body.model_alias !== model || typeof body.model_path !== "string") return false; + if (typeof body.total_slots !== "number" || body.total_slots <= 0) return false; + const defaults = body.default_generation_settings; + return ( + defaults !== null && + typeof defaults === "object" && + !Array.isArray(defaults) && + (defaults as Record).params !== null && + typeof (defaults as Record).params === "object" + ); +} + +function hasNativeMetricsResponse(result: CurlProbeResult): boolean { + if (result.ok) return result.body.includes("llamacpp:"); + if (result.httpStatus !== 501) return false; + const body = parseJsonObject(result.body); + const error = body?.error; + return ( + error !== null && + typeof error === "object" && + !Array.isArray(error) && + (error as Record).type === "not_supported_error" + ); +} + +function probeArgs(authArgs: readonly string[], url: string): string[] { + return [ + "-sS", + "--connect-timeout", + "2", + "--max-time", + "5", + "--max-filesize", + String(LLAMA_CPP_MAX_PROBE_RESPONSE_BYTES), + ...authArgs, + url, + ]; +} + +function boundedProbeFailure(result: CurlProbeResult): LlamaCppAttachmentResult | null { + if (result.curlStatus === 63) { + return failure( + "oversized-response", + "A llama.cpp fingerprint response exceeded the 256 KiB probe limit.", + ); + } + if (result.curlStatus === 28) { + return failure("probe-timeout", "A llama.cpp fingerprint probe exceeded its time limit."); + } + return null; +} + +function resolveFixedLoopbackBaseUrl(value: string): string | null { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); + if ( + parsed.protocol !== "http:" || + !["127.0.0.1", "localhost", "::1"].includes(hostname) || + parsed.port !== String(LLAMA_CPP_PORT) || + parsed.username || + parsed.password || + (parsed.pathname !== "/" && parsed.pathname !== "") || + parsed.search || + parsed.hash + ) { + return null; + } + return parsed.origin; +} + +/** + * Positively identify an operator-managed llama.cpp server before attachment. + * These cooperatively imitable signals are selection evidence, not a cryptographic + * server identity; partial, mixed, or contradictory signals fail closed. + */ +export function probeLlamaCppAttachment( + apiKey: string, + options: ProbeLlamaCppAttachmentOptions = {}, +): LlamaCppAttachmentResult { + if (!apiKey.trim()) { + return failure( + "authentication-required", + "A native llama.cpp API key is required for existing-server attachment.", + ); + } + const baseUrl = resolveFixedLoopbackBaseUrl(options.baseUrl ?? LLAMA_CPP_HOST_BASE_URL); + if (!baseUrl) { + return failure( + "invalid-endpoint", + `llama.cpp attachment is restricted to loopback port ${LLAMA_CPP_PORT}.`, + ); + } + const probe = options.runCurlProbeImpl ?? runCurlProbe; + const anonymousModels = probe(probeArgs([], `${baseUrl}/v1/models`), { + maxResponseBytes: LLAMA_CPP_MAX_PROBE_RESPONSE_BYTES, + pinnedAddresses: [], + }); + const anonymousBoundFailure = boundedProbeFailure(anonymousModels); + if (anonymousBoundFailure) return anonymousBoundFailure; + if (anonymousModels.curlStatus !== 0 || anonymousModels.httpStatus === 0) { + return failure("unreachable", `No llama.cpp server responded on fixed port ${LLAMA_CPP_PORT}.`); + } + if (anonymousModels.httpStatus !== 401 && anonymousModels.httpStatus !== 403) { + return failure( + "authentication-required", + "The server exposes its model catalog without native API-key authentication.", + ); + } + + let auth; + try { + auth = createBearerAuthConfig(apiKey, { prefix: "nemoclaw-llama-cpp-probe" }); + } catch { + return failure( + "credential-preparation", + "The llama.cpp credential could not be prepared for a protected probe.", + ); + } + try { + const probeOptions: CurlProbeOptions = { + maxResponseBytes: LLAMA_CPP_MAX_PROBE_RESPONSE_BYTES, + trustedConfigFiles: auth.trustedConfigFiles, + pinnedAddresses: [], + }; + const authenticatedModels = probe(probeArgs(auth.args, `${baseUrl}/v1/models`), probeOptions); + const authenticatedBoundFailure = boundedProbeFailure(authenticatedModels); + if (authenticatedBoundFailure) return authenticatedBoundFailure; + if (authenticatedModels.httpStatus === 401 || authenticatedModels.httpStatus === 403) { + return failure("authentication-rejected", "The llama.cpp API key was rejected."); + } + if (!authenticatedModels.ok) { + return failure( + "not-llama-cpp", + "The authenticated model catalog did not provide bounded llama.cpp selection evidence.", + ); + } + const models = parseJsonObject(authenticatedModels.body) as LlamaCppModelsResponse | null; + if (!models || !Array.isArray(models.data)) { + return failure( + "malformed-fingerprint", + "The authenticated llama.cpp model catalog was malformed.", + ); + } + const modelEntries = parseModelEntries(models); + if (!modelEntries) { + return failure( + "malformed-fingerprint", + "The authenticated llama.cpp model catalog was malformed.", + ); + } + const nativeModelEntries = modelEntries.filter(hasNativeLlamaCppModelMetadata); + if (modelEntries.length > 0 && nativeModelEntries.length === 0) { + return failure( + "not-llama-cpp", + "The model catalog did not contain native llama.cpp metadata.", + ); + } + if (nativeModelEntries.length !== modelEntries.length) { + return failure( + "conflicting-fingerprint", + "The model catalog contained conflicting native llama.cpp evidence.", + ); + } + const requestedModel = options.requestedModel?.trim() || null; + const modelEntry = selectModelEntry(modelEntries, requestedModel); + if (!modelEntry) { + return failure( + "ambiguous-model", + requestedModel + ? "The requested served model alias was not present in the llama.cpp catalog." + : "The llama.cpp server exposes multiple or no models; specify one served alias.", + ); + } + const model = typeof modelEntry.id === "string" ? modelEntry.id : ""; + if (!isSafeLlamaCppServedModelAlias(model)) { + return failure( + "unsafe-model-alias", + "llama.cpp must be started with a non-path served model alias.", + ); + } + const health = probe(probeArgs(auth.args, `${baseUrl}/health`), probeOptions); + const props = probe( + probeArgs(auth.args, `${baseUrl}/props?model=${encodeURIComponent(model)}`), + probeOptions, + ); + const metrics = probe( + probeArgs(auth.args, `${baseUrl}/metrics?model=${encodeURIComponent(model)}`), + probeOptions, + ); + for (const result of [health, props, metrics]) { + const boundFailure = boundedProbeFailure(result); + if (boundFailure) return boundFailure; + } + if ( + !hasHealthyNativeResponse(health) || + !hasMatchingNativeProps(props, model) || + !hasNativeMetricsResponse(metrics) + ) { + return failure( + "conflicting-fingerprint", + "The server returned conflicting or incomplete native llama.cpp evidence.", + ); + } + return { ok: true, model }; + } finally { + auth.cleanup(); + } +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f8ac383ba3..2b80ac4429 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1090,12 +1090,12 @@ const handleVllmSelection = createSetupNimVllmHandler({ applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning: vllmInference.isNemoClawManagedVllmRunning, persistConfiguredDualStationVllmRuntimeReceipt: vllmInference.persistConfiguredDualStationVllmRuntimeReceipt, exitProcess: (code) => process.exit(code), }); +// biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. +const handleLlamaCppSelection = setupNimFlow.createLlamaCppSelectionHandler({ isNonInteractive, resolveCredential: resolveProviderCredential, ensureNamedCredential: (envName, label) => credentialPrompt.ensureNamedCredential(envName, label), returningToProviderSelection: credentialPrompt.returningToProviderSelection, probeLlamaCppAttachment: setupNimFlow.probeLlamaCppAttachment, validateOpenAiLikeSelection, error: (message) => console.error(message), log: (message) => console.log(message), exitProcess: (code): never => process.exit(code) }); const ollamaModelSize: typeof import("./inference/ollama/model-size") = require("./inference/ollama/model-size"); - function isOpenshellInstalled(): boolean { return resolveOpenshell() !== null; } - function installOpenshell(): OpenShellInstallResult { return openshellPinFlow.runOpenshellInstall({ scriptsDir: SCRIPTS, @@ -1111,7 +1111,6 @@ function installOpenshell(): OpenShellInstallResult { log: console.log, }); } - function areRequiredDockerDriverBinariesPresent( platform: NodeJS.Platform = process.platform, binaries: DockerDriverBinaryOverrides = {}, @@ -3557,6 +3556,7 @@ function getSetupNimDeps(): SetupNimDeps { isNonInteractive, abortNonInteractive, ), + handleLlamaCppSelection, handleRemoteProviderSelection, handleNimLocalSelection, handleRunningOllamaSelection, @@ -3771,7 +3771,7 @@ const computeSetupPresetSuggestions = ( options: SetupPresetSuggestionOptions = {}, ): string[] => computeSetupPresetSuggestionsImpl( - { policies, tiers, localInferenceProviders: LOCAL_INFERENCE_PROVIDERS }, + { policies, tiers, localInferenceProviders: [...LOCAL_INFERENCE_PROVIDERS, "llama-cpp-local"] }, tierName, options, ); @@ -3784,7 +3784,7 @@ async function setupPoliciesWithSelection( { policies, tiers, - localInferenceProviders: LOCAL_INFERENCE_PROVIDERS, + localInferenceProviders: [...LOCAL_INFERENCE_PROVIDERS, "llama-cpp-local"], step, note, isNonInteractive, diff --git a/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts b/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts index 421e608494..4ab01174fd 100644 --- a/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts +++ b/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts @@ -8,6 +8,7 @@ import YAML from "yaml"; import { BUNDLED_LOCAL_INFERENCE_GATEWAY_PORTS, + COMPATIBLE_ENDPOINT_GATEWAY_PORTS, gatewayReachableCompatibleEndpointUrl, } from "./compatible-endpoint-gateway-route"; @@ -33,7 +34,7 @@ describe("compatible endpoint gateway routing", () => { it("rewrites exact HTTP loopback hosts on bundled local-inference ports (#5744)", () => { for (const host of ["localhost", "127.0.0.1", "[::1]"]) { - for (const port of BUNDLED_LOCAL_INFERENCE_GATEWAY_PORTS) { + for (const port of COMPATIBLE_ENDPOINT_GATEWAY_PORTS) { expect( gatewayReachableCompatibleEndpointUrl( "compatible-endpoint", @@ -44,6 +45,21 @@ describe("compatible endpoint gateway routing", () => { } }); + it("leaves a generic compatible-endpoint loopback URL unchanged on port 8081 (#8161)", () => { + expect( + gatewayReachableCompatibleEndpointUrl("compatible-endpoint", "http://127.0.0.1:8081/v1"), + ).toBe("http://127.0.0.1:8081/v1"); + }); + + it("rewrites only fixed loopback port 8081 for llama.cpp attachment (#8161)", () => { + expect( + gatewayReachableCompatibleEndpointUrl("llama-cpp-local", "http://127.0.0.1:8081/v1"), + ).toBe("http://host.openshell.internal:8081/v1"); + expect( + gatewayReachableCompatibleEndpointUrl("llama-cpp-local", "http://127.0.0.1:8000/v1"), + ).toBe("http://127.0.0.1:8000/v1"); + }); + it("preserves query strings and fragments for root and non-root routes (#5744)", () => { expect( gatewayReachableCompatibleEndpointUrl( diff --git a/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.ts b/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.ts index 185654313a..d95355bc82 100644 --- a/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.ts +++ b/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.ts @@ -1,16 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { LLAMA_CPP_PORT } from "../../inference/llama-cpp/contract"; import type { RunOpenshell, UpsertProvider, UpsertProviderResult } from "./types"; // Keep this list aligned with the host.openshell.internal endpoints in // nemoclaw-blueprint/policies/presets/local-inference.yaml. These are policy // ports, not environment-overridable local provider ports. -export const BUNDLED_LOCAL_INFERENCE_GATEWAY_PORTS = [11434, 11435, 8000] as const; +export const BUNDLED_LOCAL_INFERENCE_GATEWAY_PORTS = [LLAMA_CPP_PORT, 11434, 11435, 8000] as const; -const BUNDLED_LOCAL_INFERENCE_GATEWAY_PORT_SET = new Set( - BUNDLED_LOCAL_INFERENCE_GATEWAY_PORTS, -); +export const COMPATIBLE_ENDPOINT_GATEWAY_PORTS = [11434, 11435, 8000] as const; + +const COMPATIBLE_ENDPOINT_GATEWAY_PORT_SET = new Set(COMPATIBLE_ENDPOINT_GATEWAY_PORTS); +const LOOPBACK_BRIDGE_PROVIDERS = new Set(["compatible-endpoint", "llama-cpp-local"]); // #5744: keep host-side validation on the user-entered loopback URL, but // register the sandbox route through OpenShell's host bridge. Remove this when @@ -19,7 +21,9 @@ export function gatewayReachableCompatibleEndpointUrl( provider: string, endpointUrl: string | null | undefined, ): string | null | undefined { - if (provider !== "compatible-endpoint" || !endpointUrl) return endpointUrl; + if (!LOOPBACK_BRIDGE_PROVIDERS.has(provider) || !endpointUrl) { + return endpointUrl; + } const hasExactLoopbackAuthority = /^http:\/\/(?:localhost|127\.0\.0\.1|\[::1\]):[0-9]+(?:[/?#]|$)/i.test(endpointUrl); let parsed: URL; @@ -31,6 +35,11 @@ export function gatewayReachableCompatibleEndpointUrl( const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); const port = parsed.port ? Number(parsed.port) : null; const isLoopback = hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1"; + const usesAllowedBridgePort = + port !== null && + (provider === "llama-cpp-local" + ? port === LLAMA_CPP_PORT + : COMPATIBLE_ENDPOINT_GATEWAY_PORT_SET.has(port)); if ( parsed.protocol !== "http:" || parsed.username || @@ -40,7 +49,7 @@ export function gatewayReachableCompatibleEndpointUrl( !isLoopback || port === null || !Number.isInteger(port) || - !BUNDLED_LOCAL_INFERENCE_GATEWAY_PORT_SET.has(port) + !usesAllowedBridgePort ) { return endpointUrl; } diff --git a/src/lib/onboard/inference-providers/remote-openai-surface.test.ts b/src/lib/onboard/inference-providers/remote-openai-surface.test.ts index 3ecfcda015..27a23f2ef2 100644 --- a/src/lib/onboard/inference-providers/remote-openai-surface.test.ts +++ b/src/lib/onboard/inference-providers/remote-openai-surface.test.ts @@ -76,6 +76,17 @@ function createHarness() { modelMode: "input", defaultModel: MODEL, }, + "llama-cpp": { + label: "Local llama.cpp", + providerName: "llama-cpp-local", + providerType: "openai", + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + endpointUrl: "http://127.0.0.1:8081/v1", + helpUrl: null, + modelMode: "input", + defaultModel: "", + skipVerify: true, + }, }, hydrateCredentialEnv: vi.fn(() => "test-secret"), promptValidationRecovery: vi.fn(async () => "selection" as const), @@ -315,3 +326,44 @@ describe("OpenAI-compatible no-auth provider registration", () => { expect(process.env[NO_AUTH_ENV]).toBe("committed-token"); }); }); + +describe("llama.cpp existing-server provider registration", () => { + it("registers the fixed llama.cpp gateway endpoint with NEMOCLAW_LLAMACPP_LOCAL_TOKEN (#8161)", async () => { + const harness = createHarness(); + harness.deps.hydrateCredentialEnv.mockReturnValue("llama-secret"); + + await expect( + setupRemoteProviderInference( + { + sandboxName: SANDBOX, + model: "team/model-alias", + provider: "llama-cpp-local", + endpointUrl: "http://127.0.0.1:8081/v1", + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + preferredInferenceApi: "openai-completions", + }, + harness.deps, + ), + ).resolves.toEqual({ done: false }); + + expect(harness.upsertProvider).toHaveBeenCalledWith( + "llama-cpp-local", + "openai", + "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + "http://host.openshell.internal:8081/v1", + { NEMOCLAW_LLAMACPP_LOCAL_TOKEN: "llama-secret" }, + ); + expect(harness.runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "--no-verify", + "--provider", + "llama-cpp-local", + "--model", + "team/model-alias", + ], + { ignoreError: true }, + ); + }); +}); diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 5ec725290f..b1840a0532 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -280,6 +280,7 @@ export const REMOTE_PROVIDER_NAMES = [ "compatible-anthropic-endpoint", "gemini-api", "compatible-endpoint", + "llama-cpp-local", ] as const; export type RemoteProviderName = (typeof REMOTE_PROVIDER_NAMES)[number]; diff --git a/src/lib/onboard/llama-cpp-selection/index.test.ts b/src/lib/onboard/llama-cpp-selection/index.test.ts new file mode 100644 index 0000000000..8ea144970e --- /dev/null +++ b/src/lib/onboard/llama-cpp-selection/index.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + LLAMA_CPP_CREDENTIAL_ENV, + LLAMA_CPP_HOST_OPENAI_BASE_URL, +} from "../../inference/llama-cpp"; +import type { SetupNimSelectionState } from "../setup-nim-flow"; +import { createLlamaCppSelectionHandler, type LlamaCppSelectionDeps } from "./index"; + +function state(): SetupNimSelectionState { + return { + model: null, + provider: "nvidia-prod", + endpointUrl: null, + credentialEnv: null, + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: null, + nimContainer: null, + allowToolsIncompatible: false, + }; +} + +function deps(overrides: Partial = {}): LlamaCppSelectionDeps { + return { + isNonInteractive: () => false, + resolveCredential: () => "secret-token", + ensureNamedCredential: async () => "secret-token", + returningToProviderSelection: () => false, + probeLlamaCppAttachment: () => ({ ok: true, model: "team/model-alias" }), + validateOpenAiLikeSelection: async () => ({ ok: true, api: "openai-completions" }), + error: vi.fn(), + log: vi.fn(), + exitProcess: (code) => { + throw new Error(`exit ${code}`); + }, + ...overrides, + }; +} + +describe("createLlamaCppSelectionHandler", () => { + it("binds the classified alias to a credential-bearing completions route (#8161)", async () => { + const validate = vi.fn(async () => ({ ok: true, api: "openai-completions" })); + const current = state(); + const handler = createLlamaCppSelectionHandler(deps({ validateOpenAiLikeSelection: validate })); + + await expect(handler(current, null, null)).resolves.toBe("selected"); + expect(current).toMatchObject({ + provider: "llama-cpp-local", + model: "team/model-alias", + endpointUrl: LLAMA_CPP_HOST_OPENAI_BASE_URL, + credentialEnv: LLAMA_CPP_CREDENTIAL_ENV, + preferredInferenceApi: "openai-completions", + }); + expect(validate).toHaveBeenCalledWith( + "Local llama.cpp", + LLAMA_CPP_HOST_OPENAI_BASE_URL, + "team/model-alias", + LLAMA_CPP_CREDENTIAL_ENV, + expect.any(String), + null, + expect.objectContaining({ + apiKey: "secret-token", + pinnedAddresses: [], + skipResponsesProbe: true, + }), + ); + }); + + it("uses the requested non-interactive served alias as classification input (#8161)", async () => { + const probe = vi.fn(() => ({ ok: true as const, model: "team/requested" })); + const handler = createLlamaCppSelectionHandler(deps({ probeLlamaCppAttachment: probe })); + + await handler(state(), "team/requested", null); + + expect(probe).toHaveBeenCalledWith("secret-token", { + requestedModel: "team/requested", + }); + }); + + it("exits non-interactively when NEMOCLAW_LLAMACPP_LOCAL_TOKEN is absent (#8161)", async () => { + const probe = vi.fn(); + const handler = createLlamaCppSelectionHandler( + deps({ + isNonInteractive: () => true, + resolveCredential: () => null, + probeLlamaCppAttachment: probe, + }), + ); + + await expect(handler(state(), "team/model", null)).rejects.toThrow("exit 1"); + expect(probe).not.toHaveBeenCalled(); + }); + + it("routes ambiguous fingerprint evidence back to manual provider selection (#8161)", async () => { + const error = vi.fn(); + const handler = createLlamaCppSelectionHandler( + deps({ + error, + probeLlamaCppAttachment: () => ({ + ok: false, + reason: "ambiguous-model", + message: "The llama.cpp server exposes multiple models.", + }), + }), + ); + + await expect(handler(state(), null, null)).resolves.toBe("retry-selection"); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Other OpenAI-compatible endpoint")); + }); + + it("preserves a recovered alias but never claims server lifecycle ownership (#8161)", async () => { + const current = state(); + const handler = createLlamaCppSelectionHandler(deps()); + + await handler(current, null, "team/model-alias"); + + expect(current.model).toBe("team/model-alias"); + expect(current.nimContainer).toBeNull(); + expect(current).not.toHaveProperty("vllmModelIdentity"); + }); +}); diff --git a/src/lib/onboard/llama-cpp-selection/index.ts b/src/lib/onboard/llama-cpp-selection/index.ts new file mode 100644 index 0000000000..7caed97c08 --- /dev/null +++ b/src/lib/onboard/llama-cpp-selection/index.ts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + LLAMA_CPP_CREDENTIAL_ENV, + LLAMA_CPP_HOST_OPENAI_BASE_URL, + LLAMA_CPP_PROVIDER_LABEL, + LLAMA_CPP_PROVIDER_NAME, + type LlamaCppAttachmentResult, +} from "../../inference/llama-cpp"; +import type { SetupNimSelectionResult, SetupNimSelectionState } from "../setup-nim-flow"; + +type CredentialNavigation = string | Readonly<{ kind: string }>; + +export interface LlamaCppSelectionDeps { + isNonInteractive(): boolean; + resolveCredential(envName: string): string | null; + ensureNamedCredential(envName: string, label: string): Promise; + returningToProviderSelection(result: unknown): boolean; + probeLlamaCppAttachment( + apiKey: string, + options: { requestedModel?: string | null }, + ): LlamaCppAttachmentResult; + validateOpenAiLikeSelection( + label: string, + endpointUrl: string, + model: string, + credentialEnv: string | null, + retryMessage?: string, + helpUrl?: string | null, + options?: { + apiKey?: string | null; + pinnedAddresses?: readonly string[]; + skipResponsesProbe?: boolean; + }, + ): Promise<{ ok: boolean; retry?: string; api?: string | null }>; + error(message: string): void; + log(message: string): void; + exitProcess(code: number): never; +} + +/** Attach only to a positively classified, operator-run llama.cpp server. */ +export function createLlamaCppSelectionHandler( + deps: LlamaCppSelectionDeps, +): ( + state: SetupNimSelectionState, + requestedModel: string | null, + recoveredModel: string | null, +) => Promise { + return async function handleLlamaCppSelection( + state, + requestedModel, + recoveredModel, + ): Promise { + let apiKey = deps.resolveCredential(LLAMA_CPP_CREDENTIAL_ENV); + if (!apiKey && deps.isNonInteractive()) { + deps.error(` ${LLAMA_CPP_CREDENTIAL_ENV} is required for Local llama.cpp.`); + return deps.exitProcess(1); + } + if (!apiKey) { + const credential = await deps.ensureNamedCredential( + LLAMA_CPP_CREDENTIAL_ENV, + "Local llama.cpp native API key", + ); + if (deps.returningToProviderSelection(credential)) return "retry-selection"; + apiKey = typeof credential === "string" ? credential : null; + } + if (!apiKey) { + deps.error(" A native llama.cpp API key is required for existing-server attachment."); + return deps.isNonInteractive() ? deps.exitProcess(1) : "retry-selection"; + } + + state.provider = LLAMA_CPP_PROVIDER_NAME; + state.endpointUrl = LLAMA_CPP_HOST_OPENAI_BASE_URL; + state.credentialEnv = LLAMA_CPP_CREDENTIAL_ENV; + state.preferredInferenceApi = "openai-completions"; + state.model = requestedModel || recoveredModel; + state.assertRouteCompatible?.(); + + const constrainedModel = typeof state.model === "string" ? state.model : null; + const attachment = deps.probeLlamaCppAttachment(apiKey, { + requestedModel: constrainedModel, + }); + if (!attachment.ok) { + deps.error(` ${attachment.message}`); + deps.error(" NemoClaw did not attach this server as llama.cpp."); + deps.error(" Select Other OpenAI-compatible endpoint to configure it."); + return deps.isNonInteractive() ? deps.exitProcess(1) : "retry-selection"; + } + + state.model = attachment.model; + state.assertRouteCompatible?.(); + const validation = await deps.validateOpenAiLikeSelection( + LLAMA_CPP_PROVIDER_LABEL, + LLAMA_CPP_HOST_OPENAI_BASE_URL, + attachment.model, + LLAMA_CPP_CREDENTIAL_ENV, + "Choose a provider and model again.", + null, + { + apiKey, + pinnedAddresses: [], + skipResponsesProbe: true, + }, + ); + if (!validation.ok || validation.retry === "selection" || validation.retry === "model") { + return "retry-selection"; + } + state.preferredInferenceApi = "openai-completions"; + deps.log(` Attached Local llama.cpp with served model alias: ${attachment.model}`); + return "selected"; + }; +} diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts index faf39ac21a..3f8932ed7f 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts @@ -13,6 +13,7 @@ import { type ProviderInferenceStateOptions, type ProviderSelectionResult, } from "./provider-inference"; +import { guardProviderInferenceRouteSelection } from "./provider-inference-route-containment"; type Options = ProviderInferenceStateOptions; @@ -193,6 +194,32 @@ function reportDifferentRoute( } describe("provider route containment", () => { + it("defers exact endpoint comparison until llama.cpp route discovery is complete (#8161)", () => { + const { calls } = createDeps(); + const containmentDeps = { + checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, + preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, + error: calls.error, + exitProcess: calls.exit, + }; + + expect( + guardProviderInferenceRouteSelection(containmentDeps, "nemoclaw-9090", "target-sandbox", { + provider: "llama-cpp-local", + model: "team/model-alias", + endpointUrl: null, + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + preferredInferenceApi: null, + }), + ).toEqual({ + requiredModel: null, + requiredEndpointUrl: null, + requiredInferenceApi: null, + }); + expect(calls.preflightGatewayRouteDiscovery).toHaveBeenCalledOnce(); + expect(calls.checkGatewayRouteCompatibility).not.toHaveBeenCalled(); + }); + it("allows fresh selection to continue so setup can issue the mutation warning (#6315)", async () => { const { calls, deps } = createDeps(); reportDifferentRoute(calls, "nvidia-prod", "nvidia/test"); diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts index 2e88105685..9a765d7cdb 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.ts @@ -65,7 +65,9 @@ export function guardProviderInferenceRouteSelection( } const provider = typeof route.provider === "string" ? route.provider.trim() : ""; const completeCustomRoute = - !["compatible-endpoint", "compatible-anthropic-endpoint"].includes(provider) || + !["compatible-endpoint", "compatible-anthropic-endpoint", "llama-cpp-local"].includes( + provider, + ) || (typeof route.endpointUrl === "string" && route.endpointUrl.trim().length > 0 && typeof route.preferredInferenceApi === "string" && diff --git a/src/lib/onboard/policy-presets.ts b/src/lib/onboard/policy-presets.ts index fa0f24388a..1ad0514442 100644 --- a/src/lib/onboard/policy-presets.ts +++ b/src/lib/onboard/policy-presets.ts @@ -9,8 +9,8 @@ import { } from "../messaging/channels"; import { requiredObservabilityPolicyPresets } from "./observability-policy-presets"; -const { LOCAL_INFERENCE_PROVIDERS } = require("./providers") as { - LOCAL_INFERENCE_PROVIDERS: string[]; +const { LOCAL_INFERENCE_POLICY_PROVIDERS } = require("./providers") as { + LOCAL_INFERENCE_POLICY_PROVIDERS: string[]; }; import { isOpenclawAgent, requiredOpenclawOtelPolicyPresets } from "./openclaw-otel-policy-presets"; @@ -36,7 +36,7 @@ export function getSuggestedPolicyPresets({ }: SuggestedPolicyPresetOptions = {}): string[] { const suggestions = ["pypi", "npm"]; - if (provider && LOCAL_INFERENCE_PROVIDERS.includes(provider)) { + if (provider && LOCAL_INFERENCE_POLICY_PROVIDERS.includes(provider)) { suggestions.push("local-inference"); } if (isOpenclawAgent(agent)) { diff --git a/src/lib/onboard/provider-menu.test.ts b/src/lib/onboard/provider-menu.test.ts index e466296078..b92425adbb 100644 --- a/src/lib/onboard/provider-menu.test.ts +++ b/src/lib/onboard/provider-menu.test.ts @@ -54,6 +54,7 @@ describe("buildInferenceProviderMenu", () => { "anthropic", "anthropicCompatible", "gemini", + "llama-cpp", ]); }); @@ -86,6 +87,7 @@ describe("buildInferenceProviderMenu", () => { "install-ollama", "routed", "hermesProvider", + "llama-cpp", ]); expect(result.options.find((option) => option.key === "build")?.label).toBe("NVIDIA Endpoints"); expect(result.options.find((option) => option.key === "hermesProvider")?.label).toBe( @@ -100,7 +102,7 @@ describe("buildInferenceProviderMenu", () => { windowsHostInstallLabel: "Install Ollama on Windows host (requires Docker Desktop)", }); - expect(result.options.at(-1)).toEqual({ + expect(result.options.at(-2)).toEqual({ key: "install-windows-ollama", label: "Install Ollama on Windows host (requires Docker Desktop)", }); @@ -116,7 +118,7 @@ describe("buildInferenceProviderMenu", () => { reachable ? "Use Ollama on Windows host - running" : "Start Ollama on Windows host", }); - expect(result.options.at(-1)).toEqual({ + expect(result.options.at(-2)).toEqual({ key: "start-windows-ollama", label: "Use Ollama on Windows host - running", }); diff --git a/src/lib/onboard/provider-menu.ts b/src/lib/onboard/provider-menu.ts index b71de25a2f..f828f566e7 100644 --- a/src/lib/onboard/provider-menu.ts +++ b/src/lib/onboard/provider-menu.ts @@ -121,6 +121,11 @@ export function buildInferenceProviderMenu( pushUniqueRemoteProviderOption(options, input.remoteProviderConfig, providerKey); } + // Existing-server attachment stays visible without probing or claiming lifecycle ownership. + if (!options.some((option) => option.key === "llama-cpp")) { + options.push({ key: "llama-cpp", label: "Local llama.cpp" }); + } + return { options, hermesProviderAvailable: input.agentProviderOptions.includes("hermesProvider"), diff --git a/src/lib/onboard/providers.ts b/src/lib/onboard/providers.ts index f729690e18..30ee1a4e63 100644 --- a/src/lib/onboard/providers.ts +++ b/src/lib/onboard/providers.ts @@ -16,6 +16,11 @@ const { const openrouter = require("../inference/openrouter"); const { isSafeModelId } = require("../validation"); const { compactText } = require("../core/url-utils"); +const { + LLAMA_CPP_CREDENTIAL_ENV, + LLAMA_CPP_HOST_OPENAI_BASE_URL, + LLAMA_CPP_PROVIDER_NAME, +} = require("../inference/llama-cpp/contract"); const { readGatewayProviderMetadata } = require("./gateway-provider-metadata"); // ── Constants ──────────────────────────────────────────────────── @@ -59,6 +64,7 @@ const NON_INTERACTIVE_PROVIDER_KEYS = new Set([ "gemini", "hermesProvider", "ollama", + "llama-cpp", "custom", "nim-local", "vllm", @@ -69,7 +75,7 @@ const NON_INTERACTIVE_PROVIDER_KEYS = new Set([ "start-windows-ollama", ]); const NON_INTERACTIVE_PROVIDER_VALID_VALUES = - "Valid values: build, openrouter, openai, anthropic, anthropicCompatible, gemini, hermes-provider, ollama, custom, nim-local, vllm, routed, install-vllm, install-ollama, install-windows-ollama, start-windows-ollama"; + "Valid values: build, openrouter, openai, anthropic, anthropicCompatible, gemini, hermes-provider, ollama, llama-cpp, custom, nim-local, vllm, routed, install-vllm, install-ollama, install-windows-ollama, start-windows-ollama"; const PROVIDER_KEY_ROUTE_VALUES = new Set( [ "inference", @@ -172,10 +178,25 @@ const REMOTE_PROVIDER_CONFIG = { defaultModel: "", skipVerify: true, }, + "llama-cpp": { + label: "Local llama.cpp", + providerName: LLAMA_CPP_PROVIDER_NAME, + providerType: "openai", + credentialEnv: LLAMA_CPP_CREDENTIAL_ENV, + endpointUrl: LLAMA_CPP_HOST_OPENAI_BASE_URL, + helpUrl: null, + modelMode: "input", + defaultModel: "", + skipVerify: true, + }, }; // Providers that run on the host and need the local-inference policy preset. const LOCAL_INFERENCE_PROVIDERS = ["ollama-local", "vllm-local"]; +// Host-endpoint providers that need the declarative local-inference network policy. +// Keep this separate from LOCAL_INFERENCE_PROVIDERS: llama.cpp is operator-owned, +// credential-bearing, endpoint-bearing, and must never enter managed lifecycle paths. +const LOCAL_INFERENCE_POLICY_PROVIDERS = [...LOCAL_INFERENCE_PROVIDERS, "llama-cpp-local"]; // Re-exported alias matching the existing onboard.ts call sites. The canonical // definitions live in inference-config.ts so that getProviderSelectionConfig @@ -204,6 +225,8 @@ function getProviderLabel(provider) { return "Local vLLM"; case "ollama-local": return "Local Ollama"; + case "llama-cpp-local": + return "Local llama.cpp"; default: return provider; } @@ -223,6 +246,8 @@ function getEffectiveProviderName(providerKey) { return "ollama-local"; case "vllm": return "vllm-local"; + case "llama-cpp": + return "llama-cpp-local"; case "routed": return "nvidia-router"; default: @@ -567,6 +592,7 @@ module.exports = { GEMINI_ENDPOINT_URL, REMOTE_PROVIDER_CONFIG, LOCAL_INFERENCE_PROVIDERS, + LOCAL_INFERENCE_POLICY_PROVIDERS, OLLAMA_PROXY_CREDENTIAL_ENV, VLLM_LOCAL_CREDENTIAL_ENV, DISCORD_SNOWFLAKE_RE, diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index b661a5fba5..c9f5fe88c1 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -122,6 +122,7 @@ function makeDeps(overrides: Partial = {}): SetupNimFlowDeps { error: vi.fn(), exitProcess: (code) => unexpected(`exitProcess(${code})`), abortNonInteractive: (message) => unexpected(`abortNonInteractive(${message})`), + handleLlamaCppSelection: async () => unexpected("llama.cpp selection"), handleRemoteProviderSelection: async () => unexpected("remote provider selection"), handleNimLocalSelection: async () => unexpected("local NIM selection"), handleRunningOllamaSelection: async () => unexpected("running Ollama selection"), @@ -1131,6 +1132,37 @@ describe("createSetupNim", () => { expect(handleVllmSelection).not.toHaveBeenCalled(); }); + it("dispatches llama.cpp existing-server selection without a managed install path (#8161)", async () => { + const handleLlamaCppSelection = vi.fn( + async (selection, requestedModel) => { + expect(requestedModel).toBe("team/model-alias"); + selection.provider = "llama-cpp-local"; + selection.model = "team/model-alias"; + selection.endpointUrl = "http://127.0.0.1:8081/v1"; + selection.credentialEnv = "NEMOCLAW_LLAMACPP_LOCAL_TOKEN"; + selection.preferredInferenceApi = "openai-completions"; + return "selected"; + }, + ); + const setupNim = createSetupNim( + makeDeps({ + isNonInteractive: () => true, + getNonInteractiveProvider: () => "llama-cpp", + getNonInteractiveModel: () => "team/model-alias", + handleLlamaCppSelection, + }), + ); + + await expect(setupNim(null)).resolves.toMatchObject({ + provider: "llama-cpp-local", + model: "team/model-alias", + endpointUrl: "http://127.0.0.1:8081/v1", + credentialEnv: "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", + preferredInferenceApi: "openai-completions", + }); + expect(handleLlamaCppSelection).toHaveBeenCalledOnce(); + }); + it("returns interactive occupied-port selection to the provider menu", async () => { vi.stubEnv("NEMOCLAW_PROVIDER", ""); const profile = { name: "DGX Spark" } as VllmProfile; diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 541293fda5..740c8cffed 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -28,6 +28,9 @@ import type { RebuildRouteHandoff, RegistryInferenceRoute } from "./rebuild-rout import { prepareProviderDiscovery } from "./setup-nim-provider-discovery"; import type { SetupNimSelectionState as BaseSetupNimSelectionState } from "./setup-nim-selection"; +export { probeLlamaCppAttachment } from "../inference/llama-cpp"; +export { createLlamaCppSelectionHandler } from "./llama-cpp-selection"; + export type SetupNimGpu = ReturnType; export type SetupNimSelectionState = BaseSetupNimSelectionState; export type SetupNimSelectionResult = "selected" | "retry-selection"; @@ -115,6 +118,11 @@ export interface SetupNimFlowDeps { state: SetupNimSelectionState, recoveredRegistryRoute: RegistryInferenceRoute | null, ): Promise; + handleLlamaCppSelection( + state: SetupNimSelectionState, + requestedModel: string | null, + recoveredModel: string | null, + ): Promise; handleNimLocalSelection( gpu: SetupNimGpu, args: Pick< @@ -242,6 +250,65 @@ function applyGatewayRouteDiscoveryConstraints( } } +function isEndpointProviderSelection(deps: SetupNimFlowDeps, providerKey: string): boolean { + return providerKey === "llama-cpp" || Boolean(deps.remoteProviderConfig[providerKey]); +} + +async function handleEndpointProviderSelection(input: { + deps: SetupNimFlowDeps; + selected: ProviderMenuChoice; + state: SetupNimSelectionState; + requestedModel: string | null; + recoveredFromSandbox: boolean; + recoveredModel: string | null; + sandboxName: string | null; + gatewayName: string | null; + recoverySessionId: string | null | undefined; + agent: AgentDefinition | null; + recoveredRegistryRoute: RegistryInferenceRoute | null; +}): Promise { + const { + deps, + selected, + state, + requestedModel, + recoveredFromSandbox, + recoveredModel, + sandboxName, + gatewayName, + recoverySessionId, + agent, + recoveredRegistryRoute, + } = input; + if (selected.key === "llama-cpp") { + return deps.handleLlamaCppSelection( + state, + requestedModel, + recoveredFromSandbox ? recoveredModel : null, + ); + } + const remoteConfig = deps.remoteProviderConfig[selected.key]; + if (!remoteConfig) throw new Error(`Missing remote provider config for '${selected.key}'.`); + return deps.handleRemoteProviderSelection( + { + selected, + requestedModel, + recoveredFromSandbox, + recoveredModel, + sandboxName, + gatewayName, + recoverySessionId, + intendedInferenceApi: resolveValidationInferenceApi( + selected.key, + remoteConfig.providerName, + agent, + ), + }, + state, + recoveredRegistryRoute, + ); +} + /** Create the provider-selection flow and seed agent-specific Ollama defaults. */ export function createSetupNim( defaults: SetupNimFlowDeps, @@ -462,26 +529,21 @@ export function createSetupNim( hermesToolGateways = []; } - if (deps.remoteProviderConfig[selected.key]) { + if (isEndpointProviderSelection(deps, selected.key)) { const state = createSelectionState(); - const result = await deps.handleRemoteProviderSelection( - { - selected, - requestedModel, - recoveredFromSandbox, - recoveredModel, - sandboxName, - gatewayName, - recoverySessionId, - intendedInferenceApi: resolveValidationInferenceApi( - selected.key, - deps.remoteProviderConfig[selected.key].providerName, - agent, - ), - }, + const result = await handleEndpointProviderSelection({ + deps, + selected, state, + requestedModel, + recoveredFromSandbox, + recoveredModel, + sandboxName, + gatewayName, + recoverySessionId, + agent, recoveredRegistryRoute, - ); + }); ({ model, provider, diff --git a/src/lib/security/credential-env.ts b/src/lib/security/credential-env.ts index 3d9f7be754..c4aa7088dc 100644 --- a/src/lib/security/credential-env.ts +++ b/src/lib/security/credential-env.ts @@ -49,6 +49,7 @@ export const SUPPORTED_CREDENTIAL_ENV_NAMES: ReadonlySet = new Set([ "GITHUB_TOKEN", "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN", "NEMOCLAW_OLLAMA_PROXY_TOKEN", + "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", "NEMOCLAW_VLLM_LOCAL_TOKEN", ]); diff --git a/src/lib/validation.test.ts b/src/lib/validation.test.ts index 57f6deace6..d917d71154 100644 --- a/src/lib/validation.test.ts +++ b/src/lib/validation.test.ts @@ -436,6 +436,10 @@ describe("shouldSkipResponsesProbe", () => { expect(shouldSkipResponsesProbe("openrouter-api")).toBe(true); }); + it("skips the Responses probe for completions-only llama.cpp attachment (#8161)", () => { + expect(shouldSkipResponsesProbe("llama-cpp-local")).toBe(true); + }); + it("does not skip the Responses probe for other providers", () => { expect(shouldSkipResponsesProbe("openai-api")).toBe(false); expect(shouldSkipResponsesProbe("anthropic-prod")).toBe(false); diff --git a/src/lib/validation.ts b/src/lib/validation.ts index a5c468737f..c7de06fe2d 100644 --- a/src/lib/validation.ts +++ b/src/lib/validation.ts @@ -321,7 +321,8 @@ export function shouldSkipResponsesProbe(provider: string): boolean { provider === "nvidia-prod" || provider === "nvidia-nim" || provider === "gemini-api" || - provider === "openrouter-api" + provider === "openrouter-api" || + provider === "llama-cpp-local" ); } diff --git a/test/e2e/live/snapshot-credential-scanner.ts b/test/e2e/live/snapshot-credential-scanner.ts index dc6cd8695c..d6df8f7e4e 100644 --- a/test/e2e/live/snapshot-credential-scanner.ts +++ b/test/e2e/live/snapshot-credential-scanner.ts @@ -39,6 +39,7 @@ export const MODELS_JSON_CREDENTIAL_ENV_REFERENCES: ReadonlySet = new Se "GOOGLE_API_KEY", "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN", "NEMOCLAW_OLLAMA_PROXY_TOKEN", + "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", "NEMOCLAW_VLLM_LOCAL_TOKEN", "NGC_API_KEY", "NVIDIA_API_KEY", diff --git a/test/e2e/support/snapshot-credential-scanner.test.ts b/test/e2e/support/snapshot-credential-scanner.test.ts index eb344267bc..c491cb452e 100644 --- a/test/e2e/support/snapshot-credential-scanner.test.ts +++ b/test/e2e/support/snapshot-credential-scanner.test.ts @@ -24,6 +24,7 @@ describe("snapshot credential scanner", () => { "GOOGLE_API_KEY", "AWS_BEARER_TOKEN_BEDROCK", "COMPATIBLE_ANTHROPIC_API_KEY", + "NEMOCLAW_LLAMACPP_LOCAL_TOKEN", ]) { expect(SUPPORTED_CREDENTIAL_ENV_NAMES.has(name), name).toBe(true); } diff --git a/test/install-gateway-state-root.test.ts b/test/install-gateway-state-root.test.ts index a40ffe51df..422ca4d09e 100644 --- a/test/install-gateway-state-root.test.ts +++ b/test/install-gateway-state-root.test.ts @@ -192,6 +192,7 @@ nemoclaw_state_dir`, it.each([ "08000", + "08081", "11434", "18790", ])("rejects conflicting gateway port %s before writing selected state", (gatewayPort) => { @@ -211,6 +212,32 @@ save_usage_notice_acceptance_shell "test-version"`, } }); + it.each([ + "NEMOCLAW_DASHBOARD_PORT", + "NEMOCLAW_VLLM_PORT", + "NEMOCLAW_OLLAMA_PORT", + "NEMOCLAW_OLLAMA_PROXY_PORT", + "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT", + "NEMOCLAW_OPENROUTER_RUNTIME_ADAPTER_PORT", + "NEMOCLAW_HTTPS_PIN_RUNTIME_ADAPTER_PORT", + ])("rejects %s on fixed llama.cpp port 8081 before writing selected state", (envName) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-installer-llamacpp-port-")); + try { + const result = runInstallerFunctions( + home, + `${envName}=08081 +save_usage_notice_acceptance_shell "test-version"`, + ); + + expect(result.status, result.output).not.toBe(0); + expect(result.output).toContain(`${envName} must not overlap`); + expect(result.output).toContain("fixed llama.cpp inference port (8081)"); + expect(fs.existsSync(path.join(home, ".nemoclaw", "gateways", "9123"))).toBe(false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it.each( stateSymlinkCases, )("rejects a symlinked $label state ancestor before writing usage acceptance", ({ setup }) => { diff --git a/test/onboard-policy-suggestions.test.ts b/test/onboard-policy-suggestions.test.ts index b228dac898..0631444293 100644 --- a/test/onboard-policy-suggestions.test.ts +++ b/test/onboard-policy-suggestions.test.ts @@ -229,6 +229,7 @@ describe("onboard policy preset suggestions", () => { expect(ollamaPresets).toContain("npm"); expect(getSuggestedPolicyPresets({ provider: "vllm-local" })).toContain("local-inference"); + expect(getSuggestedPolicyPresets({ provider: "llama-cpp-local" })).toContain("local-inference"); expect(getSuggestedPolicyPresets({ provider: "nvidia-prod" })).not.toContain("local-inference"); expect(getSuggestedPolicyPresets({ provider: "openai-api" })).not.toContain("local-inference"); expect(getSuggestedPolicyPresets({ provider: null })).not.toContain("local-inference"); diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index 8aae189523..372ea4427c 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -71,7 +71,6 @@ const CREDENTIAL_RETRY_PROMPT_RE = const OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE = '{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"type":"function","function":{"name":"emit_ok","arguments":"{\\"ok\\":true}"}}]}}]}'; const PROVIDER_SELECTION_TEST_TIMEOUT_MS = testTimeout(60_000); - const TEST_REMOTE_PROVIDER_CONFIG = { build: { label: "NVIDIA Endpoints", providerName: "nvidia-prod" }, openai: { label: "OpenAI", providerName: "openai-api" }, @@ -183,6 +182,7 @@ function makeSetupNimFlowDeps(overrides: Partial = {}): SetupN error: () => {}, exitProcess: (code) => unexpected(`exitProcess(${code})`), abortNonInteractive: (message) => unexpected(`abortNonInteractive(${message})`), + handleLlamaCppSelection: async () => unexpected("llama.cpp selection"), handleRemoteProviderSelection: async () => unexpected("remote provider selection"), handleNimLocalSelection: async () => unexpected("local NIM selection"), handleRunningOllamaSelection: async () => unexpected("running Ollama selection"), diff --git a/test/runtime-shell.test.ts b/test/runtime-shell.test.ts index b614ced031..7db2f95c61 100644 --- a/test/runtime-shell.test.ts +++ b/test/runtime-shell.test.ts @@ -200,6 +200,23 @@ describe("shell runtime helpers", () => { expect(result.stderr).toContain(`Invalid ${name}=${value} (expected 1024-65535)`); }); + it.each([ + { name: "NEMOCLAW_VLLM_PORT", value: "8081", provider: "vllm-local" }, + { name: "NEMOCLAW_VLLM_PORT", value: "08081", provider: "vllm-local" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "8081", provider: "ollama-local" }, + { name: "NEMOCLAW_OLLAMA_PORT", value: "08081", provider: "ollama-local" }, + ])("rejects reserved llama.cpp port $value for $name", ({ name, value, provider }) => { + const result = runShell(`source "${RUNTIME_SH}"; get_local_provider_base_url ${provider}`, { + [name]: value, + }); + + expect(result.status).not.toBe(0); + expect(result.stdout.trim()).toBe(""); + expect(result.stderr).toContain( + `Invalid ${name}=${value} (conflicts with fixed llama.cpp inference port 8081)`, + ); + }); + it("returns the first non-loopback nameserver", () => { const result = runShell( `source "${RUNTIME_SH}"; first_non_loopback_nameserver $'nameserver 127.0.0.11\\nnameserver 10.0.0.2'`,