From 570629c1fdc59a457bb9d45657ffb626e9b65024 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Mon, 1 Jun 2026 05:12:30 +0000 Subject: [PATCH 1/2] fix(onboard): reuse containerized gateway and repair routed provider reachability (#4520, #4564) Two Docker-driver onboarding fixes sharing the runtime-identity / host-service reachability axis. #4520: The glibc-compat gateway runs as a host-side `docker run ... openshell-gateway` parent, so /proc//exe is /usr/bin/docker. buildDockerDriverGatewayRuntimeIdentity() encodes this with driftGatewayBin=null to skip the executable check, but both callers used `runtimeIdentity?.driftGatewayBin ?? gatewayBin`, coalescing the deliberate null back to the host binary and falsely marking a healthy compat gateway stale, triggering a recreate and port 8080 collision on the second onboard. Add resolveDriftGatewayBin() which preserves the null, and use it in refreshDockerDriverGatewayReuseState and startDockerDriverGateway. #4564: On Linux Docker-driver hosts, Model Router on localhost:4000 is the sandbox loopback, not the host router. Resume only reconciled the router and left a stale localhost provider base URL, and there was no sandbox-network reachability probe like the Ollama auth proxy has. Extract a generic host-service reachability probe (reused by the Ollama wrapper), probe host.openshell.internal: from the OpenShell Docker network inside reconcileModelRouter() with a concrete ufw remediation on tcp_failed, and re-upsert the routed provider with the normalized host alias (and the routed profile's credential env) on resume so stale localhost entries are repaired. Non-Docker-driver behavior is unchanged. New logic lives under src/lib/onboard/** to keep onboard.ts net-neutral. Signed-off-by: Yimo Jiang --- src/lib/onboard.ts | 39 +-- .../docker-driver-gateway-launch.test.ts | 56 ++++ .../onboard/docker-driver-gateway-launch.ts | 21 +- .../onboard/host-service-reachability.test.ts | 136 +++++++++ src/lib/onboard/host-service-reachability.ts | 270 ++++++++++++++++++ .../handlers/provider-inference.test.ts | 65 +++++ .../machine/handlers/provider-inference.ts | 14 + src/lib/onboard/model-router.ts | 35 +++ src/lib/onboard/ollama-proxy-reachability.ts | 265 ++--------------- src/lib/onboard/routed-inference.test.ts | 131 +++++++++ src/lib/onboard/routed-inference.ts | 124 ++++++++ test/onboard-gateway-runtime.test.ts | 28 ++ 12 files changed, 921 insertions(+), 263 deletions(-) create mode 100644 src/lib/onboard/host-service-reachability.test.ts create mode 100644 src/lib/onboard/host-service-reachability.ts create mode 100644 src/lib/onboard/routed-inference.test.ts create mode 100644 src/lib/onboard/routed-inference.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 504004f75fd..3fa1536b835 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -421,6 +421,7 @@ const { loadBlueprintProfile, reconcileModelRouter, } = modelRouter; +const routedInference: typeof import("./onboard/routed-inference") = require("./onboard/routed-inference"); const { OnboardRuntimeBoundary }: typeof import("./onboard/runtime-boundary") = require("./onboard/runtime-boundary"); const { handleAgentSetupState }: typeof import("./onboard/machine/handlers/agent-setup") = require("./onboard/machine/handlers/agent-setup"); const { handleFinalizationState }: typeof import("./onboard/machine/handlers/finalization") = require("./onboard/machine/handlers/finalization"); @@ -1204,7 +1205,7 @@ async function refreshDockerDriverGatewayReuseState( ); const runtimeIdentity = gatewayBin ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ gatewayBin, gatewayEnv: baseDesiredEnv, stateDir: getDockerDriverGatewayStateDir(), sandboxBin: resolveOpenShellSandboxBinary() }) : null; const desiredEnv = runtimeIdentity?.desiredEnv ?? baseDesiredEnv; - const driftBin = runtimeIdentity?.driftGatewayBin ?? gatewayBin; + const driftBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin(runtimeIdentity, gatewayBin); const identityBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin; const pid = getDockerDriverGatewayPid(); if (pid !== null && isDockerDriverGatewayProcessAlive()) { @@ -2419,7 +2420,7 @@ async function startDockerDriverGateway({ exitOnFailure = true, skipSandboxBridg const stateDir = getDockerDriverGatewayStateDir(); const runtimeIdentity = gatewayBin ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ gatewayBin, gatewayEnv, stateDir, sandboxBin: resolveOpenShellSandboxBinary() }) : null; const gatewayLaunch = runtimeIdentity?.launch ?? null; - const driftGatewayBin = runtimeIdentity?.driftGatewayBin ?? gatewayBin; + const driftGatewayBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin(runtimeIdentity, gatewayBin); const driftGatewayEnv = runtimeIdentity?.desiredEnv ?? gatewayEnv; const identityGatewayBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin; const { verifySandboxBridgeGatewayReachableOrExit } = @@ -5517,37 +5518,19 @@ async function setupInference( // to unrelated OpenAI-backed sandboxes. } else if (isRoutedInferenceProvider(provider)) { // Blueprint profile provider (e.g., nvidia-router for the routed profile). - // Same pattern as vllm-local: upsert the provider and set the inference route. + // reconcileModelRouter also probes sandbox→router reachability (#4564). try { await reconcileModelRouter(); } catch (err) { console.error(` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); } - const resolvedCredentialEnv = credentialEnv || DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV; - const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv); - const env = credentialValue ? { [resolvedCredentialEnv]: credentialValue } : {}; - const providerResult = upsertProvider( - provider, - "openai", - resolvedCredentialEnv, - endpointUrl, - env, - ); - if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - process.exit(providerResult.status || 1); + const routed = routedInference.upsertRoutedProvider(provider, endpointUrl, credentialEnv, { upsertProvider, hydrateCredentialEnv }); + if (!routed.ok) { + console.error(` ${routed.result.message}`); + process.exit(routed.result.status || 1); } - const inferenceArgs = [ - "inference", - "set", - "--no-verify", - "--provider", - provider, - "--model", - model, - ]; - runOpenshell(inferenceArgs); + runOpenshell(["inference", "set", "--no-verify", "--provider", provider, "--model", model]); } else { console.error(` Unsupported provider configuration: ${provider}`); process.exit(1); @@ -6877,6 +6860,10 @@ async function onboard(opts: OnboardOptions = {}): Promise { isInferenceRouteReady, isRoutedInferenceProvider, reconcileModelRouter, + reupsertRoutedProvider: (p, url, ce) => { + const r = routedInference.upsertRoutedProvider(p, url, ce, { upsertProvider, hydrateCredentialEnv }); + return { ok: r.ok, endpointUrl: r.endpointUrl, message: r.result.message, status: r.result.status }; + }, registryUpdateSandbox: (name, updates) => registry.updateSandbox(name, updates), promptValidatedSandboxName, assessHost, diff --git a/src/lib/onboard/docker-driver-gateway-launch.test.ts b/src/lib/onboard/docker-driver-gateway-launch.test.ts index a14919efb3c..2eb7bfa63db 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.test.ts @@ -10,7 +10,9 @@ import { describe, expect, it } from "vitest"; import { buildDockerDriverGatewayConfigToml, buildDockerDriverGatewayLaunch, + buildDockerDriverGatewayRuntimeIdentity, parseGlibcVersionsFromBinaryText, + resolveDriftGatewayBin, shouldUseContainerizedGateway, } from "../../../dist/lib/onboard/docker-driver-gateway-launch"; @@ -170,6 +172,60 @@ describe("docker-driver-gateway-launch", () => { }); }); + it("keeps the drift gateway binary null for the containerized compatibility gateway (#4520)", () => { + withTempBinaries(({ dir, gatewayBin, sandboxBin }) => { + const stateDir = path.join(dir, "state"); + fs.mkdirSync(stateDir); + const identity = buildDockerDriverGatewayRuntimeIdentity({ + gatewayBin, + sandboxBin, + stateDir, + platform: "linux", + env: { NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH: "1" }, + gatewayEnv: { OPENSHELL_DRIVERS: "docker" }, + }); + + expect(identity.launch?.mode).toBe("container"); + // The compat gateway parent process is `/usr/bin/docker`, not the host + // binary, so the executable check must be skipped via a null drift bin. + expect(identity.driftGatewayBin).toBeNull(); + // The identity bin still falls back to the host binary for listener PID + // matching, where the cmdline contains the gateway path. + expect(identity.identityGatewayBin).toBe(gatewayBin); + + // Callers must preserve that deliberate null rather than coalescing it + // back to the host binary (the #4520 false-stale bug). + expect(resolveDriftGatewayBin(identity, gatewayBin)).toBeNull(); + // `?? gatewayBin` would have wrongly restored the host path: + expect(identity.driftGatewayBin ?? gatewayBin).toBe(gatewayBin); + }); + }); + + it("uses the host binary as the drift binary outside compatibility mode", () => { + withTempBinaries(({ dir, gatewayBin }) => { + const identity = buildDockerDriverGatewayRuntimeIdentity({ + gatewayBin, + stateDir: dir, + platform: "linux", + env: {}, + hostGlibcVersion: "2.39", + requiredGlibcVersions: ["2.39"], + gatewayEnv: { OPENSHELL_DRIVERS: "docker" }, + }); + + expect(identity.launch?.mode).toBe("host"); + expect(identity.driftGatewayBin).toBe(gatewayBin); + expect(resolveDriftGatewayBin(identity, gatewayBin)).toBe(gatewayBin); + }); + }); + + it("falls back to the host binary when no runtime identity is available", () => { + expect(resolveDriftGatewayBin(null, "/opt/openshell/openshell-gateway")).toBe( + "/opt/openshell/openshell-gateway", + ); + expect(resolveDriftGatewayBin(null, null)).toBeNull(); + }); + it("uses the host binary when the gateway ABI is compatible", () => { withTempBinaries(({ dir, gatewayBin }) => { const launch = buildDockerDriverGatewayLaunch({ diff --git a/src/lib/onboard/docker-driver-gateway-launch.ts b/src/lib/onboard/docker-driver-gateway-launch.ts index bf7155c64ee..a9bb104d4a3 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { execFileSync, spawn, type ChildProcess } from "node:child_process"; +import { type ChildProcess, execFileSync, spawn } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; @@ -364,6 +364,25 @@ export function buildDockerDriverGatewayRuntimeIdentity( }; } +/** + * Resolve the gateway binary used for the `/proc//exe` drift comparison. + * + * In containerized Docker-compat mode the gateway runs as a host-side + * `docker run ... /opt/nemoclaw/openshell-gateway` parent, so the parent + * process's executable is `/usr/bin/docker`, not the host openshell-gateway + * binary. `buildDockerDriverGatewayRuntimeIdentity()` encodes that with + * `driftGatewayBin: null` to mean "skip the executable check". Callers must + * preserve that deliberate `null` — coalescing it back to the host binary with + * `??` makes the drift check compare `/usr/bin/docker` against the host + * gateway path and falsely mark a healthy compat gateway as stale (#4520). + */ +export function resolveDriftGatewayBin( + runtimeIdentity: DockerDriverGatewayRuntimeIdentity | null, + gatewayBin: string | null, +): string | null { + return runtimeIdentity ? runtimeIdentity.driftGatewayBin : gatewayBin; +} + export function prepareAndLogDockerDriverGatewayLaunch( launch: DockerDriverGatewayLaunch, log: (message: string) => void = console.log, diff --git a/src/lib/onboard/host-service-reachability.test.ts b/src/lib/onboard/host-service-reachability.test.ts new file mode 100644 index 00000000000..37263b6425a --- /dev/null +++ b/src/lib/onboard/host-service-reachability.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Unit tests for the generic sandbox-side host-service reachability probe. +// +// See: https://github.com/NVIDIA/NemoClaw/issues/3340 (Ollama auth proxy) and +// https://github.com/NVIDIA/NemoClaw/issues/4564 (Model Router port 4000). + +import { describe, expect, it, vi } from "vitest"; + +// Mock the docker adapter so the test never loads runner.ts (which requires +// the compiled ./platform artifact unavailable in the test environment). +vi.mock("../adapters/docker/run", () => ({ + dockerRun: vi.fn(), + dockerCapture: vi.fn(), +})); + +import { + DEFAULT_PROBE_NETWORK, + formatHostServiceUnreachableMessage, + probeHostServiceSandboxReachability, +} from "./host-service-reachability"; + +function makeNetwork( + partial: { subnet?: string; gatewayIp?: string } = {}, +): { subnet?: string; gatewayIp?: string } { + return { subnet: "172.18.0.0/16", gatewayIp: "172.18.0.1", ...partial }; +} + +describe("probeHostServiceSandboxReachability", () => { + it("returns ok and echoes the probed port when nc connects", async () => { + const result = await probeHostServiceSandboxReachability({ + port: 4000, + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 0 }), + }); + expect(result.ok).toBe(true); + expect(result.reason).toBe("ok"); + expect(result.port).toBe(4000); + expect(result.networkName).toBe(DEFAULT_PROBE_NETWORK); + }); + + it("classifies a UFW-blocked Linux Docker-driver router as tcp_failed (#4564)", async () => { + const result = await probeHostServiceSandboxReachability({ + port: 4000, + inspectNetworkImpl: () => makeNetwork(), + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 1, stderr: "nc: connect failed" }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("tcp_failed"); + expect(result.port).toBe(4000); + expect(result.detail).toContain("host.openshell.internal"); + expect(result.detail).toContain("4000"); + }); + + it("treats a missing sandbox network as probe_unavailable (non-fatal during fresh setup)", async () => { + const result = await probeHostServiceSandboxReachability({ + port: 4000, + inspectNetworkImpl: () => undefined, + usesHostGatewayRouteImpl: () => false, + runImpl: () => ({ status: 0 }), + }); + expect(result.ok).toBe(false); + expect(result.reason).toBe("probe_unavailable"); + }); + + it("probes the requested port and host alias in the docker run args", async () => { + let capturedArgs: readonly string[] = []; + await probeHostServiceSandboxReachability({ + port: 4000, + networkName: "openshell-docker", + inspectNetworkImpl: () => makeNetwork({ gatewayIp: "172.18.0.1" }), + usesHostGatewayRouteImpl: () => false, + runImpl: (args) => { + capturedArgs = args; + return { status: 0 }; + }, + }); + expect(capturedArgs).toContain("openshell-docker"); + expect(capturedArgs).toContain("host.openshell.internal:172.18.0.1"); + expect(capturedArgs).toContain("nc"); + expect(capturedArgs).toContain("host.openshell.internal"); + expect(capturedArgs).toContain("4000"); + }); +}); + +describe("formatHostServiceUnreachableMessage", () => { + it("emits a Model Router UFW remediation for the routed port (#4564)", () => { + const msg = formatHostServiceUnreachableMessage( + { + ok: false, + reason: "tcp_failed", + port: 4000, + networkName: "openshell-docker", + subnet: "172.18.0.0/16", + gatewayIp: "172.18.0.1", + }, + { serviceLabel: "Model Router", port: 4000 }, + ); + expect(msg).toContain("Model Router"); + expect(msg).toContain("host.openshell.internal:4000"); + expect(msg).toContain("sudo ufw allow from 172.18.0.0/16 to 172.18.0.1 port 4000 proto tcp"); + expect(msg).toContain("nemoclaw onboard"); + }); + + it("falls back to result.port when no explicit port option is given", () => { + const msg = formatHostServiceUnreachableMessage( + { + ok: false, + reason: "tcp_failed", + port: 4000, + networkName: "openshell-docker", + subnet: "172.18.0.0/16", + }, + { serviceLabel: "Model Router" }, + ); + expect(msg).toContain("sudo ufw allow from 172.18.0.0/16 to any port 4000 proto tcp"); + }); + + it("returns empty string for ok and probe_unavailable results", () => { + expect( + formatHostServiceUnreachableMessage( + { ok: true, reason: "ok", port: 4000, networkName: "openshell-docker" }, + { serviceLabel: "Model Router" }, + ), + ).toBe(""); + expect( + formatHostServiceUnreachableMessage( + { ok: false, reason: "probe_unavailable", port: 4000, networkName: "openshell-docker" }, + { serviceLabel: "Model Router" }, + ), + ).toBe(""); + }); +}); diff --git a/src/lib/onboard/host-service-reachability.ts b/src/lib/onboard/host-service-reachability.ts new file mode 100644 index 00000000000..615b9d9b7fe --- /dev/null +++ b/src/lib/onboard/host-service-reachability.ts @@ -0,0 +1,270 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Sandbox-side reachability probe for a host service exposed on a TCP port. + * + * Issue #3340 / #4564: On Brev VMs (and any Linux host with UFW default-deny), + * a host service — the Ollama auth proxy on 11435, the Model Router on 4000 — + * is unreachable from the sandbox's Docker bridge network even though + * host-side `curl 127.0.0.1:` succeeds. Host-side validation cannot + * detect this because `host.openshell.internal` only resolves inside the + * sandbox network. This probe runs a short-lived container on the same Docker + * network OpenShell uses for sandboxes and performs a TCP connect to the host + * service port, mirroring the exact route the real sandbox takes. A + * `tcp_failed` result means a host firewall is blocking the port; the caller + * can then surface an actionable `ufw allow` remediation before declaring + * onboard successful. + */ + +import { dockerCapture, dockerRun } from "../adapters/docker/run"; + +export const DEFAULT_PROBE_NETWORK = "openshell-docker"; +const HOST_INTERNAL_NAME = "host.openshell.internal"; +// Pinned busybox digest — same image used by the gateway bridge probe so +// it is likely already pulled and avoids a redundant registry fetch. +const PROBE_IMAGE = + "busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; +const PROBE_TIMEOUT_SEC = 5; +const PROBE_OVERHEAD_MS = 10_000; + +export type HostServiceReachabilityReason = "ok" | "tcp_failed" | "probe_unavailable"; + +export interface HostServiceReachabilityResult { + ok: boolean; + reason: HostServiceReachabilityReason; + // The probe always records the port it tested; it is optional on the type so + // callers (and the Ollama wrapper, whose formatter takes an explicit port) + // can construct result literals without it. + port?: number; + networkName: string; + subnet?: string; + gatewayIp?: string; + detail?: string; +} + +interface ProbeRunResult { + status: number | null; + signal?: NodeJS.Signals | null; + error?: string; + stderr?: string | Buffer | null; +} + +export interface HostServiceReachabilityOptions { + port: number; + networkName?: string; + timeoutSec?: number; + probeImage?: string; + runImpl?: (args: readonly string[], timeoutMs: number) => ProbeRunResult; + inspectNetworkImpl?: (networkName: string) => { subnet?: string; gatewayIp?: string } | undefined; + usesHostGatewayRouteImpl?: () => boolean; +} + +function parseNetworkIpamConfig( + raw: string, +): { subnet?: string; gatewayIp?: string } | undefined { + const text = raw.trim(); + if (!text || text === "") return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return undefined; + } + if (!Array.isArray(parsed)) return undefined; + for (const entry of parsed) { + if (!entry || typeof entry !== "object") continue; + const r = entry as Record; + const subnet = typeof r.Subnet === "string" ? r.Subnet : undefined; + const gatewayIp = typeof r.Gateway === "string" ? r.Gateway : undefined; + // Skip IPv6-only entries (contain colons) + if (gatewayIp && !gatewayIp.includes(":")) return { subnet, gatewayIp }; + } + return undefined; +} + +function defaultInspectNetwork( + networkName: string, +): { subnet?: string; gatewayIp?: string } | undefined { + const raw = dockerCapture( + ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], + { ignoreError: true }, + ); + return parseNetworkIpamConfig(raw); +} + +// Docker Desktop and VM-backed Docker use a special host-gateway alias rather +// than a specific bridge IP. UFW is not relevant on those platforms, so we +// classify probes from those environments as probe_unavailable. +function defaultUsesHostGatewayRoute(): boolean { + if (process.platform !== "linux") return true; + const info = dockerCapture( + ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], + { ignoreError: true }, + ); + return /Docker Desktop|com\.docker\.desktop\./i.test(info); +} + +function defaultRunImpl(args: readonly string[], timeoutMs: number): ProbeRunResult { + const result = dockerRun(args, { + timeout: timeoutMs, + ignoreError: true, + suppressOutput: true, + stdio: ["ignore", "pipe", "pipe"], + }); + return { + status: result.status ?? null, + signal: result.signal, + error: result.error?.message, + stderr: result.stderr, + }; +} + +function outputTail(value: unknown): string | undefined { + const raw = + Buffer.isBuffer(value) + ? value.toString("utf8") + : value == null + ? "" + : String(value); + const text = raw.trim(); + return text ? text.slice(-400) : undefined; +} + +function isNameResolutionFailure(detail: string): boolean { + return /bad address|name or service not known|temporary failure in name resolution|could not resolve|getaddrinfo/i.test( + detail, + ); +} + +export async function probeHostServiceSandboxReachability( + opts: HostServiceReachabilityOptions, +): Promise { + const networkName = + opts.networkName ?? + process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? + DEFAULT_PROBE_NETWORK; + const port = opts.port; + const timeoutSec = opts.timeoutSec ?? PROBE_TIMEOUT_SEC; + const probeImage = opts.probeImage ?? PROBE_IMAGE; + const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; + const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; + const runImpl = opts.runImpl ?? defaultRunImpl; + + const network = inspectNetwork(networkName); + if (!network) { + return { + ok: false, + reason: "probe_unavailable", + port, + networkName, + detail: `Docker network "${networkName}" not found`, + }; + } + + const isHostGateway = usesHostGatewayRoute(); + + if (!isHostGateway && !network.gatewayIp) { + return { + ok: false, + reason: "probe_unavailable", + port, + networkName, + subnet: network.subnet, + detail: `Docker network "${networkName}" has no IPv4 gateway`, + }; + } + + const hostInternalTarget = isHostGateway ? "host-gateway" : (network.gatewayIp as string); + + const probeArgs = [ + "run", + "--rm", + "--pull=missing", + "--network", + networkName, + "--add-host", + `${HOST_INTERNAL_NAME}:${hostInternalTarget}`, + probeImage, + "nc", + `-zw${timeoutSec}`, + HOST_INTERNAL_NAME, + String(port), + ]; + + const result = runImpl(probeArgs, timeoutSec * 1000 + PROBE_OVERHEAD_MS); + + if (result.status === 0) { + return { + ok: true, + reason: "ok", + port, + networkName, + subnet: network.subnet, + gatewayIp: network.gatewayIp, + }; + } + + const detail = [ + result.error, + outputTail(result.stderr), + result.signal ? `signal ${result.signal}` : undefined, + result.status !== null ? `exit ${result.status}` : undefined, + ] + .filter((s): s is string => Boolean(s)) + .join(" | "); + + // Classify as probe_unavailable for: non-nc exit codes, DNS failures, + // or host-gateway mode (Docker Desktop / macOS — no UFW concern there). + if (result.status !== 1 || isNameResolutionFailure(detail) || isHostGateway) { + return { + ok: false, + reason: "probe_unavailable", + port, + networkName, + subnet: network.subnet, + gatewayIp: network.gatewayIp, + detail: detail || "probe did not complete", + }; + } + + return { + ok: false, + reason: "tcp_failed", + port, + networkName, + subnet: network.subnet, + gatewayIp: network.gatewayIp, + detail: `sandbox container on "${networkName}" could not reach ${HOST_INTERNAL_NAME}:${port}`, + }; +} + +export function formatHostServiceUnreachableMessage( + result: HostServiceReachabilityResult, + options: { serviceLabel: string; port?: number }, +): string { + if (result.ok || result.reason !== "tcp_failed") return ""; + + const port = options.port ?? result.port; + const allowCmd = + result.subnet && result.gatewayIp + ? ` sudo ufw allow from ${result.subnet} to ${result.gatewayIp} port ${port} proto tcp` + : result.subnet + ? ` sudo ufw allow from ${result.subnet} to any port ${port} proto tcp` + : [ + ` SUBNET=$(docker network inspect ${result.networkName ?? DEFAULT_PROBE_NETWORK} --format '{{(index .IPAM.Config 0).Subnet}}')`, + ` sudo ufw allow from "$SUBNET" to any port ${port} proto tcp`, + ].join("\n"); + + return [ + ` ✗ Sandbox containers cannot reach the ${options.serviceLabel} at ${HOST_INTERNAL_NAME}:${port}.`, + " A host firewall may be blocking traffic from the OpenShell Docker bridge.", + " To allow it:", + allowCmd, + " Then re-run `nemoclaw onboard`.", + ].join("\n"); +} + +export const __test = { + parseNetworkIpamConfig, +}; diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index 5414e898a5d..0f672787d2b 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -42,6 +42,12 @@ function createDeps(overrides: Partial false), reconcileRouter: vi.fn(async () => undefined), + reupsertRoutedProvider: vi.fn( + (_provider: string, endpointUrl: string | null, _credentialEnv: string | null) => ({ + ok: true as const, + endpointUrl: "http://host.openshell.internal:4000/v1", + }), + ), updateSandbox: vi.fn(), promptName: vi.fn(async () => "my-assistant"), promptYesNo: vi.fn(async () => true), @@ -73,6 +79,7 @@ function createDeps(overrides: Partial provider === "nvidia-router", reconcileModelRouter: calls.reconcileRouter, + reupsertRoutedProvider: calls.reupsertRoutedProvider, registryUpdateSandbox: calls.updateSandbox, promptValidatedSandboxName: calls.promptName, assessHost: () => ({ cpus: 8 }), @@ -330,6 +337,64 @@ describe("handleProviderInferenceState", () => { expect(calls.reconcileRouter).toHaveBeenCalledOnce(); }); + // Regression: #4564. On resume the routed provider was only reconciled, never + // re-upserted, so a stale localhost base URL recorded by an earlier run could + // survive in the gateway and break inference.local from the sandbox. + it("re-upserts the routed provider with the host alias on resume (#4564)", async () => { + const session = createSession({ + provider: "nvidia-router", + model: "router/model", + endpointUrl: "http://localhost:4000/v1", + credentialEnv: "NVIDIA_API_KEY", + }); + session.steps.provider_selection.status = "complete"; + const { deps, calls } = createDeps({ isInferenceRouteReady: vi.fn(() => true) }); + + const result = await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "router-sandbox", + }); + + expect(calls.reconcileRouter).toHaveBeenCalledOnce(); + expect(calls.reupsertRoutedProvider).toHaveBeenCalledWith( + "nvidia-router", + "http://localhost:4000/v1", + "NVIDIA_API_KEY", + ); + expect(calls.setupInference).not.toHaveBeenCalled(); + expect(result.endpointUrl).toBe("http://host.openshell.internal:4000/v1"); + }); + + it("aborts resume when re-upserting the routed provider fails (#4564)", async () => { + const session = createSession({ + provider: "nvidia-router", + model: "router/model", + endpointUrl: "http://localhost:4000/v1", + }); + session.steps.provider_selection.status = "complete"; + const { deps, calls } = createDeps({ + isInferenceRouteReady: vi.fn(() => true), + reupsertRoutedProvider: vi.fn(() => ({ + ok: false, + endpointUrl: "http://host.openshell.internal:4000/v1", + message: "provider update failed", + status: 7, + })), + }); + + await expect( + handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "router-sandbox", + }), + ).rejects.toThrow("exit 7"); + + expect(calls.error).toHaveBeenCalledWith(" provider update failed"); + expect(calls.exit).toHaveBeenCalledWith(7); + }); + it("returns to provider selection when inference setup requests a retry", async () => { const setupNim = vi .fn() diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 44d2cf5ed51..6bea5f43d72 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -84,6 +84,11 @@ export interface ProviderInferenceStateOptions { isInferenceRouteReady(provider: string, model: string): boolean; isRoutedInferenceProvider(provider: string): boolean; reconcileModelRouter(): Promise; + reupsertRoutedProvider( + provider: string, + endpointUrl: string | null, + credentialEnv: string | null, + ): { ok: boolean; endpointUrl: string; message?: string; status?: number }; registryUpdateSandbox(sandboxName: string, updates: { nimContainer?: string | null }): void; promptValidatedSandboxName(agent: Agent): Promise; assessHost(): Host; @@ -304,6 +309,15 @@ export async function handleProviderInferenceState({ deps.error(` ✗ Failed to reconcile model router: ${err instanceof Error ? err.message : String(err)}`); deps.exitProcess(1); } + // #4564: re-upsert the gateway provider with the sandbox-facing + // endpoint so a stale localhost base URL recorded by an earlier run is + // repaired on resume instead of surviving and breaking inference.local. + const reupserted = deps.reupsertRoutedProvider(provider, endpointUrl, credentialEnv); + if (!reupserted.ok) { + deps.error(` ${reupserted.message ?? "Failed to update the routed inference provider."}`); + deps.exitProcess(reupserted.status ?? 1); + } + endpointUrl = reupserted.endpointUrl; } deps.skippedStepMessage("inference", `${provider} / ${model}`); await deps.recordStateSkipped("inference", { diff --git a/src/lib/onboard/model-router.ts b/src/lib/onboard/model-router.ts index 8d8ec43b433..1c3b5b7c37d 100644 --- a/src/lib/onboard/model-router.ts +++ b/src/lib/onboard/model-router.ts @@ -18,6 +18,10 @@ import type { Session } from "../state/onboard-session"; import * as onboardSession from "../state/onboard-session"; import { buildSubprocessEnv } from "../subprocess-env"; import { hydrateCredentialEnv } from "./credential-env"; +import { + formatHostServiceUnreachableMessage, + probeHostServiceSandboxReachability, +} from "./host-service-reachability"; import { doesModelRouterProcessOwnPort, isRouterHealthy, @@ -419,6 +423,35 @@ export function isRoutedInferenceProvider(provider: string | null | undefined): return Boolean(bp?.provider_name && provider === bp.provider_name); } +const MODEL_ROUTER_SERVICE_LABEL = "Model Router"; + +/** + * Verify the host Model Router is reachable from the OpenShell Docker network. + * + * `isRouterHealthy()` only proves the router answers on the host loopback. On + * Linux Docker-driver hosts with UFW default-deny, a sandbox container can + * still fail to reach `host.openshell.internal:` even though the + * host curl succeeds (#4564). This mirrors the Ollama auth-proxy probe: on a + * `tcp_failed` result print the concrete `ufw allow` remediation and fail so + * onboarding does not declare an unreachable router healthy. A + * `probe_unavailable` result (Docker Desktop, missing network during fresh + * setup before the sandbox network exists, DNS) is non-fatal. + */ +async function verifyModelRouterSandboxReachability(routerPort: number): Promise { + const reachability = await probeHostServiceSandboxReachability({ port: routerPort }); + if (!reachability.ok && reachability.reason === "tcp_failed") { + console.error( + formatHostServiceUnreachableMessage(reachability, { + serviceLabel: MODEL_ROUTER_SERVICE_LABEL, + port: routerPort, + }), + ); + throw new Error( + `Sandbox containers cannot reach the Model Router at host.openshell.internal:${routerPort}.`, + ); + } +} + export async function reconcileModelRouter(): Promise { const bp = getRoutedProfile(); const routerPort = bp.router.port || 4000; @@ -444,6 +477,7 @@ export async function reconcileModelRouter(): Promise { recordedProcessOwnsRouter ) { console.log(` ✓ Model router is already healthy on port ${routerPort}`); + await verifyModelRouterSandboxReachability(routerPort); return; } if (recordedProcessOwnsRouter) { @@ -464,4 +498,5 @@ export async function reconcileModelRouter(): Promise { current.routerCredentialHash = routerCredentialHash; return current; }); + await verifyModelRouterSandboxReachability(routerPort); } diff --git a/src/lib/onboard/ollama-proxy-reachability.ts b/src/lib/onboard/ollama-proxy-reachability.ts index 69b728a6f52..895472236e0 100644 --- a/src/lib/onboard/ollama-proxy-reachability.ts +++ b/src/lib/onboard/ollama-proxy-reachability.ts @@ -6,254 +6,47 @@ * * Issue #3340: On Brev VMs (and any Linux host with UFW default-deny), the * Ollama auth proxy on port 11435 is unreachable from the sandbox's Docker - * bridge network. Host-side validation cannot detect this because - * host.openshell.internal only resolves inside the sandbox network. This - * probe runs a short-lived container on the same Docker network OpenShell - * uses for sandboxes and performs a TCP connect to the proxy port, mirroring - * the exact route the real sandbox takes. A tcp_failed result means a host - * firewall is blocking the port; the caller can then surface an actionable - * ufw remediation before declaring onboard successful. + * bridge network. This is a thin Ollama-specific wrapper over the generic + * host-service reachability probe (see ./host-service-reachability); the + * generic helper runs a short-lived container on the OpenShell Docker network + * and TCP-connects to host.openshell.internal:. */ -import { dockerCapture, dockerRun } from "../adapters/docker/run"; import { OLLAMA_PROXY_PORT } from "../core/ports"; - -export const DEFAULT_OLLAMA_PROBE_NETWORK = "openshell-docker"; -const HOST_INTERNAL_NAME = "host.openshell.internal"; -// Pinned busybox digest — same image used by the gateway bridge probe so -// it is likely already pulled and avoids a redundant registry fetch. -const PROBE_IMAGE = - "busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; -const PROBE_TIMEOUT_SEC = 5; -const PROBE_OVERHEAD_MS = 10_000; - -export type OllamaProxyReachabilityReason = "ok" | "tcp_failed" | "probe_unavailable"; - -export interface OllamaProxyReachabilityResult { - ok: boolean; - reason: OllamaProxyReachabilityReason; - networkName: string; - subnet?: string; - gatewayIp?: string; - detail?: string; -} - -interface ProbeRunResult { - status: number | null; - signal?: NodeJS.Signals | null; - error?: string; - stderr?: string | Buffer | null; -} - -export interface OllamaProxyReachabilityOptions { - port?: number; - networkName?: string; - timeoutSec?: number; - probeImage?: string; - runImpl?: (args: readonly string[], timeoutMs: number) => ProbeRunResult; - inspectNetworkImpl?: (networkName: string) => { subnet?: string; gatewayIp?: string } | undefined; - usesHostGatewayRouteImpl?: () => boolean; -} - -function parseNetworkIpamConfig( - raw: string, -): { subnet?: string; gatewayIp?: string } | undefined { - const text = raw.trim(); - if (!text || text === "") return undefined; - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch { - return undefined; - } - if (!Array.isArray(parsed)) return undefined; - for (const entry of parsed) { - if (!entry || typeof entry !== "object") continue; - const r = entry as Record; - const subnet = typeof r.Subnet === "string" ? r.Subnet : undefined; - const gatewayIp = typeof r.Gateway === "string" ? r.Gateway : undefined; - // Skip IPv6-only entries (contain colons) - if (gatewayIp && !gatewayIp.includes(":")) return { subnet, gatewayIp }; - } - return undefined; -} - -function defaultInspectNetwork( - networkName: string, -): { subnet?: string; gatewayIp?: string } | undefined { - const raw = dockerCapture( - ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], - { ignoreError: true }, - ); - return parseNetworkIpamConfig(raw); -} - -// Docker Desktop and VM-backed Docker use a special host-gateway alias rather -// than a specific bridge IP. UFW is not relevant on those platforms, so we -// classify probes from those environments as probe_unavailable. -function defaultUsesHostGatewayRoute(): boolean { - if (process.platform !== "linux") return true; - const info = dockerCapture( - ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], - { ignoreError: true }, - ); - return /Docker Desktop|com\.docker\.desktop\./i.test(info); -} - -function defaultRunImpl(args: readonly string[], timeoutMs: number): ProbeRunResult { - const result = dockerRun(args, { - timeout: timeoutMs, - ignoreError: true, - suppressOutput: true, - stdio: ["ignore", "pipe", "pipe"], - }); - return { - status: result.status ?? null, - signal: result.signal, - error: result.error?.message, - stderr: result.stderr, - }; -} - -function outputTail(value: unknown): string | undefined { - const raw = - Buffer.isBuffer(value) - ? value.toString("utf8") - : value == null - ? "" - : String(value); - const text = raw.trim(); - return text ? text.slice(-400) : undefined; -} - -function isNameResolutionFailure(detail: string): boolean { - return /bad address|name or service not known|temporary failure in name resolution|could not resolve|getaddrinfo/i.test( - detail, - ); -} +import { + DEFAULT_PROBE_NETWORK, + formatHostServiceUnreachableMessage, + type HostServiceReachabilityOptions, + type HostServiceReachabilityReason, + type HostServiceReachabilityResult, + __test as hostServiceTest, + probeHostServiceSandboxReachability, +} from "./host-service-reachability"; + +export const DEFAULT_OLLAMA_PROBE_NETWORK = DEFAULT_PROBE_NETWORK; +const OLLAMA_SERVICE_LABEL = "Ollama auth proxy"; + +export type OllamaProxyReachabilityReason = HostServiceReachabilityReason; +export type OllamaProxyReachabilityResult = HostServiceReachabilityResult; +export type OllamaProxyReachabilityOptions = Partial; export async function probeOllamaProxySandboxReachability( opts: OllamaProxyReachabilityOptions = {}, ): Promise { - const networkName = - opts.networkName ?? - process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? - DEFAULT_OLLAMA_PROBE_NETWORK; - const port = opts.port ?? OLLAMA_PROXY_PORT; - const timeoutSec = opts.timeoutSec ?? PROBE_TIMEOUT_SEC; - const probeImage = opts.probeImage ?? PROBE_IMAGE; - const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; - const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; - const runImpl = opts.runImpl ?? defaultRunImpl; - - const network = inspectNetwork(networkName); - if (!network) { - return { - ok: false, - reason: "probe_unavailable", - networkName, - detail: `Docker network "${networkName}" not found`, - }; - } - - const isHostGateway = usesHostGatewayRoute(); - - if (!isHostGateway && !network.gatewayIp) { - return { - ok: false, - reason: "probe_unavailable", - networkName, - subnet: network.subnet, - detail: `Docker network "${networkName}" has no IPv4 gateway`, - }; - } - - const hostInternalTarget = isHostGateway ? "host-gateway" : (network.gatewayIp as string); - - const probeArgs = [ - "run", - "--rm", - "--pull=missing", - "--network", - networkName, - "--add-host", - `${HOST_INTERNAL_NAME}:${hostInternalTarget}`, - probeImage, - "nc", - `-zw${timeoutSec}`, - HOST_INTERNAL_NAME, - String(port), - ]; - - const result = runImpl(probeArgs, timeoutSec * 1000 + PROBE_OVERHEAD_MS); - - if (result.status === 0) { - return { - ok: true, - reason: "ok", - networkName, - subnet: network.subnet, - gatewayIp: network.gatewayIp, - }; - } - - const detail = [ - result.error, - outputTail(result.stderr), - result.signal ? `signal ${result.signal}` : undefined, - result.status !== null ? `exit ${result.status}` : undefined, - ] - .filter((s): s is string => Boolean(s)) - .join(" | "); - - // Classify as probe_unavailable for: non-nc exit codes, DNS failures, - // or host-gateway mode (Docker Desktop / macOS — no UFW concern there). - if (result.status !== 1 || isNameResolutionFailure(detail) || isHostGateway) { - return { - ok: false, - reason: "probe_unavailable", - networkName, - subnet: network.subnet, - gatewayIp: network.gatewayIp, - detail: detail || "probe did not complete", - }; - } - - return { - ok: false, - reason: "tcp_failed", - networkName, - subnet: network.subnet, - gatewayIp: network.gatewayIp, - detail: `sandbox container on "${networkName}" could not reach ${HOST_INTERNAL_NAME}:${port}`, - }; + return probeHostServiceSandboxReachability({ + ...opts, + port: opts.port ?? OLLAMA_PROXY_PORT, + }); } export function formatOllamaProxyUnreachableMessage( result: OllamaProxyReachabilityResult, port: number = OLLAMA_PROXY_PORT, ): string { - if (result.ok || result.reason !== "tcp_failed") return ""; - - const allowCmd = - result.subnet && result.gatewayIp - ? ` sudo ufw allow from ${result.subnet} to ${result.gatewayIp} port ${port} proto tcp` - : result.subnet - ? ` sudo ufw allow from ${result.subnet} to any port ${port} proto tcp` - : [ - ` SUBNET=$(docker network inspect ${result.networkName ?? DEFAULT_OLLAMA_PROBE_NETWORK} --format '{{(index .IPAM.Config 0).Subnet}}')`, - ` sudo ufw allow from "$SUBNET" to any port ${port} proto tcp`, - ].join("\n"); - - return [ - ` ✗ Sandbox containers cannot reach the Ollama auth proxy at ${HOST_INTERNAL_NAME}:${port}.`, - " A host firewall may be blocking traffic from the OpenShell Docker bridge.", - " To allow it:", - allowCmd, - " Then re-run `nemoclaw onboard`.", - ].join("\n"); + return formatHostServiceUnreachableMessage(result, { + serviceLabel: OLLAMA_SERVICE_LABEL, + port, + }); } -export const __test = { - parseNetworkIpamConfig, -}; +export const __test = hostServiceTest; diff --git a/src/lib/onboard/routed-inference.test.ts b/src/lib/onboard/routed-inference.test.ts new file mode 100644 index 00000000000..8f5fb5cf668 --- /dev/null +++ b/src/lib/onboard/routed-inference.test.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Unit tests for routed (Model Router) provider endpoint normalization and +// upsert. See: https://github.com/NVIDIA/NemoClaw/issues/4564 + +import { describe, expect, it, vi } from "vitest"; + +// Mock the heavy transitive imports so this test does not load runner.ts / +// the compiled ./platform artifact. +vi.mock("../inference/local", () => ({ + HOST_GATEWAY_URL: "http://host.openshell.internal", +})); +vi.mock("./model-router", () => ({ + DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV: "NVIDIA_API_KEY", + loadBlueprintProfile: vi.fn(() => ({ endpoint: "http://localhost:4000/v1" })), +})); + +import { + normalizeRoutedEndpointUrl, + resolveRoutedCredentialEnv, + upsertRoutedProvider, +} from "./routed-inference"; + +describe("normalizeRoutedEndpointUrl (#4564)", () => { + it("rewrites localhost to the sandbox-facing host alias", () => { + expect(normalizeRoutedEndpointUrl("http://localhost:4000/v1")).toBe( + "http://host.openshell.internal:4000/v1", + ); + }); + + it("rewrites 127.0.0.1 to the host alias", () => { + expect(normalizeRoutedEndpointUrl("http://127.0.0.1:4000/v1")).toBe( + "http://host.openshell.internal:4000/v1", + ); + }); + + it("leaves an already-aliased endpoint untouched", () => { + expect(normalizeRoutedEndpointUrl("http://host.openshell.internal:4000/v1")).toBe( + "http://host.openshell.internal:4000/v1", + ); + }); + + it("falls back to the blueprint endpoint when none is recorded, then normalizes it", () => { + // The mocked loadBlueprintProfile returns http://localhost:4000/v1. + expect(normalizeRoutedEndpointUrl(null)).toBe("http://host.openshell.internal:4000/v1"); + expect(normalizeRoutedEndpointUrl("")).toBe("http://host.openshell.internal:4000/v1"); + }); +}); + +describe("resolveRoutedCredentialEnv (#4564)", () => { + it("prefers an explicitly recorded credential env", () => { + const loadProfile = vi.fn(() => ({ credential_env: "CUSTOM_KEY" })) as never; + expect(resolveRoutedCredentialEnv("SESSION_KEY", loadProfile)).toBe("SESSION_KEY"); + }); + + it("falls back to the routed profile credential env before the NVIDIA default", () => { + const loadProfile = vi.fn(() => ({ + credential_env: "CUSTOM_KEY", + router: { credential_env: "ROUTER_KEY" }, + })) as never; + // router.credential_env wins (mirrors reconcileModelRouter resolution). + expect(resolveRoutedCredentialEnv(null, loadProfile)).toBe("ROUTER_KEY"); + }); + + it("falls back to the profile-level credential env when the router has none", () => { + const loadProfile = vi.fn(() => ({ credential_env: "CUSTOM_KEY" })) as never; + expect(resolveRoutedCredentialEnv(null, loadProfile)).toBe("CUSTOM_KEY"); + }); + + it("uses the NVIDIA default when no profile credential env is set", () => { + const loadProfile = vi.fn(() => ({ endpoint: "http://localhost:4000/v1" })) as never; + expect(resolveRoutedCredentialEnv(null, loadProfile)).toBe("NVIDIA_API_KEY"); + }); +}); + +describe("upsertRoutedProvider (#4564)", () => { + it("upserts the provider with the normalized host alias base URL", () => { + const upsertProvider = vi.fn(() => ({ ok: true })); + const hydrateCredentialEnv = vi.fn(() => "nvapi-secret"); + + const result = upsertRoutedProvider("nvidia-router", "http://localhost:4000/v1", "NVIDIA_API_KEY", { + upsertProvider, + hydrateCredentialEnv, + }); + + expect(result.ok).toBe(true); + expect(result.endpointUrl).toBe("http://host.openshell.internal:4000/v1"); + expect(result.resolvedCredentialEnv).toBe("NVIDIA_API_KEY"); + expect(upsertProvider).toHaveBeenCalledWith( + "nvidia-router", + "openai", + "NVIDIA_API_KEY", + "http://host.openshell.internal:4000/v1", + { NVIDIA_API_KEY: "nvapi-secret" }, + ); + }); + + it("defaults the credential env and omits an empty credential from the env block", () => { + const upsertProvider = vi.fn(() => ({ ok: true })); + const hydrateCredentialEnv = vi.fn(() => undefined); + + const result = upsertRoutedProvider("nvidia-router", "http://localhost:4000/v1", null, { + upsertProvider, + hydrateCredentialEnv, + }); + + expect(result.resolvedCredentialEnv).toBe("NVIDIA_API_KEY"); + expect(upsertProvider).toHaveBeenCalledWith( + "nvidia-router", + "openai", + "NVIDIA_API_KEY", + "http://host.openshell.internal:4000/v1", + {}, + ); + }); + + it("propagates a failed upsert result", () => { + const upsertProvider = vi.fn(() => ({ ok: false, message: "boom", status: 3 })); + const hydrateCredentialEnv = vi.fn(() => "nvapi-secret"); + + const result = upsertRoutedProvider("nvidia-router", "http://localhost:4000/v1", "NVIDIA_API_KEY", { + upsertProvider, + hydrateCredentialEnv, + }); + + expect(result.ok).toBe(false); + expect(result.result.message).toBe("boom"); + expect(result.result.status).toBe(3); + }); +}); diff --git a/src/lib/onboard/routed-inference.ts b/src/lib/onboard/routed-inference.ts new file mode 100644 index 00000000000..62f0368984c --- /dev/null +++ b/src/lib/onboard/routed-inference.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Helpers for wiring the routed (Model Router) inference provider into the + * OpenShell gateway. + * + * Model Router runs on the host (default port 4000), but sandboxes reach it + * through the OpenShell gateway. On Linux Docker-driver hosts `localhost:4000` + * is the sandbox/proxy loopback, not the host router, so the gateway provider + * base URL must use the `host.openshell.internal` alias. These helpers + * normalize the blueprint endpoint to that sandbox-facing alias and upsert the + * provider, so both fresh setup and resume repair a stale `localhost` base URL + * left behind by an earlier run (#4564). + */ + +import { HOST_GATEWAY_URL } from "../inference/local"; +import { DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV, loadBlueprintProfile } from "./model-router"; + +export type UpsertProviderResult = { + ok: boolean; + message?: string; + status?: number; +}; + +export type RoutedProviderDeps = { + upsertProvider: ( + name: string, + type: string, + credentialEnv: string, + baseUrl: string | null, + env: NodeJS.ProcessEnv, + ) => UpsertProviderResult; + hydrateCredentialEnv: (credentialEnv: string) => string | null | undefined; +}; + +export type RoutedProviderUpsert = { + ok: boolean; + endpointUrl: string; + resolvedCredentialEnv: string; + result: UpsertProviderResult; +}; + +/** + * Rewrite a routed-provider endpoint to the sandbox-facing host alias. + * + * `localhost`/`127.0.0.1` endpoints are rewritten to + * `http://host.openshell.internal:`. A missing endpoint falls back + * to the routed blueprint profile so a resume with no recorded endpoint still + * repairs to the correct alias rather than leaving the gateway pointed at a + * stale loopback URL. + */ +export function normalizeRoutedEndpointUrl( + endpointUrl: string | null | undefined, + loadProfile: typeof loadBlueprintProfile = loadBlueprintProfile, +): string { + let url = (endpointUrl || "").trim(); + if (!url) { + url = (loadProfile("routed")?.endpoint || "").trim(); + } + if (!url) return url; + if (/localhost|127\.0\.0\.1/.test(url)) { + try { + const parsed = new URL(url); + return `${HOST_GATEWAY_URL}:${parsed.port}${parsed.pathname}`; + } catch { + return url; + } + } + return url; +} + +/** + * Resolve the credential env for the routed provider. + * + * Mirrors `reconcileModelRouter()`'s resolution order so the gateway provider + * is bound to the same key the router process reads: an explicit recorded env + * first, then the routed blueprint profile's credential env, and only then the + * `NVIDIA_API_KEY` default. Without the profile step a resume with no recorded + * credential env would re-upsert the provider against `NVIDIA_API_KEY` even + * when the routed profile defines a custom `credential_env`, breaking + * inference.local (#4564). + */ +export function resolveRoutedCredentialEnv( + credentialEnv: string | null, + loadProfile: typeof loadBlueprintProfile = loadBlueprintProfile, +): string { + if (credentialEnv) return credentialEnv; + const profile = loadProfile("routed"); + return ( + profile?.router?.credential_env || + profile?.credential_env || + DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV + ); +} + +/** + * Upsert the routed provider into the gateway with a normalized, sandbox-facing + * base URL. Used by both fresh routed setup and resume repair. + */ +export function upsertRoutedProvider( + provider: string, + endpointUrl: string | null, + credentialEnv: string | null, + deps: RoutedProviderDeps, +): RoutedProviderUpsert { + const resolvedCredentialEnv = resolveRoutedCredentialEnv(credentialEnv); + const normalizedEndpoint = normalizeRoutedEndpointUrl(endpointUrl); + const credentialValue = deps.hydrateCredentialEnv(resolvedCredentialEnv); + const env = credentialValue ? { [resolvedCredentialEnv]: credentialValue } : {}; + const result = deps.upsertProvider( + provider, + "openai", + resolvedCredentialEnv, + normalizedEndpoint, + env, + ); + return { + ok: result.ok, + endpointUrl: normalizedEndpoint, + resolvedCredentialEnv, + result, + }; +} diff --git a/test/onboard-gateway-runtime.test.ts b/test/onboard-gateway-runtime.test.ts index 641651b1c7a..c84b6687927 100644 --- a/test/onboard-gateway-runtime.test.ts +++ b/test/onboard-gateway-runtime.test.ts @@ -255,6 +255,34 @@ describe("onboard gateway runtime helpers", () => { ).toContain("process environment"); }); + it("reuses a healthy containerized-compat gateway whose parent is /usr/bin/docker (#4520)", () => { + const desiredEnv = getDockerDriverGatewayEnv("openshell 0.0.37", "linux"); + + // The compat gateway is a `docker run ... /opt/nemoclaw/openshell-gateway` + // parent, so /proc//exe is /usr/bin/docker. The runtime identity sets + // driftGatewayBin=null to skip the executable comparison; with that null + // preserved, a healthy compat gateway is NOT judged stale. + expect( + getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: desiredEnv, + processExe: "/usr/bin/docker", + desiredEnv, + gatewayBin: null, + }), + ).toBeNull(); + + // Regression guard: if a caller coalesces the null back to the host binary + // (the old `?? gatewayBin` bug), the same healthy gateway is falsely stale. + expect( + getDockerDriverGatewayRuntimeDriftFromSnapshot({ + processEnv: desiredEnv, + processExe: "/usr/bin/docker", + desiredEnv, + gatewayBin: "/home/user/.local/bin/openshell-gateway", + })?.reason, + ).toContain("executable=/usr/bin/docker"); + }); + it("recognizes an existing Docker-driver gateway listener on Docker-driver platforms", () => { const opts = { platform: "linux" as NodeJS.Platform, From 3e7b8499c722e079f7acae77b5600d557bc0b3ee Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Mon, 1 Jun 2026 05:25:35 +0000 Subject: [PATCH 2/2] fix(onboard): guard portless routed endpoint and preserve query/hash normalizeRoutedEndpointUrl emitted a dangling colon (host.openshell.internal:/v1) for a portless localhost endpoint and dropped any query/hash. Only append the port when present and carry through search and hash. Addresses CodeRabbit review on #4601. Signed-off-by: Yimo Jiang --- src/lib/onboard/routed-inference.test.ts | 6 ++++++ src/lib/onboard/routed-inference.ts | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/routed-inference.test.ts b/src/lib/onboard/routed-inference.test.ts index 8f5fb5cf668..0a7d4ea974b 100644 --- a/src/lib/onboard/routed-inference.test.ts +++ b/src/lib/onboard/routed-inference.test.ts @@ -35,6 +35,12 @@ describe("normalizeRoutedEndpointUrl (#4564)", () => { ); }); + it("omits the colon when the endpoint has no explicit port and preserves query/hash", () => { + expect(normalizeRoutedEndpointUrl("http://localhost/v1?x=1#frag")).toBe( + "http://host.openshell.internal/v1?x=1#frag", + ); + }); + it("leaves an already-aliased endpoint untouched", () => { expect(normalizeRoutedEndpointUrl("http://host.openshell.internal:4000/v1")).toBe( "http://host.openshell.internal:4000/v1", diff --git a/src/lib/onboard/routed-inference.ts b/src/lib/onboard/routed-inference.ts index 62f0368984c..ebdc56f9c66 100644 --- a/src/lib/onboard/routed-inference.ts +++ b/src/lib/onboard/routed-inference.ts @@ -62,7 +62,8 @@ export function normalizeRoutedEndpointUrl( if (/localhost|127\.0\.0\.1/.test(url)) { try { const parsed = new URL(url); - return `${HOST_GATEWAY_URL}:${parsed.port}${parsed.pathname}`; + const port = parsed.port ? `:${parsed.port}` : ""; + return `${HOST_GATEWAY_URL}${port}${parsed.pathname}${parsed.search}${parsed.hash}`; } catch { return url; }