Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 13 additions & 26 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,7 @@ const {
loadBlueprintProfile,
reconcileModelRouter,
} = modelRouter;
const routedInference: typeof import("./onboard/routed-inference") = require("./onboard/routed-inference");
const { OnboardRuntimeBoundary }: typeof import("./onboard/runtime-boundary") = require("./onboard/runtime-boundary");
const { handleAgentSetupState }: typeof import("./onboard/machine/handlers/agent-setup") = require("./onboard/machine/handlers/agent-setup");
const { handleFinalizationState }: typeof import("./onboard/machine/handlers/finalization") = require("./onboard/machine/handlers/finalization");
Expand Down Expand Up @@ -1204,7 +1205,7 @@ async function refreshDockerDriverGatewayReuseState(
);
const runtimeIdentity = gatewayBin ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ gatewayBin, gatewayEnv: baseDesiredEnv, stateDir: getDockerDriverGatewayStateDir(), sandboxBin: resolveOpenShellSandboxBinary() }) : null;
const desiredEnv = runtimeIdentity?.desiredEnv ?? baseDesiredEnv;
const driftBin = runtimeIdentity?.driftGatewayBin ?? gatewayBin;
const driftBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin(runtimeIdentity, gatewayBin);
const identityBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin;
const pid = getDockerDriverGatewayPid();
if (pid !== null && isDockerDriverGatewayProcessAlive()) {
Expand Down Expand Up @@ -2419,7 +2420,7 @@ async function startDockerDriverGateway({ exitOnFailure = true, skipSandboxBridg
const stateDir = getDockerDriverGatewayStateDir();
const runtimeIdentity = gatewayBin ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ gatewayBin, gatewayEnv, stateDir, sandboxBin: resolveOpenShellSandboxBinary() }) : null;
const gatewayLaunch = runtimeIdentity?.launch ?? null;
const driftGatewayBin = runtimeIdentity?.driftGatewayBin ?? gatewayBin;
const driftGatewayBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin(runtimeIdentity, gatewayBin);
const driftGatewayEnv = runtimeIdentity?.desiredEnv ?? gatewayEnv;
const identityGatewayBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin;
const { verifySandboxBridgeGatewayReachableOrExit } =
Expand Down Expand Up @@ -5517,37 +5518,19 @@ async function setupInference(
// to unrelated OpenAI-backed sandboxes.
} else if (isRoutedInferenceProvider(provider)) {
// Blueprint profile provider (e.g., nvidia-router for the routed profile).
// Same pattern as vllm-local: upsert the provider and set the inference route.
// reconcileModelRouter also probes sandbox→router reachability (#4564).
try {
await reconcileModelRouter();
} catch (err) {
console.error(` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`);
process.exit(1);
}
const resolvedCredentialEnv = credentialEnv || DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV;
const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv);
const env = credentialValue ? { [resolvedCredentialEnv]: credentialValue } : {};
const providerResult = upsertProvider(
provider,
"openai",
resolvedCredentialEnv,
endpointUrl,
env,
);
if (!providerResult.ok) {
console.error(` ${providerResult.message}`);
process.exit(providerResult.status || 1);
const routed = routedInference.upsertRoutedProvider(provider, endpointUrl, credentialEnv, { upsertProvider, hydrateCredentialEnv });
if (!routed.ok) {
console.error(` ${routed.result.message}`);
process.exit(routed.result.status || 1);
}
const inferenceArgs = [
"inference",
"set",
"--no-verify",
"--provider",
provider,
"--model",
model,
];
runOpenshell(inferenceArgs);
runOpenshell(["inference", "set", "--no-verify", "--provider", provider, "--model", model]);
} else {
console.error(` Unsupported provider configuration: ${provider}`);
process.exit(1);
Expand Down Expand Up @@ -6877,6 +6860,10 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
isInferenceRouteReady,
isRoutedInferenceProvider,
reconcileModelRouter,
reupsertRoutedProvider: (p, url, ce) => {
const r = routedInference.upsertRoutedProvider(p, url, ce, { upsertProvider, hydrateCredentialEnv });
return { ok: r.ok, endpointUrl: r.endpointUrl, message: r.result.message, status: r.result.status };
},
registryUpdateSandbox: (name, updates) => registry.updateSandbox(name, updates),
promptValidatedSandboxName,
assessHost,
Expand Down
56 changes: 56 additions & 0 deletions src/lib/onboard/docker-driver-gateway-launch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import { describe, expect, it } from "vitest";
import {
buildDockerDriverGatewayConfigToml,
buildDockerDriverGatewayLaunch,
buildDockerDriverGatewayRuntimeIdentity,
parseGlibcVersionsFromBinaryText,
resolveDriftGatewayBin,
shouldUseContainerizedGateway,
} from "../../../dist/lib/onboard/docker-driver-gateway-launch";

Expand Down Expand Up @@ -170,6 +172,60 @@ describe("docker-driver-gateway-launch", () => {
});
});

it("keeps the drift gateway binary null for the containerized compatibility gateway (#4520)", () => {
withTempBinaries(({ dir, gatewayBin, sandboxBin }) => {
const stateDir = path.join(dir, "state");
fs.mkdirSync(stateDir);
const identity = buildDockerDriverGatewayRuntimeIdentity({
gatewayBin,
sandboxBin,
stateDir,
platform: "linux",
env: { NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH: "1" },
gatewayEnv: { OPENSHELL_DRIVERS: "docker" },
});

expect(identity.launch?.mode).toBe("container");
// The compat gateway parent process is `/usr/bin/docker`, not the host
// binary, so the executable check must be skipped via a null drift bin.
expect(identity.driftGatewayBin).toBeNull();
// The identity bin still falls back to the host binary for listener PID
// matching, where the cmdline contains the gateway path.
expect(identity.identityGatewayBin).toBe(gatewayBin);

// Callers must preserve that deliberate null rather than coalescing it
// back to the host binary (the #4520 false-stale bug).
expect(resolveDriftGatewayBin(identity, gatewayBin)).toBeNull();
// `?? gatewayBin` would have wrongly restored the host path:
expect(identity.driftGatewayBin ?? gatewayBin).toBe(gatewayBin);
});
});

it("uses the host binary as the drift binary outside compatibility mode", () => {
withTempBinaries(({ dir, gatewayBin }) => {
const identity = buildDockerDriverGatewayRuntimeIdentity({
gatewayBin,
stateDir: dir,
platform: "linux",
env: {},
hostGlibcVersion: "2.39",
requiredGlibcVersions: ["2.39"],
gatewayEnv: { OPENSHELL_DRIVERS: "docker" },
});

expect(identity.launch?.mode).toBe("host");
expect(identity.driftGatewayBin).toBe(gatewayBin);
expect(resolveDriftGatewayBin(identity, gatewayBin)).toBe(gatewayBin);
});
});

it("falls back to the host binary when no runtime identity is available", () => {
expect(resolveDriftGatewayBin(null, "/opt/openshell/openshell-gateway")).toBe(
"/opt/openshell/openshell-gateway",
);
expect(resolveDriftGatewayBin(null, null)).toBeNull();
});

it("uses the host binary when the gateway ABI is compatible", () => {
withTempBinaries(({ dir, gatewayBin }) => {
const launch = buildDockerDriverGatewayLaunch({
Expand Down
21 changes: 20 additions & 1 deletion src/lib/onboard/docker-driver-gateway-launch.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { execFileSync, spawn, type ChildProcess } from "node:child_process";
import { type ChildProcess, execFileSync, spawn } from "node:child_process";
import fs from "node:fs";
import path from "node:path";

Expand Down Expand Up @@ -364,6 +364,25 @@ export function buildDockerDriverGatewayRuntimeIdentity(
};
}

/**
* Resolve the gateway binary used for the `/proc/<pid>/exe` drift comparison.
*
* In containerized Docker-compat mode the gateway runs as a host-side
* `docker run ... /opt/nemoclaw/openshell-gateway` parent, so the parent
* process's executable is `/usr/bin/docker`, not the host openshell-gateway
* binary. `buildDockerDriverGatewayRuntimeIdentity()` encodes that with
* `driftGatewayBin: null` to mean "skip the executable check". Callers must
* preserve that deliberate `null` — coalescing it back to the host binary with
* `??` makes the drift check compare `/usr/bin/docker` against the host
* gateway path and falsely mark a healthy compat gateway as stale (#4520).
*/
export function resolveDriftGatewayBin(
runtimeIdentity: DockerDriverGatewayRuntimeIdentity | null,
gatewayBin: string | null,
): string | null {
return runtimeIdentity ? runtimeIdentity.driftGatewayBin : gatewayBin;
}

export function prepareAndLogDockerDriverGatewayLaunch(
launch: DockerDriverGatewayLaunch,
log: (message: string) => void = console.log,
Expand Down
136 changes: 136 additions & 0 deletions src/lib/onboard/host-service-reachability.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Unit tests for the generic sandbox-side host-service reachability probe.
//
// See: https://github.com/NVIDIA/NemoClaw/issues/3340 (Ollama auth proxy) and
// https://github.com/NVIDIA/NemoClaw/issues/4564 (Model Router port 4000).

import { describe, expect, it, vi } from "vitest";

// Mock the docker adapter so the test never loads runner.ts (which requires
// the compiled ./platform artifact unavailable in the test environment).
vi.mock("../adapters/docker/run", () => ({
dockerRun: vi.fn(),
dockerCapture: vi.fn(),
}));

import {
DEFAULT_PROBE_NETWORK,
formatHostServiceUnreachableMessage,
probeHostServiceSandboxReachability,
} from "./host-service-reachability";

function makeNetwork(
partial: { subnet?: string; gatewayIp?: string } = {},
): { subnet?: string; gatewayIp?: string } {
return { subnet: "172.18.0.0/16", gatewayIp: "172.18.0.1", ...partial };
}

describe("probeHostServiceSandboxReachability", () => {
it("returns ok and echoes the probed port when nc connects", async () => {
const result = await probeHostServiceSandboxReachability({
port: 4000,
inspectNetworkImpl: () => makeNetwork(),
usesHostGatewayRouteImpl: () => false,
runImpl: () => ({ status: 0 }),
});
expect(result.ok).toBe(true);
expect(result.reason).toBe("ok");
expect(result.port).toBe(4000);
expect(result.networkName).toBe(DEFAULT_PROBE_NETWORK);
});

it("classifies a UFW-blocked Linux Docker-driver router as tcp_failed (#4564)", async () => {
const result = await probeHostServiceSandboxReachability({
port: 4000,
inspectNetworkImpl: () => makeNetwork(),
usesHostGatewayRouteImpl: () => false,
runImpl: () => ({ status: 1, stderr: "nc: connect failed" }),
});
expect(result.ok).toBe(false);
expect(result.reason).toBe("tcp_failed");
expect(result.port).toBe(4000);
expect(result.detail).toContain("host.openshell.internal");
expect(result.detail).toContain("4000");
});

it("treats a missing sandbox network as probe_unavailable (non-fatal during fresh setup)", async () => {
const result = await probeHostServiceSandboxReachability({
port: 4000,
inspectNetworkImpl: () => undefined,
usesHostGatewayRouteImpl: () => false,
runImpl: () => ({ status: 0 }),
});
expect(result.ok).toBe(false);
expect(result.reason).toBe("probe_unavailable");
});

it("probes the requested port and host alias in the docker run args", async () => {
let capturedArgs: readonly string[] = [];
await probeHostServiceSandboxReachability({
port: 4000,
networkName: "openshell-docker",
inspectNetworkImpl: () => makeNetwork({ gatewayIp: "172.18.0.1" }),
usesHostGatewayRouteImpl: () => false,
runImpl: (args) => {
capturedArgs = args;
return { status: 0 };
},
});
expect(capturedArgs).toContain("openshell-docker");
expect(capturedArgs).toContain("host.openshell.internal:172.18.0.1");
expect(capturedArgs).toContain("nc");
expect(capturedArgs).toContain("host.openshell.internal");
expect(capturedArgs).toContain("4000");
});
});

describe("formatHostServiceUnreachableMessage", () => {
it("emits a Model Router UFW remediation for the routed port (#4564)", () => {
const msg = formatHostServiceUnreachableMessage(
{
ok: false,
reason: "tcp_failed",
port: 4000,
networkName: "openshell-docker",
subnet: "172.18.0.0/16",
gatewayIp: "172.18.0.1",
},
{ serviceLabel: "Model Router", port: 4000 },
);
expect(msg).toContain("Model Router");
expect(msg).toContain("host.openshell.internal:4000");
expect(msg).toContain("sudo ufw allow from 172.18.0.0/16 to 172.18.0.1 port 4000 proto tcp");
expect(msg).toContain("nemoclaw onboard");
});

it("falls back to result.port when no explicit port option is given", () => {
const msg = formatHostServiceUnreachableMessage(
{
ok: false,
reason: "tcp_failed",
port: 4000,
networkName: "openshell-docker",
subnet: "172.18.0.0/16",
},
{ serviceLabel: "Model Router" },
);
expect(msg).toContain("sudo ufw allow from 172.18.0.0/16 to any port 4000 proto tcp");
});

it("returns empty string for ok and probe_unavailable results", () => {
expect(
formatHostServiceUnreachableMessage(
{ ok: true, reason: "ok", port: 4000, networkName: "openshell-docker" },
{ serviceLabel: "Model Router" },
),
).toBe("");
expect(
formatHostServiceUnreachableMessage(
{ ok: false, reason: "probe_unavailable", port: 4000, networkName: "openshell-docker" },
{ serviceLabel: "Model Router" },
),
).toBe("");
});
});
Loading
Loading