Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 25 additions & 43 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -390,16 +390,21 @@ import { getSuggestedPolicyPresets } from "./onboard/policy-presets";
import {
computeSetupPresetSuggestions as computeSetupPresetSuggestionsImpl,
isStaleBuiltinBravePolicyPreset,
setupPoliciesWithSelection as setupPoliciesWithSelectionImpl,
type SetupPolicySelectionOptions,
type SetupPresetSuggestionOptions,
setupPoliciesWithSelection as setupPoliciesWithSelectionImpl,
} from "./onboard/policy-selection";
import {
getResumeSandboxGpuOverrides,
resolveSandboxGpuConfig,
type SandboxGpuConfig,
type SandboxGpuFlag,
} from "./onboard/sandbox-gpu-mode";
import {
exitOnSandboxGpuConfigErrors,
sandboxGpuRemediationLines,
validateSandboxGpuPreflight,
} from "./onboard/sandbox-gpu-preflight";
import type { SelectionDrift } from "./onboard/selection-drift";
import { formatOnboardConfigSummary, formatSandboxBuildEstimateNote } from "./onboard/summary";
import type {
Expand Down Expand Up @@ -1198,35 +1203,6 @@ function resolveSandboxGpuFlagFromOptions(
return null;
}

function sandboxGpuRemediationLines(): string[] {
return [
"Install/configure NVIDIA Container Toolkit CDI, then restart Docker:",
" sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml",
" sudo systemctl restart docker",
"Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0.",
];
}

function validateSandboxGpuPreflight(config: SandboxGpuConfig): void {
if (config.errors.length > 0) {
console.error("");
for (const error of config.errors) console.error(` ✗ ${error}`);
process.exit(1);
}
if (!config.sandboxGpuEnabled) return;
if (process.platform !== "linux") return;

const cdiSpecDirs = getDockerCdiSpecDirs();
const cdiSpecFiles = findReadableNvidiaCdiSpecFiles(cdiSpecDirs);
if (cdiSpecFiles.length === 0) {
console.error("");
console.error(" ✗ Docker CDI GPU support was not detected.");
for (const line of sandboxGpuRemediationLines()) console.error(` ${line}`);
process.exit(1);
}
console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`);
}

// ── Base image resolution ───────────────────────────────────────
// Pulls candidate sandbox-base images from GHCR and inspects them to get the
// actual repo digest when available. This avoids the registry mismatch that
Expand Down Expand Up @@ -3304,8 +3280,16 @@ async function preflight(
}
console.log(" ✓ Docker is running");
require("./onboard/http-proxy-preflight").warnIfHostProxyMissesLoopback();
const gpu = nim.detectGpu();
const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, {
flag: resolveSandboxGpuFlagFromOptions(preflightOpts),
device: preflightOpts.sandboxGpuDevice ?? null,
});
exitOnSandboxGpuConfigErrors(sandboxGpuConfig);
const optedOutGpuPassthrough =
preflightOpts.optedOutGpuPassthrough === true || preflightOpts.noGpu === true;
preflightOpts.optedOutGpuPassthrough === true ||
preflightOpts.noGpu === true ||
!sandboxGpuConfig.sandboxGpuEnabled;
assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough);

// DNS resolution from inside containers (#2101). A corp firewall that
Expand Down Expand Up @@ -3764,7 +3748,6 @@ async function preflight(
dockerDriverGatewayEnv.warnIfGatewayWildcardBindAddress();

// GPU
const gpu = nim.detectGpu();
if (gpu && gpu.type === "nvidia") {
const lines = nim.formatNvidiaGpuPreflightLines(gpu);
console.log(` ✓ ${lines[0]}`);
Expand All @@ -3783,10 +3766,6 @@ async function preflight(
console.log(" ⓘ Local NIM unavailable — no GPU detected");
}

const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, {
flag: resolveSandboxGpuFlagFromOptions(preflightOpts),
device: preflightOpts.sandboxGpuDevice ?? null,
});
validateSandboxGpuPreflight(sandboxGpuConfig);
if (sandboxGpuConfig.sandboxGpuEnabled) {
console.log(
Expand Down Expand Up @@ -9364,6 +9343,11 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
if (resumePreflight) {
skippedStepMessage("preflight", "cached");
gpu = nim.detectGpu();
const resumeSandboxGpuConfig = resolveSandboxGpuConfig(gpu, {
flag: effectiveSandboxGpuFlag,
device: effectiveSandboxGpuDevice,
});
exitOnSandboxGpuConfigErrors(resumeSandboxGpuConfig);
// Re-check the CDI spec gap on resume (#3152). The cached preflight
// result does not capture host CDI state, and the original onboard
// attempt that wrote the cache likely aborted at gateway-start with
Expand All @@ -9373,14 +9357,11 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
// does not need to re-pass `--no-gpu` to keep that intent (the same
// resolution is replayed a few lines below for `gpuPassthrough`).
const resumeOptedOutGpuPassthrough =
opts.noGpu === true || (opts.gpu !== true && session?.gpuPassthrough === false);
opts.noGpu === true ||
(opts.gpu !== true && session?.gpuPassthrough === false) ||
!resumeSandboxGpuConfig.sandboxGpuEnabled;
assertCdiNvidiaGpuSpecPresent(assessHost(), resumeOptedOutGpuPassthrough);
validateSandboxGpuPreflight(
resolveSandboxGpuConfig(gpu, {
flag: effectiveSandboxGpuFlag,
device: effectiveSandboxGpuDevice,
}),
);
validateSandboxGpuPreflight(resumeSandboxGpuConfig);
} else {
startRecordedStep("preflight");
gpu = await preflight({ ...opts, optedOutGpuPassthrough: opts.noGpu === true });
Expand Down Expand Up @@ -9504,6 +9485,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
gpuPassthrough,
gatewayName: GATEWAY_NAME,
currentSandboxName: recordedSandboxName || requestedSandboxName,
hostGpuPlatform: gpu?.platform ?? null,
recreateSandbox: isRecreateSandbox(),
confirmedDockerDriverGateway:
isLinuxDockerDriverGatewayEnabled() &&
Expand Down
44 changes: 43 additions & 1 deletion src/lib/onboard/gateway-gpu-passthrough.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ vi.mock("../adapters/docker", () => ({
dockerInspect: vi.fn(),
}));

import * as docker from "../adapters/docker";
import type { GatewayReuseState } from "../state/gateway";
import {
canRestartCpuOnlyGatewayForGpuIntent,
decideGatewayGpuReuseForGpuIntent,
inspectLegacyGatewayGpuPassthroughResult,
reconcileGatewayGpuReuseForGpuIntent,
shouldInspectLegacyGatewayGpuPassthrough,
} from "./gateway-gpu-passthrough";
import type { GatewayReuseState } from "../state/gateway";

describe("gateway GPU passthrough inspection", () => {
const healthy: GatewayReuseState = "healthy";
Expand Down Expand Up @@ -138,4 +140,44 @@ describe("gateway GPU passthrough inspection", () => {
expect(canRestartCpuOnlyGatewayForGpuIntent(["alpha"], "beta", true)).toBe(false);
expect(canRestartCpuOnlyGatewayForGpuIntent(["alpha", "beta"], "alpha", true)).toBe(false);
});

it("aborts unsupported Jetson GPU passthrough before gateway inspection or cleanup", () => {
vi.mocked(docker.dockerInspect).mockClear();
const stopDashboardForwards = vi.fn();
const retireLegacyGatewayForDockerDriverUpgrade = vi.fn();
const destroyGatewayRuntimeForGpuReuse = vi.fn();
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => {
throw new Error(`exit:${code}`);
}) as never);

try {
expect(() =>
reconcileGatewayGpuReuseForGpuIntent({
gatewayReuseState: healthy,
gpuPassthrough: true,
gatewayName: "nemoclaw",
currentSandboxName: "jetson-box",
hostGpuPlatform: "jetson",
recreateSandbox: true,
confirmedDockerDriverGateway: false,
stopDashboardForwards,
retireLegacyGatewayForDockerDriverUpgrade,
destroyGatewayRuntimeForGpuReuse,
}),
).toThrow("exit:1");

const message = errorSpy.mock.calls.map((call) => call[0]).join("\n");
expect(message).toContain("Jetson/Tegra sandbox GPU passthrough is not supported");
expect(message).toContain("--no-gpu");
expect(message).not.toContain("destroy --yes");
expect(docker.dockerInspect).not.toHaveBeenCalled();
expect(stopDashboardForwards).not.toHaveBeenCalled();
expect(retireLegacyGatewayForDockerDriverUpgrade).not.toHaveBeenCalled();
expect(destroyGatewayRuntimeForGpuReuse).not.toHaveBeenCalled();
} finally {
errorSpy.mockRestore();
exitSpy.mockRestore();
}
});
});
12 changes: 10 additions & 2 deletions src/lib/onboard/gateway-gpu-passthrough.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import * as docker from "../adapters/docker";
import type { NvidiaPlatform } from "../inference/nim";
import type { GatewayReuseState } from "../state/gateway";
import * as registry from "../state/registry";
import * as docker from "../adapters/docker";
import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform";
import { destroyGatewayForReuse } from "./gateway-cleanup";
import { reportGpuPassthroughRecovery } from "./gpu-recovery";
import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform";

export type LegacyGatewayGpuInspection = "gpu-enabled" | "cpu-only" | "not-found" | "unknown";

Expand All @@ -23,6 +24,7 @@ export type GatewayGpuReuseReconcileOptions = {
gpuPassthrough: boolean;
gatewayName: string;
currentSandboxName: string | null;
hostGpuPlatform?: NvidiaPlatform | null;
recreateSandbox: boolean;
confirmedDockerDriverGateway: boolean;
stopDashboardForwards: () => void;
Expand Down Expand Up @@ -132,12 +134,18 @@ export function reconcileGatewayGpuReuseForGpuIntent({
gpuPassthrough,
gatewayName,
currentSandboxName,
hostGpuPlatform = null,
recreateSandbox,
confirmedDockerDriverGateway,
stopDashboardForwards,
retireLegacyGatewayForDockerDriverUpgrade,
destroyGatewayRuntimeForGpuReuse,
}: GatewayGpuReuseReconcileOptions): GatewayReuseState {
if (gpuPassthrough && hostGpuPlatform === "jetson") {
reportGpuPassthroughRecovery(console.error, () => [], { unsupportedPlatform: "jetson" });
process.exit(1);
}

if (!shouldInspectLegacyGatewayGpuPassthrough(
gatewayReuseState,
gpuPassthrough,
Expand Down
26 changes: 26 additions & 0 deletions src/lib/onboard/gpu-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,18 @@ describe("gpuPassthroughRecoveryLines", () => {
// No double-spaced "nemoclaw destroy" rendering.
expect(joined).not.toMatch(/nemoclaw\s{2,}destroy/);
});

it("does not suggest destroy/recreate as sufficient for unsupported Jetson passthrough", () => {
const lines = gpuPassthroughRecoveryLines(["jetson-box"], {
unsupportedPlatform: "jetson",
});
const joined = lines.join("\n");
expect(joined).toContain("Jetson/Tegra sandbox GPU passthrough is not supported");
expect(joined).toContain("--no-gpu");
expect(joined).toContain("NEMOCLAW_SANDBOX_GPU=0");
expect(joined).not.toContain("destroy --yes");
expect(joined).not.toContain("nemoclaw onboard --gpu");
});
});

describe("reportGpuPassthroughRecovery", () => {
Expand All @@ -94,4 +106,18 @@ describe("reportGpuPassthroughRecovery", () => {
expect(joined).toContain("nemoclaw alpha destroy --yes");
expect(joined).toContain("nemoclaw beta destroy --yes --cleanup-gateway");
});

it("does not load registered names for unsupported Jetson passthrough", () => {
const emit = vi.fn();
const loadNames = vi.fn(() => ["jetson-box"]);
reportGpuPassthroughRecovery(emit, loadNames, {
unsupportedPlatform: "jetson",
});
const joined = emit.mock.calls.map((c) => c[0]).join("\n");

expect(loadNames).not.toHaveBeenCalled();
expect(joined).toContain("Jetson/Tegra sandbox GPU passthrough is not supported");
expect(joined).toContain("nemoclaw onboard --no-gpu");
expect(joined).not.toContain("jetson-box");
});
});
26 changes: 24 additions & 2 deletions src/lib/onboard/gpu-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@
*/

import * as registry from "../state/registry";
import {
JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE,
JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE,
} from "./sandbox-gpu-mode";

export type GpuPassthroughRecoveryOptions = {
unsupportedPlatform?: "jetson" | null;
};

/**
* Returns the multi-line recovery hint for the GPU-passthrough mismatch
Expand All @@ -29,7 +37,19 @@ import * as registry from "../state/registry";
* line each; only the last carries `--cleanup-gateway` so the gateway lives
* until every sandbox is gone.
*/
export function gpuPassthroughRecoveryLines(names: readonly string[] | null): string[] {
export function gpuPassthroughRecoveryLines(
names: readonly string[] | null,
options: GpuPassthroughRecoveryOptions = {},
): string[] {
if (options.unsupportedPlatform === "jetson") {
return [
` ${JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE}`,
` ${JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE}`,
" Use CPU sandbox mode instead:",
" nemoclaw onboard --no-gpu",
];
}

const cleanNames = (names ?? []).map((n) => n.trim()).filter((n) => n.length > 0);

if (cleanNames.length === 0) {
Expand Down Expand Up @@ -90,6 +110,8 @@ export function getRegisteredSandboxNamesForGpuRecovery(): string[] {
export function reportGpuPassthroughRecovery(
emit: (line: string) => void,
loadNames: () => string[] = getRegisteredSandboxNamesForGpuRecovery,
options: GpuPassthroughRecoveryOptions = {},
): void {
for (const line of gpuPassthroughRecoveryLines(loadNames())) emit(line);
const names = options.unsupportedPlatform === "jetson" ? [] : loadNames();
for (const line of gpuPassthroughRecoveryLines(names, options)) emit(line);
}
29 changes: 24 additions & 5 deletions src/lib/onboard/sandbox-gpu-mode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
import { describe, expect, it } from "vitest";

import type { GpuDetection } from "../inference/nim";
import { getResumeSandboxGpuOverrides, resolveSandboxGpuConfig } from "./sandbox-gpu-mode";
import {
getResumeSandboxGpuOverrides,
JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE,
JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE,
resolveSandboxGpuConfig,
} from "./sandbox-gpu-mode";

function gpu(overrides: Partial<GpuDetection> = {}): GpuDetection {
return {
Expand Down Expand Up @@ -81,9 +86,12 @@ describe("sandbox GPU mode helpers", () => {
expect(explicitFlagEnable.errors).toEqual([]);
});

it("defaults to CPU sandbox on Jetson when NEMOCLAW_SANDBOX_GPU is unset", () => {
it("defaults to CPU sandbox on Jetson unless GPU passthrough is forced", () => {
const jetson = gpu({ platform: "jetson" });
expect(resolveSandboxGpuConfig(jetson, { env: {} }).sandboxGpuEnabled).toBe(false);
expect(resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU: "auto" } }).mode).toBe(
"0",
);
const jetsonDeviceOnly = resolveSandboxGpuConfig(jetson, {
env: { NEMOCLAW_SANDBOX_GPU_DEVICE: "nvidia.com/gpu=0" },
});
Expand All @@ -93,9 +101,20 @@ describe("sandbox GPU mode helpers", () => {
const jetsonExplicitEnable = resolveSandboxGpuConfig(jetson, {
env: { NEMOCLAW_SANDBOX_GPU: "1", NEMOCLAW_SANDBOX_GPU_DEVICE: "nvidia.com/gpu=0" },
});
expect(jetsonExplicitEnable.sandboxGpuEnabled).toBe(true);
expect(jetsonExplicitEnable.sandboxGpuDevice).toBe("nvidia.com/gpu=0");
expect(resolveSandboxGpuConfig(jetson, { flag: "enable", env: {} }).mode).toBe("1");
expect(jetsonExplicitEnable.errors.join("\n")).toContain(
JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE,
);
expect(jetsonExplicitEnable.errors.join("\n")).toContain(
JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE,
);
const jetsonFlagEnable = resolveSandboxGpuConfig(jetson, { flag: "enable", env: {} });
expect(jetsonFlagEnable.mode).toBe("1");
expect(jetsonFlagEnable.errors.join("\n")).toContain(
JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE,
);
expect(jetsonFlagEnable.errors.join("\n")).toContain(
JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE,
);
});

it("resumes sandbox GPU auto mode without turning CPU fallback into explicit opt-out", () => {
Expand Down
Loading
Loading