diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index bcf606e74b3..b870833c414 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -390,9 +390,9 @@ import { getSuggestedPolicyPresets } from "./onboard/policy-presets"; import { computeSetupPresetSuggestions as computeSetupPresetSuggestionsImpl, isStaleBuiltinBravePolicyPreset, - setupPoliciesWithSelection as setupPoliciesWithSelectionImpl, type SetupPolicySelectionOptions, type SetupPresetSuggestionOptions, + setupPoliciesWithSelection as setupPoliciesWithSelectionImpl, } from "./onboard/policy-selection"; import { getResumeSandboxGpuOverrides, @@ -400,6 +400,11 @@ import { type SandboxGpuConfig, type SandboxGpuFlag, } from "./onboard/sandbox-gpu-mode"; +import { + exitOnSandboxGpuConfigErrors, + sandboxGpuRemediationLines, + validateSandboxGpuPreflight, +} from "./onboard/sandbox-gpu-preflight"; import type { SelectionDrift } from "./onboard/selection-drift"; import { formatOnboardConfigSummary, formatSandboxBuildEstimateNote } from "./onboard/summary"; import type { @@ -1198,35 +1203,6 @@ function resolveSandboxGpuFlagFromOptions( return null; } -function sandboxGpuRemediationLines(): string[] { - return [ - "Install/configure NVIDIA Container Toolkit CDI, then restart Docker:", - " sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml", - " sudo systemctl restart docker", - "Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0.", - ]; -} - -function validateSandboxGpuPreflight(config: SandboxGpuConfig): void { - if (config.errors.length > 0) { - console.error(""); - for (const error of config.errors) console.error(` ✗ ${error}`); - process.exit(1); - } - if (!config.sandboxGpuEnabled) return; - if (process.platform !== "linux") return; - - const cdiSpecDirs = getDockerCdiSpecDirs(); - const cdiSpecFiles = findReadableNvidiaCdiSpecFiles(cdiSpecDirs); - if (cdiSpecFiles.length === 0) { - console.error(""); - console.error(" ✗ Docker CDI GPU support was not detected."); - for (const line of sandboxGpuRemediationLines()) console.error(` ${line}`); - process.exit(1); - } - console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`); -} - // ── Base image resolution ─────────────────────────────────────── // Pulls candidate sandbox-base images from GHCR and inspects them to get the // actual repo digest when available. This avoids the registry mismatch that @@ -3304,8 +3280,16 @@ async function preflight( } console.log(" ✓ Docker is running"); require("./onboard/http-proxy-preflight").warnIfHostProxyMissesLoopback(); + const gpu = nim.detectGpu(); + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: resolveSandboxGpuFlagFromOptions(preflightOpts), + device: preflightOpts.sandboxGpuDevice ?? null, + }); + exitOnSandboxGpuConfigErrors(sandboxGpuConfig); const optedOutGpuPassthrough = - preflightOpts.optedOutGpuPassthrough === true || preflightOpts.noGpu === true; + preflightOpts.optedOutGpuPassthrough === true || + preflightOpts.noGpu === true || + !sandboxGpuConfig.sandboxGpuEnabled; assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough); // DNS resolution from inside containers (#2101). A corp firewall that @@ -3764,7 +3748,6 @@ async function preflight( dockerDriverGatewayEnv.warnIfGatewayWildcardBindAddress(); // GPU - const gpu = nim.detectGpu(); if (gpu && gpu.type === "nvidia") { const lines = nim.formatNvidiaGpuPreflightLines(gpu); console.log(` ✓ ${lines[0]}`); @@ -3783,10 +3766,6 @@ async function preflight( console.log(" ⓘ Local NIM unavailable — no GPU detected"); } - const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { - flag: resolveSandboxGpuFlagFromOptions(preflightOpts), - device: preflightOpts.sandboxGpuDevice ?? null, - }); validateSandboxGpuPreflight(sandboxGpuConfig); if (sandboxGpuConfig.sandboxGpuEnabled) { console.log( @@ -9364,6 +9343,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { if (resumePreflight) { skippedStepMessage("preflight", "cached"); gpu = nim.detectGpu(); + const resumeSandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: effectiveSandboxGpuFlag, + device: effectiveSandboxGpuDevice, + }); + exitOnSandboxGpuConfigErrors(resumeSandboxGpuConfig); // Re-check the CDI spec gap on resume (#3152). The cached preflight // result does not capture host CDI state, and the original onboard // attempt that wrote the cache likely aborted at gateway-start with @@ -9373,14 +9357,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { // does not need to re-pass `--no-gpu` to keep that intent (the same // resolution is replayed a few lines below for `gpuPassthrough`). const resumeOptedOutGpuPassthrough = - opts.noGpu === true || (opts.gpu !== true && session?.gpuPassthrough === false); + opts.noGpu === true || + (opts.gpu !== true && session?.gpuPassthrough === false) || + !resumeSandboxGpuConfig.sandboxGpuEnabled; assertCdiNvidiaGpuSpecPresent(assessHost(), resumeOptedOutGpuPassthrough); - validateSandboxGpuPreflight( - resolveSandboxGpuConfig(gpu, { - flag: effectiveSandboxGpuFlag, - device: effectiveSandboxGpuDevice, - }), - ); + validateSandboxGpuPreflight(resumeSandboxGpuConfig); } else { startRecordedStep("preflight"); gpu = await preflight({ ...opts, optedOutGpuPassthrough: opts.noGpu === true }); @@ -9504,6 +9485,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { gpuPassthrough, gatewayName: GATEWAY_NAME, currentSandboxName: recordedSandboxName || requestedSandboxName, + hostGpuPlatform: gpu?.platform ?? null, recreateSandbox: isRecreateSandbox(), confirmedDockerDriverGateway: isLinuxDockerDriverGatewayEnabled() && diff --git a/src/lib/onboard/gateway-gpu-passthrough.test.ts b/src/lib/onboard/gateway-gpu-passthrough.test.ts index d3e8ef4712f..67c2c6936cb 100644 --- a/src/lib/onboard/gateway-gpu-passthrough.test.ts +++ b/src/lib/onboard/gateway-gpu-passthrough.test.ts @@ -7,13 +7,15 @@ vi.mock("../adapters/docker", () => ({ dockerInspect: vi.fn(), })); +import * as docker from "../adapters/docker"; +import type { GatewayReuseState } from "../state/gateway"; import { canRestartCpuOnlyGatewayForGpuIntent, decideGatewayGpuReuseForGpuIntent, inspectLegacyGatewayGpuPassthroughResult, + reconcileGatewayGpuReuseForGpuIntent, shouldInspectLegacyGatewayGpuPassthrough, } from "./gateway-gpu-passthrough"; -import type { GatewayReuseState } from "../state/gateway"; describe("gateway GPU passthrough inspection", () => { const healthy: GatewayReuseState = "healthy"; @@ -138,4 +140,44 @@ describe("gateway GPU passthrough inspection", () => { expect(canRestartCpuOnlyGatewayForGpuIntent(["alpha"], "beta", true)).toBe(false); expect(canRestartCpuOnlyGatewayForGpuIntent(["alpha", "beta"], "alpha", true)).toBe(false); }); + + it("aborts unsupported Jetson GPU passthrough before gateway inspection or cleanup", () => { + vi.mocked(docker.dockerInspect).mockClear(); + const stopDashboardForwards = vi.fn(); + const retireLegacyGatewayForDockerDriverUpgrade = vi.fn(); + const destroyGatewayRuntimeForGpuReuse = vi.fn(); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { + throw new Error(`exit:${code}`); + }) as never); + + try { + expect(() => + reconcileGatewayGpuReuseForGpuIntent({ + gatewayReuseState: healthy, + gpuPassthrough: true, + gatewayName: "nemoclaw", + currentSandboxName: "jetson-box", + hostGpuPlatform: "jetson", + recreateSandbox: true, + confirmedDockerDriverGateway: false, + stopDashboardForwards, + retireLegacyGatewayForDockerDriverUpgrade, + destroyGatewayRuntimeForGpuReuse, + }), + ).toThrow("exit:1"); + + const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); + expect(message).toContain("Jetson/Tegra sandbox GPU passthrough is not supported"); + expect(message).toContain("--no-gpu"); + expect(message).not.toContain("destroy --yes"); + expect(docker.dockerInspect).not.toHaveBeenCalled(); + expect(stopDashboardForwards).not.toHaveBeenCalled(); + expect(retireLegacyGatewayForDockerDriverUpgrade).not.toHaveBeenCalled(); + expect(destroyGatewayRuntimeForGpuReuse).not.toHaveBeenCalled(); + } finally { + errorSpy.mockRestore(); + exitSpy.mockRestore(); + } + }); }); diff --git a/src/lib/onboard/gateway-gpu-passthrough.ts b/src/lib/onboard/gateway-gpu-passthrough.ts index 40199cb548c..4745c723c1b 100644 --- a/src/lib/onboard/gateway-gpu-passthrough.ts +++ b/src/lib/onboard/gateway-gpu-passthrough.ts @@ -1,12 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import * as docker from "../adapters/docker"; +import type { NvidiaPlatform } from "../inference/nim"; import type { GatewayReuseState } from "../state/gateway"; import * as registry from "../state/registry"; -import * as docker from "../adapters/docker"; +import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; import { destroyGatewayForReuse } from "./gateway-cleanup"; import { reportGpuPassthroughRecovery } from "./gpu-recovery"; -import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; export type LegacyGatewayGpuInspection = "gpu-enabled" | "cpu-only" | "not-found" | "unknown"; @@ -23,6 +24,7 @@ export type GatewayGpuReuseReconcileOptions = { gpuPassthrough: boolean; gatewayName: string; currentSandboxName: string | null; + hostGpuPlatform?: NvidiaPlatform | null; recreateSandbox: boolean; confirmedDockerDriverGateway: boolean; stopDashboardForwards: () => void; @@ -132,12 +134,18 @@ export function reconcileGatewayGpuReuseForGpuIntent({ gpuPassthrough, gatewayName, currentSandboxName, + hostGpuPlatform = null, recreateSandbox, confirmedDockerDriverGateway, stopDashboardForwards, retireLegacyGatewayForDockerDriverUpgrade, destroyGatewayRuntimeForGpuReuse, }: GatewayGpuReuseReconcileOptions): GatewayReuseState { + if (gpuPassthrough && hostGpuPlatform === "jetson") { + reportGpuPassthroughRecovery(console.error, () => [], { unsupportedPlatform: "jetson" }); + process.exit(1); + } + if (!shouldInspectLegacyGatewayGpuPassthrough( gatewayReuseState, gpuPassthrough, diff --git a/src/lib/onboard/gpu-recovery.test.ts b/src/lib/onboard/gpu-recovery.test.ts index 7b899f38802..c0e3d545b4b 100644 --- a/src/lib/onboard/gpu-recovery.test.ts +++ b/src/lib/onboard/gpu-recovery.test.ts @@ -75,6 +75,18 @@ describe("gpuPassthroughRecoveryLines", () => { // No double-spaced "nemoclaw destroy" rendering. expect(joined).not.toMatch(/nemoclaw\s{2,}destroy/); }); + + it("does not suggest destroy/recreate as sufficient for unsupported Jetson passthrough", () => { + const lines = gpuPassthroughRecoveryLines(["jetson-box"], { + unsupportedPlatform: "jetson", + }); + const joined = lines.join("\n"); + expect(joined).toContain("Jetson/Tegra sandbox GPU passthrough is not supported"); + expect(joined).toContain("--no-gpu"); + expect(joined).toContain("NEMOCLAW_SANDBOX_GPU=0"); + expect(joined).not.toContain("destroy --yes"); + expect(joined).not.toContain("nemoclaw onboard --gpu"); + }); }); describe("reportGpuPassthroughRecovery", () => { @@ -94,4 +106,18 @@ describe("reportGpuPassthroughRecovery", () => { expect(joined).toContain("nemoclaw alpha destroy --yes"); expect(joined).toContain("nemoclaw beta destroy --yes --cleanup-gateway"); }); + + it("does not load registered names for unsupported Jetson passthrough", () => { + const emit = vi.fn(); + const loadNames = vi.fn(() => ["jetson-box"]); + reportGpuPassthroughRecovery(emit, loadNames, { + unsupportedPlatform: "jetson", + }); + const joined = emit.mock.calls.map((c) => c[0]).join("\n"); + + expect(loadNames).not.toHaveBeenCalled(); + expect(joined).toContain("Jetson/Tegra sandbox GPU passthrough is not supported"); + expect(joined).toContain("nemoclaw onboard --no-gpu"); + expect(joined).not.toContain("jetson-box"); + }); }); diff --git a/src/lib/onboard/gpu-recovery.ts b/src/lib/onboard/gpu-recovery.ts index 5ec977a4e62..4bc3fff8226 100644 --- a/src/lib/onboard/gpu-recovery.ts +++ b/src/lib/onboard/gpu-recovery.ts @@ -14,6 +14,14 @@ */ import * as registry from "../state/registry"; +import { + JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, + JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, +} from "./sandbox-gpu-mode"; + +export type GpuPassthroughRecoveryOptions = { + unsupportedPlatform?: "jetson" | null; +}; /** * Returns the multi-line recovery hint for the GPU-passthrough mismatch @@ -29,7 +37,19 @@ import * as registry from "../state/registry"; * line each; only the last carries `--cleanup-gateway` so the gateway lives * until every sandbox is gone. */ -export function gpuPassthroughRecoveryLines(names: readonly string[] | null): string[] { +export function gpuPassthroughRecoveryLines( + names: readonly string[] | null, + options: GpuPassthroughRecoveryOptions = {}, +): string[] { + if (options.unsupportedPlatform === "jetson") { + return [ + ` ${JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE}`, + ` ${JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE}`, + " Use CPU sandbox mode instead:", + " nemoclaw onboard --no-gpu", + ]; + } + const cleanNames = (names ?? []).map((n) => n.trim()).filter((n) => n.length > 0); if (cleanNames.length === 0) { @@ -90,6 +110,8 @@ export function getRegisteredSandboxNamesForGpuRecovery(): string[] { export function reportGpuPassthroughRecovery( emit: (line: string) => void, loadNames: () => string[] = getRegisteredSandboxNamesForGpuRecovery, + options: GpuPassthroughRecoveryOptions = {}, ): void { - for (const line of gpuPassthroughRecoveryLines(loadNames())) emit(line); + const names = options.unsupportedPlatform === "jetson" ? [] : loadNames(); + for (const line of gpuPassthroughRecoveryLines(names, options)) emit(line); } diff --git a/src/lib/onboard/sandbox-gpu-mode.test.ts b/src/lib/onboard/sandbox-gpu-mode.test.ts index 25c106055d8..95b92776a15 100644 --- a/src/lib/onboard/sandbox-gpu-mode.test.ts +++ b/src/lib/onboard/sandbox-gpu-mode.test.ts @@ -4,7 +4,12 @@ import { describe, expect, it } from "vitest"; import type { GpuDetection } from "../inference/nim"; -import { getResumeSandboxGpuOverrides, resolveSandboxGpuConfig } from "./sandbox-gpu-mode"; +import { + getResumeSandboxGpuOverrides, + JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, + JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, + resolveSandboxGpuConfig, +} from "./sandbox-gpu-mode"; function gpu(overrides: Partial = {}): GpuDetection { return { @@ -81,9 +86,12 @@ describe("sandbox GPU mode helpers", () => { expect(explicitFlagEnable.errors).toEqual([]); }); - it("defaults to CPU sandbox on Jetson when NEMOCLAW_SANDBOX_GPU is unset", () => { + it("defaults to CPU sandbox on Jetson unless GPU passthrough is forced", () => { const jetson = gpu({ platform: "jetson" }); expect(resolveSandboxGpuConfig(jetson, { env: {} }).sandboxGpuEnabled).toBe(false); + expect(resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU: "auto" } }).mode).toBe( + "0", + ); const jetsonDeviceOnly = resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU_DEVICE: "nvidia.com/gpu=0" }, }); @@ -93,9 +101,20 @@ describe("sandbox GPU mode helpers", () => { const jetsonExplicitEnable = resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU: "1", NEMOCLAW_SANDBOX_GPU_DEVICE: "nvidia.com/gpu=0" }, }); - expect(jetsonExplicitEnable.sandboxGpuEnabled).toBe(true); - expect(jetsonExplicitEnable.sandboxGpuDevice).toBe("nvidia.com/gpu=0"); - expect(resolveSandboxGpuConfig(jetson, { flag: "enable", env: {} }).mode).toBe("1"); + expect(jetsonExplicitEnable.errors.join("\n")).toContain( + JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, + ); + expect(jetsonExplicitEnable.errors.join("\n")).toContain( + JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, + ); + const jetsonFlagEnable = resolveSandboxGpuConfig(jetson, { flag: "enable", env: {} }); + expect(jetsonFlagEnable.mode).toBe("1"); + expect(jetsonFlagEnable.errors.join("\n")).toContain( + JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, + ); + expect(jetsonFlagEnable.errors.join("\n")).toContain( + JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, + ); }); it("resumes sandbox GPU auto mode without turning CPU fallback into explicit opt-out", () => { diff --git a/src/lib/onboard/sandbox-gpu-mode.ts b/src/lib/onboard/sandbox-gpu-mode.ts index 4aa94a9fd2f..845ba7ad135 100644 --- a/src/lib/onboard/sandbox-gpu-mode.ts +++ b/src/lib/onboard/sandbox-gpu-mode.ts @@ -14,6 +14,11 @@ export type SandboxGpuConfig = { errors: string[]; }; +export const JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE = + "Jetson/Tegra sandbox GPU passthrough is not supported by NemoClaw/OpenShell."; +export const JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE = + "Destroying/recreating the sandbox or gateway will not enable it; re-run with --no-gpu or NEMOCLAW_SANDBOX_GPU=0."; + export type ResumeSandboxGpuOverrides = { flag: SandboxGpuFlag; device: string | null; @@ -40,8 +45,11 @@ export function resolveSandboxGpuMode(args: { flag?: SandboxGpuFlag; }): SandboxGpuMode { let mode: SandboxGpuMode = args.envMode ?? "auto"; - // GPU sandbox passthrough does not currently work on Jetson; disable by default - if (args.gpu?.platform === "jetson" && args.envMode === null) mode = "0"; + // GPU sandbox passthrough is not supported on Jetson/Tegra; keep auto/default + // behavior on the CPU sandbox path unless the user explicitly forces GPU. + if (args.gpu?.platform === "jetson" && (args.envMode === null || args.envMode === "auto")) { + mode = "0"; + } if (args.flag === "enable") mode = "1"; if (args.flag === "disable") mode = "0"; return mode; @@ -78,6 +86,10 @@ export function resolveSandboxGpuConfig( if (mode === "1" && !hostGpuDetected) { errors.push("Sandbox GPU was requested, but no NVIDIA GPU was detected on the host."); } + if (mode === "1" && gpu?.platform === "jetson") { + errors.push(JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE); + errors.push(JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE); + } return { mode, diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts new file mode 100644 index 00000000000..d386303368a --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { findReadableNvidiaCdiSpecFiles, getDockerCdiSpecDirs } from "./docker-cdi"; +import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; + +export function sandboxGpuRemediationLines(): string[] { + return [ + "Install/configure NVIDIA Container Toolkit CDI, then restart Docker:", + " sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml", + " sudo systemctl restart docker", + "Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0.", + ]; +} + +export function exitOnSandboxGpuConfigErrors(config: SandboxGpuConfig): void { + if (config.errors.length > 0) { + console.error(""); + for (const error of config.errors) console.error(` ✗ ${error}`); + process.exit(1); + } +} + +export function validateSandboxGpuPreflight(config: SandboxGpuConfig): void { + exitOnSandboxGpuConfigErrors(config); + if (!config.sandboxGpuEnabled) return; + if (process.platform !== "linux") return; + + const cdiSpecDirs = getDockerCdiSpecDirs(); + const cdiSpecFiles = findReadableNvidiaCdiSpecFiles(cdiSpecDirs); + if (cdiSpecFiles.length === 0) { + console.error(""); + console.error(" ✗ Docker CDI GPU support was not detected."); + for (const line of sandboxGpuRemediationLines()) console.error(` ${line}`); + process.exit(1); + } + console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`); +}