diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b7a889468c7..3912df10595 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -420,6 +420,7 @@ import { } from "./onboard/sandbox-gpu-mode"; import { exitOnSandboxGpuConfigErrors, + formatSandboxGpuPassthroughNote, sandboxGpuRemediationLines, validateSandboxGpuPreflight, } from "./onboard/sandbox-gpu-preflight"; @@ -3205,19 +3206,14 @@ function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = // ── Step 1: Preflight ──────────────────────────────────────────── -// CDI spec gap (#3152). When Docker is configured for CDI device injection -// (CDISpecDirs is set) but no nvidia.com/gpu spec is present, OpenShell's -// `gateway start --gpu` fails minutes later with `unresolvable CDI devices -// nvidia.com/gpu=all`. Block now and surface `nvidia-ctk cdi generate`. The -// check is a no-op when the user opts out of GPU passthrough (--no-gpu), -// since the legacy nvidia runtime does not need a CDI spec. -// -// Extracted so the same guard runs on the `--resume` branch, where preflight() -// itself is skipped via the cached session. +// Keep the Docker CDI guard near preflight so resume hits the same early failure path. +// Jetson/Tegra uses Docker's NVIDIA runtime backend and is exempt from CDI. function assertCdiNvidiaGpuSpecPresent( host: ReturnType, optedOutGpuPassthrough: boolean, + hostGpuPlatform: string | null | undefined = null, ): void { + if (hostGpuPlatform === "jetson") return; if (!host.cdiNvidiaGpuSpecMissing || optedOutGpuPassthrough) return; console.error( " Docker is configured for CDI device injection (CDISpecDirs is set), but no", @@ -3264,7 +3260,7 @@ async function preflight( preflightOpts.optedOutGpuPassthrough === true || preflightOpts.noGpu === true || !sandboxGpuConfig.sandboxGpuEnabled; - assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough); + assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough, sandboxGpuConfig.hostGpuPlatform); // DNS resolution from inside containers (#2101). A corp firewall that // blocks outbound UDP:53 to public resolvers leaves the sandbox build @@ -5465,6 +5461,7 @@ async function createSandbox( gpuDevice: effectiveSandboxGpuConfig.sandboxGpuDevice, openshellSandboxCommand: sandboxStartupCommand, timeoutSecs: sandboxReadyTimeoutSecs, + backend: effectiveSandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", deps: { runOpenshell, runCaptureOpenshell, sleep }, }); const createResult = await streamSandboxCreate(createCommand, sandboxEnv, { @@ -9291,10 +9288,7 @@ async function onboard(opts: OnboardOptions = {}): Promise { assessHost, assertCdiNvidiaGpuSpecPresent, resolveSandboxGpuConfig, - validateSandboxGpuPreflight: (config) => { - exitOnSandboxGpuConfigErrors(config); - validateSandboxGpuPreflight(config); - }, + validateSandboxGpuPreflight, skippedStepMessage, startRecordedStep, recordStepComplete, @@ -9311,14 +9305,16 @@ async function onboard(opts: OnboardOptions = {}): Promise { const gpu = preflightResult.gpu ?? null; if (gpuPassthrough) { note( - resumeHasResolvedGpuIntent && recordedGpuPassthroughBeforePreflight - ? " [resume] Continuing GPU passthrough from the saved onboarding session." - : requestedGpuPassthrough || sandboxGpuConfig.mode === "1" - ? " GPU passthrough requested; passing --gpu to OpenShell gateway and sandbox creation." - : " NVIDIA GPU detected; enabling OpenShell GPU passthrough. Use --no-gpu to opt out.", + formatSandboxGpuPassthroughNote({ + hostGpuPlatform: sandboxGpuConfig.hostGpuPlatform, + resumeHasResolvedGpuIntent, + recordedGpuPassthroughBeforePreflight, + requestedGpuPassthrough, + sandboxGpuMode: sandboxGpuConfig.mode, + }), ); } else if (gpu?.platform === "jetson") { - note(" GPU sandbox passthrough disabled by default on Jetson."); + note(" Sandbox GPU disabled by configuration on Jetson/Tegra."); } else if (process.platform === "linux" && !opts.noGpu) { try { const lspci = spawnSync("lspci", { encoding: "utf-8", timeout: 5000 }); diff --git a/src/lib/onboard/docker-gpu-patch.test.ts b/src/lib/onboard/docker-gpu-patch.test.ts index 016467bcb1e..0b3eccb43cc 100644 --- a/src/lib/onboard/docker-gpu-patch.test.ts +++ b/src/lib/onboard/docker-gpu-patch.test.ts @@ -13,6 +13,7 @@ import { buildDockerGpuMode, buildDockerGpuModeCandidates, collectDockerGpuPatchDiagnostics, + type DockerContainerInspect, detectSandboxFallbackDns, dockerReportsNvidiaCdiDevices, formatDockerInspectNetworkSummary, @@ -21,7 +22,6 @@ import { recreateOpenShellDockerSandboxWithGpu, selectDockerGpuPatchMode, shouldApplyDockerGpuPatch, - type DockerContainerInspect, } from "../../../dist/lib/onboard/docker-gpu-patch"; function inspectFixture(): DockerContainerInspect { @@ -300,6 +300,22 @@ describe("docker-gpu-patch", () => { expect(buildDockerGpuMode("gpus", "1,2").args).toEqual(["--gpus", "device=1,2"]); }); + it("uses Jetson NVIDIA runtime args without selecting generic --gpus or CDI candidates", () => { + expect(buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }).args).toEqual([ + "--runtime", + "nvidia", + "--env", + "NVIDIA_VISIBLE_DEVICES=all", + "--env", + "NVIDIA_DRIVER_CAPABILITIES=compute,utility", + ]); + expect( + buildDockerGpuModeCandidates("all", { backend: "jetson", cdiAvailable: true }).map( + (m) => m.kind, + ), + ).toEqual(["nvidia-runtime"]); + }); + it("uses a Docker-GPU-specific supervisor reconnect wait with an override", () => { expect(getDockerGpuSupervisorReconnectTimeoutSecs(180, {})).toBe(900); expect(getDockerGpuSupervisorReconnectTimeoutSecs(600, {})).toBe(900); @@ -376,6 +392,34 @@ describe("docker-gpu-patch", () => { ]); }); + it("probes only NVIDIA runtime for Jetson Docker GPU mode", () => { + const dockerCapture = vi.fn(() => ""); + const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id" })); + + const selected = selectDockerGpuPatchMode( + { image: "openshell/sandbox:abc", backend: "jetson" }, + { + dockerCapture, + dockerRun, + dockerRm: vi.fn(() => ({ status: 0 })), + }, + ); + + expect(selected.mode?.kind).toBe("nvidia-runtime"); + expect(selected.attempts.map((attempt) => attempt.mode.kind)).toEqual(["nvidia-runtime"]); + expect(dockerRun).toHaveBeenCalledWith( + expect.arrayContaining([ + "create", + "--runtime", + "nvidia", + "--env", + "NVIDIA_DRIVER_CAPABILITIES=compute,utility", + ]), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerCapture).not.toHaveBeenCalled(); + }); + it("tries CDI only when Docker reports readable NVIDIA CDI specs", () => { expect(buildDockerGpuModeCandidates("all", { cdiAvailable: false }).map((m) => m.kind)).toEqual( ["gpus", "nvidia-runtime"], diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 90abea45021..a5e562bee4d 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -69,6 +69,7 @@ export type DockerGpuPatchDeps = { }; export type DockerGpuPatchModeKind = "gpus" | "nvidia-runtime" | "cdi"; +export type DockerGpuPatchBackend = "generic" | "jetson"; export type DockerGpuPatchMode = { kind: DockerGpuPatchModeKind; @@ -294,7 +295,11 @@ function normalizeGpuDeviceForCdi(device: string | null | undefined): string { return `nvidia.com/gpu=${dockerDevice || "all"}`; } -export function buildDockerGpuMode(kind: DockerGpuPatchModeKind, device?: string | null): DockerGpuPatchMode { +export function buildDockerGpuMode( + kind: DockerGpuPatchModeKind, + device?: string | null, + options: { backend?: DockerGpuPatchBackend } = {}, +): DockerGpuPatchMode { const dockerDevice = normalizeGpuDeviceForDocker(device); if (kind === "gpus") { const gpuValue = dockerDevice === "all" ? "all" : `device=${dockerDevice}`; @@ -306,11 +311,15 @@ export function buildDockerGpuMode(kind: DockerGpuPatchModeKind, device?: string }; } if (kind === "nvidia-runtime") { + const args = ["--runtime", "nvidia", "--env", `NVIDIA_VISIBLE_DEVICES=${dockerDevice}`]; + if (options.backend === "jetson") { + args.push("--env", "NVIDIA_DRIVER_CAPABILITIES=compute,utility"); + } return { kind, label: `--runtime nvidia (NVIDIA_VISIBLE_DEVICES=${dockerDevice})`, device: dockerDevice, - args: ["--runtime", "nvidia", "--env", `NVIDIA_VISIBLE_DEVICES=${dockerDevice}`], + args, }; } const cdiDevice = normalizeGpuDeviceForCdi(device); @@ -324,12 +333,12 @@ export function buildDockerGpuMode(kind: DockerGpuPatchModeKind, device?: string export function buildDockerGpuModeCandidates( device?: string | null, - options: { cdiAvailable?: boolean } = {}, + options: { cdiAvailable?: boolean; backend?: DockerGpuPatchBackend } = {}, ): DockerGpuPatchMode[] { - const candidates = [ - buildDockerGpuMode("gpus", device), - buildDockerGpuMode("nvidia-runtime", device), - ]; + if (options.backend === "jetson") { + return [buildDockerGpuMode("nvidia-runtime", device, { backend: "jetson" })]; + } + const candidates = [buildDockerGpuMode("gpus", device), buildDockerGpuMode("nvidia-runtime", device)]; if (options.cdiAvailable) candidates.push(buildDockerGpuMode("cdi", device)); return candidates; } @@ -685,12 +694,15 @@ function probeDockerGpuMode( } export function selectDockerGpuPatchMode( - options: { image: string; device?: string | null }, + options: { image: string; device?: string | null; backend?: DockerGpuPatchBackend }, deps: DockerGpuPatchDeps = {}, ): { mode: DockerGpuPatchMode | null; attempts: DockerGpuPatchModeAttempt[] } { - const cdiAvailable = dockerReportsNvidiaCdiDevices(deps); + const cdiAvailable = options.backend === "jetson" ? false : dockerReportsNvidiaCdiDevices(deps); const attempts: DockerGpuPatchModeAttempt[] = []; - for (const mode of buildDockerGpuModeCandidates(options.device, { cdiAvailable })) { + for (const mode of buildDockerGpuModeCandidates(options.device, { + cdiAvailable, + backend: options.backend, + })) { const result = probeDockerGpuMode(mode, options.image, deps); const attempt = { mode, ok: result.ok, error: result.error }; attempts.push(attempt); @@ -784,6 +796,7 @@ export function recreateOpenShellDockerSandboxWithGpu( timeoutSecs?: number; waitForSupervisor?: boolean; openshellSandboxCommand?: readonly string[] | null; + backend?: DockerGpuPatchBackend; }, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchResult { @@ -807,13 +820,17 @@ export function recreateOpenShellDockerSandboxWithGpu( if (!image) throw new Error("OpenShell sandbox container inspect did not include an image."); const selection = selectDockerGpuPatchMode( - { image, device: options.gpuDevice }, + { image, device: options.gpuDevice, backend: options.backend }, deps, ); context.modeAttempts = selection.attempts; context.selectedMode = selection.mode; if (!selection.mode) { - throw new Error("Docker did not accept --gpus, NVIDIA runtime, or CDI GPU modes."); + const modeMessage = + options.backend === "jetson" + ? "Docker did not accept the Jetson NVIDIA runtime GPU mode." + : "Docker did not accept --gpus, NVIDIA runtime, or CDI GPU modes."; + throw new Error(modeMessage); } const originalName = dockerContainerName(inspect); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index a63538413c8..cb8f657c181 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { + DockerGpuPatchBackend, DockerGpuPatchDeps, DockerGpuPatchMode, DockerGpuPatchResult, @@ -27,12 +28,14 @@ type DockerGpuSandboxCreatePatchOptions = { gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; timeoutSecs: number; + backend?: DockerGpuPatchBackend; deps: DockerGpuSandboxCreateDeps; }; type DockerGpuSandboxConfig = { sandboxGpuEnabled: boolean; sandboxGpuDevice?: string | null; + hostGpuPlatform?: string | null; }; type DockerGpuSandboxCreatePlan = { @@ -61,6 +64,7 @@ export function createDockerGpuSandboxCreatePatch( gpuDevice: options.gpuDevice, openshellSandboxCommand: options.openshellSandboxCommand ?? null, timeoutSecs: options.timeoutSecs, + backend: options.backend, }; return { @@ -145,7 +149,9 @@ export function shouldUseDockerGpuPatchForCreate( }); if (enabled) { options.log?.( - " Docker-driver GPU patch active; creating sandbox first, then recreating the Docker container with GPU access.", + config.hostGpuPlatform === "jetson" + ? " Jetson Docker GPU patch active; creating sandbox first, then recreating the Docker container with NVIDIA runtime GPU access." + : " Docker-driver GPU patch active; creating sandbox first, then recreating the Docker container with GPU access.", ); } return enabled; @@ -158,7 +164,9 @@ export function resolveDockerGpuSandboxCreatePlan( const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, options); const logMessage = config.sandboxGpuEnabled ? useDockerGpuPatch - ? " Docker-driver GPU patch active; allowing /proc writes required by Docker GPU initialization." + ? config.hostGpuPlatform === "jetson" + ? " Jetson sandbox GPU enabled; using NVIDIA Container Runtime instead of CDI/--gpus." + : " Docker-driver GPU patch active; allowing /proc writes required by Docker GPU initialization." : " Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment." : null; return { useDockerGpuPatch, logMessage }; diff --git a/src/lib/onboard/gateway-gpu-passthrough.test.ts b/src/lib/onboard/gateway-gpu-passthrough.test.ts index 67c2c6936cb..5e7d9ab1487 100644 --- a/src/lib/onboard/gateway-gpu-passthrough.test.ts +++ b/src/lib/onboard/gateway-gpu-passthrough.test.ts @@ -141,43 +141,29 @@ describe("gateway GPU passthrough inspection", () => { expect(canRestartCpuOnlyGatewayForGpuIntent(["alpha", "beta"], "alpha", true)).toBe(false); }); - it("aborts unsupported Jetson GPU passthrough before gateway inspection or cleanup", () => { + it("does not categorically abort Jetson GPU passthrough on Docker-driver gateways", () => { vi.mocked(docker.dockerInspect).mockClear(); const stopDashboardForwards = vi.fn(); const retireLegacyGatewayForDockerDriverUpgrade = vi.fn(); const destroyGatewayRuntimeForGpuReuse = vi.fn(); - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { - throw new Error(`exit:${code}`); - }) as never); - - try { - expect(() => - reconcileGatewayGpuReuseForGpuIntent({ - gatewayReuseState: healthy, - gpuPassthrough: true, - gatewayName: "nemoclaw", - currentSandboxName: "jetson-box", - hostGpuPlatform: "jetson", - recreateSandbox: true, - confirmedDockerDriverGateway: false, - stopDashboardForwards, - retireLegacyGatewayForDockerDriverUpgrade, - destroyGatewayRuntimeForGpuReuse, - }), - ).toThrow("exit:1"); - - const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); - expect(message).toContain("Jetson/Tegra sandbox GPU passthrough is not supported"); - expect(message).toContain("--no-gpu"); - expect(message).not.toContain("destroy --yes"); - expect(docker.dockerInspect).not.toHaveBeenCalled(); - expect(stopDashboardForwards).not.toHaveBeenCalled(); - expect(retireLegacyGatewayForDockerDriverUpgrade).not.toHaveBeenCalled(); - expect(destroyGatewayRuntimeForGpuReuse).not.toHaveBeenCalled(); - } finally { - errorSpy.mockRestore(); - exitSpy.mockRestore(); - } + + const result = reconcileGatewayGpuReuseForGpuIntent({ + gatewayReuseState: healthy, + gpuPassthrough: true, + gatewayName: "nemoclaw", + currentSandboxName: "jetson-box", + hostGpuPlatform: "jetson", + recreateSandbox: true, + confirmedDockerDriverGateway: true, + stopDashboardForwards, + retireLegacyGatewayForDockerDriverUpgrade, + destroyGatewayRuntimeForGpuReuse, + }); + + expect(result).toBe(healthy); + expect(docker.dockerInspect).not.toHaveBeenCalled(); + expect(stopDashboardForwards).not.toHaveBeenCalled(); + expect(retireLegacyGatewayForDockerDriverUpgrade).not.toHaveBeenCalled(); + expect(destroyGatewayRuntimeForGpuReuse).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/gateway-gpu-passthrough.ts b/src/lib/onboard/gateway-gpu-passthrough.ts index 4745c723c1b..d083ad60a7f 100644 --- a/src/lib/onboard/gateway-gpu-passthrough.ts +++ b/src/lib/onboard/gateway-gpu-passthrough.ts @@ -134,18 +134,12 @@ export function reconcileGatewayGpuReuseForGpuIntent({ gpuPassthrough, gatewayName, currentSandboxName, - hostGpuPlatform = null, recreateSandbox, confirmedDockerDriverGateway, stopDashboardForwards, retireLegacyGatewayForDockerDriverUpgrade, destroyGatewayRuntimeForGpuReuse, }: GatewayGpuReuseReconcileOptions): GatewayReuseState { - if (gpuPassthrough && hostGpuPlatform === "jetson") { - reportGpuPassthroughRecovery(console.error, () => [], { unsupportedPlatform: "jetson" }); - process.exit(1); - } - if (!shouldInspectLegacyGatewayGpuPassthrough( gatewayReuseState, gpuPassthrough, diff --git a/src/lib/onboard/gpu-recovery.test.ts b/src/lib/onboard/gpu-recovery.test.ts index c0e3d545b4b..9dfa50ffece 100644 --- a/src/lib/onboard/gpu-recovery.test.ts +++ b/src/lib/onboard/gpu-recovery.test.ts @@ -76,14 +76,14 @@ describe("gpuPassthroughRecoveryLines", () => { expect(joined).not.toMatch(/nemoclaw\s{2,}destroy/); }); - it("does not suggest destroy/recreate as sufficient for unsupported Jetson passthrough", () => { + it("does not suggest destroy/recreate as sufficient for a missing Jetson NVIDIA runtime", () => { const lines = gpuPassthroughRecoveryLines(["jetson-box"], { - unsupportedPlatform: "jetson", + missingRuntimePlatform: "jetson", }); const joined = lines.join("\n"); - expect(joined).toContain("Jetson/Tegra sandbox GPU passthrough is not supported"); + expect(joined).toContain("Jetson/Tegra sandbox GPU requires Docker NVIDIA runtime support"); expect(joined).toContain("--no-gpu"); - expect(joined).toContain("NEMOCLAW_SANDBOX_GPU=0"); + expect(joined).toContain("missing NVIDIA runtime"); expect(joined).not.toContain("destroy --yes"); expect(joined).not.toContain("nemoclaw onboard --gpu"); }); @@ -107,16 +107,16 @@ describe("reportGpuPassthroughRecovery", () => { expect(joined).toContain("nemoclaw beta destroy --yes --cleanup-gateway"); }); - it("does not load registered names for unsupported Jetson passthrough", () => { + it("does not load registered names for missing Jetson NVIDIA runtime recovery", () => { const emit = vi.fn(); const loadNames = vi.fn(() => ["jetson-box"]); reportGpuPassthroughRecovery(emit, loadNames, { - unsupportedPlatform: "jetson", + missingRuntimePlatform: "jetson", }); const joined = emit.mock.calls.map((c) => c[0]).join("\n"); expect(loadNames).not.toHaveBeenCalled(); - expect(joined).toContain("Jetson/Tegra sandbox GPU passthrough is not supported"); + expect(joined).toContain("Jetson/Tegra sandbox GPU requires Docker NVIDIA runtime support"); expect(joined).toContain("nemoclaw onboard --no-gpu"); expect(joined).not.toContain("jetson-box"); }); diff --git a/src/lib/onboard/gpu-recovery.ts b/src/lib/onboard/gpu-recovery.ts index 4bc3fff8226..e5f9488591c 100644 --- a/src/lib/onboard/gpu-recovery.ts +++ b/src/lib/onboard/gpu-recovery.ts @@ -14,13 +14,9 @@ */ import * as registry from "../state/registry"; -import { - JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, - JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, -} from "./sandbox-gpu-mode"; export type GpuPassthroughRecoveryOptions = { - unsupportedPlatform?: "jetson" | null; + missingRuntimePlatform?: "jetson" | null; }; /** @@ -41,10 +37,10 @@ export function gpuPassthroughRecoveryLines( names: readonly string[] | null, options: GpuPassthroughRecoveryOptions = {}, ): string[] { - if (options.unsupportedPlatform === "jetson") { + if (options.missingRuntimePlatform === "jetson") { return [ - ` ${JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE}`, - ` ${JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE}`, + " Jetson/Tegra sandbox GPU requires Docker NVIDIA runtime support.", + " Destroying/recreating the sandbox or gateway will not repair a missing NVIDIA runtime.", " Use CPU sandbox mode instead:", " nemoclaw onboard --no-gpu", ]; @@ -112,6 +108,6 @@ export function reportGpuPassthroughRecovery( loadNames: () => string[] = getRegisteredSandboxNamesForGpuRecovery, options: GpuPassthroughRecoveryOptions = {}, ): void { - const names = options.unsupportedPlatform === "jetson" ? [] : loadNames(); + const names = options.missingRuntimePlatform === "jetson" ? [] : loadNames(); for (const line of gpuPassthroughRecoveryLines(names, options)) emit(line); } diff --git a/src/lib/onboard/machine/handlers/preflight.test.ts b/src/lib/onboard/machine/handlers/preflight.test.ts index fa4b859915a..7b46cddb062 100644 --- a/src/lib/onboard/machine/handlers/preflight.test.ts +++ b/src/lib/onboard/machine/handlers/preflight.test.ts @@ -131,11 +131,41 @@ describe("handlePreflightState", () => { expect(harness.deps.assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith( { cdiNvidiaGpuSpecMissing: false }, true, + undefined, ); expect(harness.deps.validateSandboxGpuPreflight).toHaveBeenCalledOnce(); expect(result.resumePreflight).toBe(true); }); + it("passes host GPU platform into the resumed CDI guard", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + const assertCdiNvidiaGpuSpecPresent = vi.fn(); + const harness = createDeps({ + assertCdiNvidiaGpuSpecPresent, + resolveSandboxGpuConfig: vi.fn( + (_gpu: Gpu, opts: { flag: "enable" | "disable" | null; device: string | null | undefined }) => ({ + sandboxGpuEnabled: opts.flag === "enable", + mode: opts.flag === "enable" ? "1" : "0", + sandboxGpuDevice: opts.device, + hostGpuPlatform: "jetson", + }), + ), + }); + + await handlePreflightState({ + ...baseOptions(harness.deps, session), + resume: true, + explicitSandboxGpuFlag: "enable", + }); + + expect(assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith( + { cdiNvidiaGpuSpecMissing: false }, + true, + "jetson", + ); + }); + it("restores saved sandbox GPU intent only when resume has no explicit override", async () => { const session = createSession(); session.steps.preflight.status = "complete"; diff --git a/src/lib/onboard/machine/handlers/preflight.ts b/src/lib/onboard/machine/handlers/preflight.ts index ccecf7860db..04ad297fdc0 100644 --- a/src/lib/onboard/machine/handlers/preflight.ts +++ b/src/lib/onboard/machine/handlers/preflight.ts @@ -13,6 +13,7 @@ export interface PreflightSandboxGpuOverrides { export interface PreflightSandboxGpuConfig { sandboxGpuEnabled: boolean; mode: string; + hostGpuPlatform?: string | null; sandboxGpuDevice?: string | null; errors?: readonly string[]; } @@ -41,7 +42,11 @@ export interface PreflightStateOptions< detectGpu(): Gpu; runPreflight(options: { optedOutGpuPassthrough?: boolean }): Promise; assessHost(): Host; - assertCdiNvidiaGpuSpecPresent(host: Host, optedOutGpuPassthrough: boolean): void; + assertCdiNvidiaGpuSpecPresent( + host: Host, + optedOutGpuPassthrough: boolean, + hostGpuPlatform?: string | null, + ): void; resolveSandboxGpuConfig( gpu: Gpu, options: { flag: PreflightSandboxGpuFlag; device: string | null | undefined }, @@ -114,7 +119,11 @@ export async function handlePreflightState< deps.validateSandboxGpuPreflight(resumeSandboxGpuConfig); const resumeOptedOutGpuPassthrough = noGpu || (!gpuRequested && session?.gpuPassthrough === false) || !resumeSandboxGpuConfig.sandboxGpuEnabled; - deps.assertCdiNvidiaGpuSpecPresent(deps.assessHost(), resumeOptedOutGpuPassthrough); + deps.assertCdiNvidiaGpuSpecPresent( + deps.assessHost(), + resumeOptedOutGpuPassthrough, + resumeSandboxGpuConfig.hostGpuPlatform, + ); } else { await deps.startRecordedStep("preflight"); gpu = await deps.runPreflight({ optedOutGpuPassthrough: noGpu }); diff --git a/src/lib/onboard/sandbox-gpu-mode.test.ts b/src/lib/onboard/sandbox-gpu-mode.test.ts index 95b92776a15..6670fc065cc 100644 --- a/src/lib/onboard/sandbox-gpu-mode.test.ts +++ b/src/lib/onboard/sandbox-gpu-mode.test.ts @@ -6,8 +6,6 @@ import { describe, expect, it } from "vitest"; import type { GpuDetection } from "../inference/nim"; import { getResumeSandboxGpuOverrides, - JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, - JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, resolveSandboxGpuConfig, } from "./sandbox-gpu-mode"; @@ -86,35 +84,33 @@ describe("sandbox GPU mode helpers", () => { expect(explicitFlagEnable.errors).toEqual([]); }); - it("defaults to CPU sandbox on Jetson unless GPU passthrough is forced", () => { + it("enables sandbox GPU on Jetson without rejecting the platform", () => { const jetson = gpu({ platform: "jetson" }); - expect(resolveSandboxGpuConfig(jetson, { env: {} }).sandboxGpuEnabled).toBe(false); - expect(resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU: "auto" } }).mode).toBe( - "0", - ); + const auto = resolveSandboxGpuConfig(jetson, { env: {} }); + expect(auto.mode).toBe("auto"); + expect(auto.sandboxGpuEnabled).toBe(true); + expect(auto.hostGpuPlatform).toBe("jetson"); + + const envAuto = resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU: "auto" } }); + expect(envAuto.mode).toBe("auto"); + expect(envAuto.sandboxGpuEnabled).toBe(true); + const jetsonDeviceOnly = resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU_DEVICE: "nvidia.com/gpu=0" }, }); - expect(jetsonDeviceOnly.sandboxGpuEnabled).toBe(false); + expect(jetsonDeviceOnly.sandboxGpuEnabled).toBe(true); expect(jetsonDeviceOnly.sandboxGpuDevice).toBeNull(); expect(jetsonDeviceOnly.errors.join("\n")).toContain("requires sandbox GPU mode 1"); + const jetsonExplicitEnable = resolveSandboxGpuConfig(jetson, { env: { NEMOCLAW_SANDBOX_GPU: "1", NEMOCLAW_SANDBOX_GPU_DEVICE: "nvidia.com/gpu=0" }, }); - expect(jetsonExplicitEnable.errors.join("\n")).toContain( - JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, - ); - expect(jetsonExplicitEnable.errors.join("\n")).toContain( - JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, - ); + expect(jetsonExplicitEnable.errors).toEqual([]); + expect(jetsonExplicitEnable.sandboxGpuDevice).toBe("nvidia.com/gpu=0"); + const jetsonFlagEnable = resolveSandboxGpuConfig(jetson, { flag: "enable", env: {} }); expect(jetsonFlagEnable.mode).toBe("1"); - expect(jetsonFlagEnable.errors.join("\n")).toContain( - JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE, - ); - expect(jetsonFlagEnable.errors.join("\n")).toContain( - JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE, - ); + expect(jetsonFlagEnable.errors).toEqual([]); }); it("resumes sandbox GPU auto mode without turning CPU fallback into explicit opt-out", () => { diff --git a/src/lib/onboard/sandbox-gpu-mode.ts b/src/lib/onboard/sandbox-gpu-mode.ts index 845ba7ad135..dc6034e3018 100644 --- a/src/lib/onboard/sandbox-gpu-mode.ts +++ b/src/lib/onboard/sandbox-gpu-mode.ts @@ -9,16 +9,12 @@ export type SandboxGpuFlag = "enable" | "disable" | null; export type SandboxGpuConfig = { mode: SandboxGpuMode; hostGpuDetected: boolean; + hostGpuPlatform: GpuDetection["platform"] | null; sandboxGpuEnabled: boolean; sandboxGpuDevice: string | null; errors: string[]; }; -export const JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE = - "Jetson/Tegra sandbox GPU passthrough is not supported by NemoClaw/OpenShell."; -export const JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE = - "Destroying/recreating the sandbox or gateway will not enable it; re-run with --no-gpu or NEMOCLAW_SANDBOX_GPU=0."; - export type ResumeSandboxGpuOverrides = { flag: SandboxGpuFlag; device: string | null; @@ -45,11 +41,6 @@ export function resolveSandboxGpuMode(args: { flag?: SandboxGpuFlag; }): SandboxGpuMode { let mode: SandboxGpuMode = args.envMode ?? "auto"; - // GPU sandbox passthrough is not supported on Jetson/Tegra; keep auto/default - // behavior on the CPU sandbox path unless the user explicitly forces GPU. - if (args.gpu?.platform === "jetson" && (args.envMode === null || args.envMode === "auto")) { - mode = "0"; - } if (args.flag === "enable") mode = "1"; if (args.flag === "disable") mode = "0"; return mode; @@ -86,14 +77,11 @@ export function resolveSandboxGpuConfig( if (mode === "1" && !hostGpuDetected) { errors.push("Sandbox GPU was requested, but no NVIDIA GPU was detected on the host."); } - if (mode === "1" && gpu?.platform === "jetson") { - errors.push(JETSON_SANDBOX_GPU_UNSUPPORTED_MESSAGE); - errors.push(JETSON_SANDBOX_GPU_WORKAROUND_MESSAGE); - } return { mode, hostGpuDetected, + hostGpuPlatform: gpu?.platform ?? null, sandboxGpuEnabled: mode === "1" || (mode === "auto" && hostGpuDetected), sandboxGpuDevice, errors, diff --git a/src/lib/onboard/sandbox-gpu-preflight.test.ts b/src/lib/onboard/sandbox-gpu-preflight.test.ts new file mode 100644 index 00000000000..e1f71f4c7ec --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-preflight.test.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +vi.mock("../adapters/docker", () => ({ + dockerInfoFormat: vi.fn(), +})); + +import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; +import { + dockerNvidiaRuntimeAvailable, + formatSandboxGpuPassthroughNote, + parseDockerRuntimeNames, + validateSandboxGpuPreflight, +} from "./sandbox-gpu-preflight"; + +function sandboxGpuConfig(overrides: Partial = {}): SandboxGpuConfig { + return { + mode: "auto", + hostGpuDetected: true, + hostGpuPlatform: "linux", + sandboxGpuEnabled: true, + sandboxGpuDevice: null, + errors: [], + ...overrides, + }; +} + +describe("sandbox GPU preflight", () => { + it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { + expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( + "Docker NVIDIA runtime", + ); + expect( + formatSandboxGpuPassthroughNote({ + resumeHasResolvedGpuIntent: true, + recordedGpuPassthroughBeforePreflight: true, + }), + ).toContain("Continuing GPU passthrough"); + expect(formatSandboxGpuPassthroughNote({ requestedGpuPassthrough: true })).toContain( + "GPU passthrough requested", + ); + }); + + it("parses Docker runtime names from JSON and plain-text output", () => { + expect(parseDockerRuntimeNames('{"io.containerd.runc.v2":{},"nvidia":{}}')).toContain( + "nvidia", + ); + expect(parseDockerRuntimeNames("runc nvidia io.containerd.runc.v2")).toContain("nvidia"); + expect(parseDockerRuntimeNames("")).toEqual([]); + }); + + it("checks Jetson sandbox GPU support through Docker NVIDIA runtime availability", () => { + const dockerInfo = vi.fn(() => '{"runc":{},"nvidia":{}}'); + expect(dockerNvidiaRuntimeAvailable({ dockerInfoFormat: dockerInfo })).toBe(true); + + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig({ hostGpuPlatform: "jetson" }), { + platform: "linux", + dockerInfoFormat: dockerInfo, + getDockerCdiSpecDirs: vi.fn(() => { + throw new Error("Jetson preflight must not require CDI"); + }), + findReadableNvidiaCdiSpecFiles: vi.fn(() => { + throw new Error("Jetson preflight must not inspect CDI specs"); + }), + }), + ).not.toThrow(); + expect(dockerInfo).toHaveBeenCalledWith( + "{{json .Runtimes}}", + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("keeps generic Linux sandbox GPU preflight on the CDI path", () => { + const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]); + const findReadableNvidiaCdiSpecFiles = vi.fn(() => ["/etc/cdi/nvidia.yaml"]); + const dockerInfo = vi.fn(() => '{"runc":{},"nvidia":{}}'); + + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig(), { + platform: "linux", + dockerInfoFormat: dockerInfo, + getDockerCdiSpecDirs, + findReadableNvidiaCdiSpecFiles, + }), + ).not.toThrow(); + expect(getDockerCdiSpecDirs).toHaveBeenCalled(); + expect(findReadableNvidiaCdiSpecFiles).toHaveBeenCalledWith(["/etc/cdi"]); + expect(dockerInfo).not.toHaveBeenCalled(); + }); + + it("exits with an explicit Jetson NVIDIA runtime message when runtime support is missing", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number | string | null) => { + throw new Error(`exit:${code}`); + }) as never); + + try { + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig({ hostGpuPlatform: "jetson" }), { + platform: "linux", + dockerInfoFormat: vi.fn(() => '{"runc":{}}'), + }), + ).toThrow("exit:1"); + const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); + expect(message).toContain("Docker NVIDIA runtime was not detected"); + expect(message).toContain("NVIDIA Container Runtime semantics, not CDI"); + expect(message).toContain("nvidia-ctk runtime configure --runtime=docker"); + } finally { + errorSpy.mockRestore(); + exitSpy.mockRestore(); + } + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index d386303368a..2778b1fb3bc 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -1,9 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { dockerInfoFormat } from "../adapters/docker"; import { findReadableNvidiaCdiSpecFiles, getDockerCdiSpecDirs } from "./docker-cdi"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; +const SANDBOX_GPU_PREFLIGHT_TIMEOUT_MS = 30_000; + +export type SandboxGpuPreflightDeps = { + platform?: NodeJS.Platform; + dockerInfoFormat?: (format: string, opts?: Record) => string; + getDockerCdiSpecDirs?: () => string[]; + findReadableNvidiaCdiSpecFiles?: (dirs: string[]) => string[]; +}; + export function sandboxGpuRemediationLines(): string[] { return [ "Install/configure NVIDIA Container Toolkit CDI, then restart Docker:", @@ -21,13 +31,92 @@ export function exitOnSandboxGpuConfigErrors(config: SandboxGpuConfig): void { } } -export function validateSandboxGpuPreflight(config: SandboxGpuConfig): void { +export function formatSandboxGpuPassthroughNote(options: { + hostGpuPlatform?: string | null; + resumeHasResolvedGpuIntent?: boolean; + recordedGpuPassthroughBeforePreflight?: boolean; + requestedGpuPassthrough?: boolean; + sandboxGpuMode?: string | null; +}): string { + if (options.hostGpuPlatform === "jetson") { + return " NVIDIA Jetson/Tegra GPU detected; enabling sandbox GPU through Docker NVIDIA runtime. Use --no-gpu to opt out."; + } + if (options.resumeHasResolvedGpuIntent && options.recordedGpuPassthroughBeforePreflight) { + return " [resume] Continuing GPU passthrough from the saved onboarding session."; + } + if (options.requestedGpuPassthrough || options.sandboxGpuMode === "1") { + return " GPU passthrough requested; passing --gpu to OpenShell gateway and sandbox creation."; + } + return " NVIDIA GPU detected; enabling OpenShell GPU passthrough. Use --no-gpu to opt out."; +} + +export function parseDockerRuntimeNames(value: string | null | undefined): string[] { + const raw = String(value || "").trim(); + if (!raw || raw === "") return []; + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + return parsed.map((entry) => String(entry || "").trim()).filter(Boolean); + } + if (parsed && typeof parsed === "object") { + return Object.keys(parsed) + .map((entry) => entry.trim()) + .filter(Boolean); + } + } catch { + // Fall through to the plain-text parser below. + } + return raw + .split(/[\s,{}":]+/) + .map((entry) => entry.trim()) + .filter(Boolean); +} + +export function dockerNvidiaRuntimeAvailable(deps: SandboxGpuPreflightDeps = {}): boolean { + const dockerInfo = deps.dockerInfoFormat ?? dockerInfoFormat; + try { + const runtimeOutput = dockerInfo("{{json .Runtimes}}", { + ignoreError: true, + timeout: SANDBOX_GPU_PREFLIGHT_TIMEOUT_MS, + }); + return parseDockerRuntimeNames(runtimeOutput).includes("nvidia"); + } catch { + return false; + } +} + +function validateJetsonSandboxGpuPreflight(deps: SandboxGpuPreflightDeps): void { + if (!dockerNvidiaRuntimeAvailable(deps)) { + console.error(""); + console.error(" ✗ Docker NVIDIA runtime was not detected for Jetson/Tegra sandbox GPU."); + console.error(" Jetson sandbox GPU uses NVIDIA Container Runtime semantics, not CDI."); + console.error(" Install/configure NVIDIA Container Toolkit for Docker, then restart Docker:"); + console.error(" sudo nvidia-ctk runtime configure --runtime=docker"); + console.error(" sudo systemctl restart docker"); + console.error(" Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0."); + process.exit(1); + } + console.log(" ✓ Docker NVIDIA runtime detected for Jetson/Tegra sandbox GPU"); +} + +export function validateSandboxGpuPreflight( + config: SandboxGpuConfig, + deps: SandboxGpuPreflightDeps = {}, +): void { exitOnSandboxGpuConfigErrors(config); if (!config.sandboxGpuEnabled) return; - if (process.platform !== "linux") return; + const platform = deps.platform ?? process.platform; + if (platform !== "linux") return; + + if (config.hostGpuPlatform === "jetson") { + validateJetsonSandboxGpuPreflight(deps); + return; + } - const cdiSpecDirs = getDockerCdiSpecDirs(); - const cdiSpecFiles = findReadableNvidiaCdiSpecFiles(cdiSpecDirs); + const cdiSpecDirs = (deps.getDockerCdiSpecDirs ?? getDockerCdiSpecDirs)(); + const cdiSpecFiles = (deps.findReadableNvidiaCdiSpecFiles ?? findReadableNvidiaCdiSpecFiles)( + cdiSpecDirs, + ); if (cdiSpecFiles.length === 0) { console.error(""); console.error(" ✗ Docker CDI GPU support was not detected.");