diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 271b17923f4..e27672ff29e 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -266,7 +266,7 @@ { "name": "Windows-on-ARM GPU passthrough", "status": "unsupported", - "notes": "Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:188`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565." + "notes": "Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:214`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565." }, { "name": "Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal)", diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index c4d20a3dd80..e3352911d77 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -887,6 +887,13 @@ $$nemoclaw onboard --from ./Dockerfile.custom When `$$nemoclaw onboard` detects an NVIDIA GPU on the host, it enables OpenShell GPU passthrough at both the gateway and sandbox level by default. Detection proceeds along two paths. The `nvidia-smi`-based paths (the primary `--query-gpu=name,memory.total,memory.free` probe and the unified-memory `--query-gpu=name` fallback) require `nvidia-smi` to succeed and, on hosts whose firmware does not classify as a known NVIDIA platform (DGX Spark, DGX Station, Jetson, or Tegra), additionally require that the GPU name does not match the placeholder family observed on the Windows-on-ARM WSL2 nvidia-smi shim (`JMJWOA-Generic-*`) and that either the host is not ARM64 Linux (the observed shim is Windows-on-ARM only) or the NVIDIA kernel driver is bound (`/proc/driver/nvidia/` present), so that placeholder shims on non-NVIDIA hardware are not mistaken for real GPUs. +NemoClaw treats a recognized NVIDIA product model from `/sys/class/dmi/id/product_name` or `/sys/firmware/devicetree/base/model`, or a known Tegra device node, as authoritative platform identity. +Remove the native ARM64 proof exception when every supported host that uses it exposes a stable value through one of these signals and `detectNvidiaPlatform()` recognizes that value. +On a native or Docker Desktop-backed WSL ARM64 Linux host without firmware-confirmed NVIDIA platform metadata, the primary memory-query probe can clear a `JMJWOA-Generic-*` name only after a bounded CUDA workload succeeds. +The workload runs `docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample@sha256:7c7540bdf1f942d4fb6db97069fd6c289471b54ac29e3c7fcdf914cf77af7d41`. +Docker may pull the CUDA sample image from `nvcr.io` and keeps the image in the local cache after the container exits. +The names-only unified-memory fallback does not run this workload and rejects denylisted names. +Other non-firmware-vouched hosts also reject denylisted names. Jetson/Tegra hosts that ship without `nvidia-smi` continue to be detected via the devicetree firmware fallback (`/sys/firmware/devicetree/base/model`) or the Tegra device-node fallback (`/dev/nvhost-gpu`, `/dev/nvhost-ctrl-gpu`, `/dev/nvhost-ctrl`, or `/dev/nvmap`); both bypass the trust-tier gate above. Use `--no-gpu` to opt out when you want host-side inference providers only and do not need direct GPU access inside the sandbox. Use `--gpu` to require GPU passthrough and fail fast if an NVIDIA GPU is not detected. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 502553460f7..b090fc7eba2 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -169,7 +169,7 @@ They are listed here so launch material, sales conversations, and support triage | Non-Ubuntu/Debian Linux distros | Unsupported | Installer assumes `apt-get`. Fedora/Rocky/Alma/Arch/NixOS are not validated and the installer's package-manager probes do not cover them. See open issue #899 (Fedora hang). | | Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | | Air-gapped / offline installs | Unsupported | Onboard assumes network reachability for package fetches, container pulls, and provider validation. See open issues #4872 and #2218 (production-deployment epic covering air-gapped support, China network guidance, multi-host topology). | -| Windows-on-ARM GPU passthrough | Unsupported | Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:188`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565. | +| Windows-on-ARM GPU passthrough | Unsupported | Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:214`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565. | | Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts`). NemoClaw does not install non-NVIDIA accelerator drivers. | | Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses | Unsupported | LangChain Deep Agents Code is the only integrated LangChain-family agent runtime (refer to the Agents section above; status `Tested`). Other LangChain agent runtimes, AutoGen, CrewAI, and any agent runtime not listed in the Agents table are not integrated. Bringing more agent runtimes is tracked as a research epic (refer to open issue #4861) but is not on the current roadmap. | | Multi-user host sharing | Unsupported | Sandboxes are scoped to a single host user. NemoClaw treats multi-user hosts as a risk and warns at onboard; see `docs/security/openclaw-controls.mdx` Multi-user detection. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 8068c3ab5c2..cb54077961f 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -831,8 +831,27 @@ The wizard cleans up stale port forwards and waits for gateway readiness automat ### GPU setup fails with a placeholder GPU name -On Windows or WSL hosts, some systems report a placeholder display adapter name even when no NVIDIA GPU firmware is present. +On Windows, WSL, and native Linux ARM64 hosts, some systems report a placeholder display adapter name even when no NVIDIA GPU firmware is present. NVIDIA NIM and GPU-backed sandbox setup require a real NVIDIA GPU. + +When the primary memory-query probe reports a placeholder name on a native or Docker Desktop-backed WSL ARM64 Linux host without firmware-confirmed NVIDIA platform metadata, onboarding runs one bounded Docker CUDA workload. +NemoClaw treats a recognized NVIDIA product model from `/sys/class/dmi/id/product_name` or `/sys/firmware/devicetree/base/model`, or a known Tegra device node, as authoritative platform identity. +Remove the native ARM64 proof exception when every supported host that uses it exposes a stable value through one of these signals and `detectNvidiaPlatform()` recognizes that value. +Docker may pull the CUDA sample image from `nvcr.io` and keeps the image in the local cache after the container exits. +The workload uses this command: + +```bash +docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample@sha256:7c7540bdf1f942d4fb6db97069fd6c289471b54ac29e3c7fcdf914cf77af7d41 +``` + +The run is bounded to 3 minutes. +Set `NEMOCLAW_WSL_GPU_PROOF_TIMEOUT_MS` to a positive millisecond value to change that bound. +Increase the value on slow links. +A passing workload qualifies the detected GPU for GPU passthrough. +A failed or timed-out workload leaves the GPU unproven and does not enable GPU passthrough. +The names-only unified-memory fallback does not run this workload and rejects denylisted names. +WSL hosts that are not Docker Desktop-backed do not run the workload and continue to report the GPU as unavailable. + If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers. Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route. @@ -2712,7 +2731,7 @@ nvidia-ctk cdi list On WSL with Docker Desktop, confirm Docker Desktop WSL integration is enabled for your distro and verify Docker GPU access from WSL: ```bash -docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benchmark +docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample@sha256:7c7540bdf1f942d4fb6db97069fd6c289471b54ac29e3c7fcdf914cf77af7d41 ``` If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu` instead. diff --git a/src/lib/inference/gpu-trust.ts b/src/lib/inference/gpu-trust.ts index f91cf27e755..1df768f0019 100644 --- a/src/lib/inference/gpu-trust.ts +++ b/src/lib/inference/gpu-trust.ts @@ -24,6 +24,20 @@ const NVIDIA_DRIVER_PROC_PATH = "/proc/driver/nvidia"; export function isDenylistedNvidiaGpuName(name: string): boolean { return NVIDIA_GPU_NAME_DENYLIST_PATTERN.test(name); } +/** Escape untrusted GPU names without allowing terminal-control sequences. */ +export function escapeGpuNameForTerminal(value: string): string { + return [...value] + .map((character) => { + const codePoint = character.codePointAt(0) ?? 0; + const isC0 = codePoint <= 0x1f; + const isDeleteOrC1 = codePoint >= 0x7f && codePoint <= 0x9f; + const isLineSeparator = codePoint === 0x2028 || codePoint === 0x2029; + const isFormatControl = /^\p{Cf}$/u.test(character); + if (!isC0 && !isDeleteOrC1 && !isLineSeparator && !isFormatControl) return character; + return "\\u{" + codePoint.toString(16).padStart(4, "0") + "}"; + }) + .join(""); +} // Result of a bounded Docker `--gpus` CUDA proof. `passed` is true only when a // real CUDA workload (not just nvidia-smi) succeeded — that is the signal that @@ -39,7 +53,8 @@ export interface DockerGpuProofResult { // Optional accept-path used by `detectGpu()` when an ARM64 Linux host reports a // denylisted `JMJWOA-Generic-*` placeholder. The prover returns `null` when the -// host is not a proof candidate (not ARM64 WSL Docker Desktop), preserving the +// host is not a proof candidate (not ARM64 Linux that is native or Docker +// Desktop-backed WSL, #8096), preserving the // #3988 fail-closed default; otherwise it returns the bounded Docker GPU proof // outcome so a passing real GPU can be trusted without trusting the name alone. export type Arm64WslDockerDesktopGpuProver = (gpuNames: string[]) => DockerGpuProofResult | null; diff --git a/src/lib/inference/nim.test.ts b/src/lib/inference/nim.test.ts index ee2cc913786..c239278489f 100644 --- a/src/lib/inference/nim.test.ts +++ b/src/lib/inference/nim.test.ts @@ -434,6 +434,13 @@ describe("nim", () => { }); describe("detectGpu", () => { + const proveArm64WslDockerDesktopGpu = vi.fn(() => ({ + passed: true, + timedOut: false, + exitCode: 0, + diagnostic: "", + })); + function withGenericLinuxFirmware(fn: () => void): void { const fs = require("fs"); const origReadFileSync = fs.readFileSync; @@ -684,7 +691,7 @@ describe("nim", () => { try { withFirmwareModel("Microsoft Corporation Virtual Machine", () => { - expect(nimModule.detectGpu()).toBeNull(); + expect(nimModule.detectGpu({ proveArm64WslDockerDesktopGpu: null })).toBeNull(); }); } finally { restore(); @@ -707,55 +714,48 @@ describe("nim", () => { try { withFirmwareModel("Microsoft Corporation Virtual Machine", () => { - expect(nimModule.detectGpu()).toBeNull(); + expect(nimModule.detectGpu({ proveArm64WslDockerDesktopGpu })).toBeNull(); }); } finally { restore(); } }); - // #4565: a real Windows-ARM N1X + WSL2 + Docker Desktop host reports the - // same `JMJWOA-Generic-*` placeholder as the Snapdragon shim, but it can - // pass a bounded Docker `--gpus` CUDA proof. When the injected prover - // confirms the proof, the denylisted name is accepted and the detection is - // tagged so the sandbox preflight reaches the Docker Desktop WSL branch. - it("accepts a denylisted ARM64 GPU when the bounded Docker GPU proof passes (#4565)", () => { + // A passing proof accepts the raw name, but terminal formatting must escape it. + it("escapes terminal controls after a denylisted ARM64 GPU proof passes (#4565)", () => { + const gpuName = "JMJWOA-Generic-\u001b\u0085\u200d\u2028\u2029GPU"; const runCapture = vi.fn((cmd: string | string[]) => { if (!Array.isArray(cmd)) throw new Error("expected argv array"); if (cmd[0] === "nvidia-smi" && cmd.some((a: string) => a.includes("name,memory.total"))) { - return "JMJWOA-Generic-GPU, 65471, 65000\n"; + return `${gpuName}, 65471, 65000\n`; } return ""; }); const { nimModule, restore } = loadNimWithMockedRunner(runCapture); - const proveArm64WslDockerDesktopGpu = vi.fn(() => ({ - passed: true, - timedOut: false, - exitCode: 0, - diagnostic: "", - })); - try { withFirmwareModel("Microsoft Corporation Virtual Machine", () => { const result = nimModule.detectGpu({ proveArm64WslDockerDesktopGpu }); expect(result).toMatchObject({ type: "nvidia", - name: "JMJWOA-Generic-GPU", + name: gpuName, count: 1, totalMemoryMB: 65471, wslDockerDesktopGpuProofPassed: true, }); - expect(proveArm64WslDockerDesktopGpu).toHaveBeenCalledWith(["JMJWOA-Generic-GPU"]); + expect(proveArm64WslDockerDesktopGpu).toHaveBeenCalledWith([gpuName]); + expect(nimModule.formatNvidiaGpuPreflightLines(result)).toEqual([ + "NVIDIA GPU detected (JMJWOA-Generic-\\u{001b}\\u{0085}\\u{200d}\\u{2028}\\u{2029}GPU, 65471 MB)", + ]); }); } finally { restore(); } }); - // Snapdragon WoA fail-closed: the same placeholder name, but the bounded - // CUDA proof fails because there is no usable NVIDIA device. The detection - // must stay null so #3988/#4424 is not reopened. - it("keeps rejecting a denylisted ARM64 GPU when the Docker GPU proof fails (#4565/#3988)", () => { + // A generic ARM64 host can be native Linux or Windows on ARM with WSL2. + // When the bounded CUDA proof fails, public detection must return null so + // the native Linux path does not reopen #3988/#4424. + it("rejects a denylisted generic ARM64 GPU when the Docker proof fails (#4565/#8096/#3988)", () => { const runCapture = vi.fn((cmd: string | string[]) => { if (!Array.isArray(cmd)) throw new Error("expected argv array"); if (cmd[0] === "nvidia-smi" && cmd.some((a: string) => a.includes("name,memory.total"))) { diff --git a/src/lib/inference/nim.ts b/src/lib/inference/nim.ts index be99ae7a74a..0b055dc9aa7 100644 --- a/src/lib/inference/nim.ts +++ b/src/lib/inference/nim.ts @@ -26,6 +26,7 @@ import { isSafeModelId } from "../validation"; import { isDgxStationGb300Product } from "./dgx-station-identity"; import { type Arm64WslDockerDesktopGpuProver, + escapeGpuNameForTerminal, isDenylistedNvidiaGpuName, isPlausibleNvidiaGpuName, nvidiaHostLooksGenuine, @@ -84,27 +85,27 @@ export interface GpuDetection { // skips `computeIntensive` registry entries on these hosts. computeConstrained?: boolean; // Set when a denylisted `JMJWOA-Generic-*` placeholder name was accepted only - // because a bounded Docker `--gpus` CUDA proof passed (Windows-ARM N1X + WSL2 - // + Docker Desktop, #4565). Diagnostic marker that this detection cleared a - // live proof rather than firmware/name trust. The sandbox GPU preflight still - // reaches the Docker Desktop WSL compatibility branch via its own - // `detectWslDockerDesktopStatus()` check (consistent because the proof itself - // requires Docker Desktop WSL); this flag does not gate that branch. + // because a bounded Docker `--gpus` CUDA proof passed, on Windows-ARM N1X + // (WSL2 + Docker Desktop, #4565) or native ARM64 Linux (#8096). Diagnostic + // marker that this detection cleared a live proof rather than firmware/name + // trust. The sandbox GPU preflight reaches the Docker Desktop WSL + // compatibility branch via its own `detectWslDockerDesktopStatus()` check, + // which stays false on a native Linux host; this flag does not gate that + // branch. wslDockerDesktopGpuProofPassed?: boolean; } export interface DetectGpuDeps { - // Optional accept-path for ARM64 WSL Docker Desktop `JMJWOA-Generic-*` GPUs - // (#4565). Injected in tests; in production `detectGpu()` lazily builds the - // default prover from the onboard WSL Docker Desktop module only when it is - // about to reject a denylisted ARM64 name. + // Optional accept-path for native or Docker Desktop-backed WSL ARM64 Linux + // hosts that report a `JMJWOA-Generic-*` GPU (#4565/#8096). Injected in tests; + // in production `detectGpu()` lazily builds the default prover only when it + // is about to reject a denylisted ARM64 name. proveArm64WslDockerDesktopGpu?: Arm64WslDockerDesktopGpuProver | null; } -// Lazily construct the default ARM64 WSL Docker Desktop GPU prover. Kept lazy -// (and behind a require) so the inference layer does not statically depend on -// the onboard layer, and so the bounded Docker proof is only wired when we -// actually reach the denylist-reject path on an ARM64 host. +// Lazily construct the default ARM64 Linux GPU prover. Keep it behind a require +// so the inference layer does not statically depend on the onboard layer. The +// bounded Docker proof is wired only at the denylist-reject path. function defaultArm64WslDockerDesktopGpuProver(): Arm64WslDockerDesktopGpuProver | null { try { return require("../onboard/wsl-docker-desktop-gpu").createArm64WslDockerDesktopGpuProver(); @@ -164,7 +165,8 @@ export function groupGpusByName(gpus: readonly NimGpu[]): GpuGroup[] { // See #2669 for the multi-GPU case the previous fix missed. export function formatNvidiaGpuPreflightLines(gpu: GpuDetection): string[] { if (gpu.name) { - const detail = gpu.count > 1 ? `${gpu.count}x ${gpu.name}` : gpu.name; + const name = escapeGpuNameForTerminal(gpu.name); + const detail = gpu.count > 1 ? `${gpu.count}x ${name}` : name; return [`NVIDIA GPU detected (${detail}, ${gpu.totalMemoryMB} MB)`]; } if (gpu.gpus && gpu.gpus.length > 0) { @@ -174,7 +176,7 @@ export function formatNvidiaGpuPreflightLines(gpu: GpuDetection): string[] { const anyDuplicate = groups.some((grp) => grp.count > 1); for (const grp of groups) { const prefix = anyDuplicate ? `${grp.count}x ` : ""; - lines.push(` - ${prefix}${grp.name} (${grp.memoryMB} MB)`); + lines.push(` - ${prefix}${escapeGpuNameForTerminal(grp.name)} (${grp.memoryMB} MB)`); } return lines; } @@ -459,12 +461,18 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { if (firmwareConfirmsNvidia) { trusted = parsed; } else if (parsed.some((p: ParsedGpu) => isDenylistedNvidiaGpuName(p.name))) { + // The all-GPU CUDA workload proves that at least one usable device + // exists. It does not establish which nvidia-smi rows or capacities + // are genuine, so a denylisted multi-row response stays untrusted. + if (parsed.length !== 1) { + return null; + } // A denylisted `JMJWOA-Generic-*` placeholder. Both real Windows-ARM // N1X (WSL2 + Docker Desktop) and the Snapdragon nvidia-smi shim emit // this name, so the name and `/proc/driver/nvidia` are insufficient. - // Give the host one bounded Docker `--gpus` CUDA proof: only the real - // GPU can run the workload, so a pass safely accepts N1X while the - // shim keeps failing closed (#4565 without reopening #3988/#4424). + // A bounded Docker `--gpus` workload proves that the single reported + // row has a usable CUDA device. The Snapdragon shim cannot pass it + // (#4565 without reopening #3988/#4424). const prover = deps.proveArm64WslDockerDesktopGpu === undefined ? defaultArm64WslDockerDesktopGpuProver() @@ -473,13 +481,7 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { if (!proof || !proof.passed) { return null; } - // The proof confirms a usable GPU, but it does not vouch for every - // row. Keep only the placeholder rows it covers plus any plausibly- - // named NVIDIA rows; drop unrecognized garbage so a mixed-row spoof - // cannot inflate totalMemoryMB with a phantom device. - trusted = parsed.filter( - (p: ParsedGpu) => isDenylistedNvidiaGpuName(p.name) || isPlausibleNvidiaGpuName(p.name), - ); + trusted = parsed; wslDockerDesktopGpuProofPassed = true; } else { if (!nvidiaHostLooksGenuine()) { @@ -510,9 +512,10 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { nimCapable: canRunNimWithMemory(totalMemoryMB), platform, spark: platform === "spark", - // The proof-passed Windows-ARM N1X iGPU is memory-shared like Jetson - // and cannot serve a computeIntensive model in-loop, so tag it + // The proof-passed ARM64 N1X GPU is memory-shared like Jetson and + // cannot serve a computeIntensive model in-loop, so tag it // computeConstrained to exclude those Ollama bootstrap models (#3707). + // This covers the N1X part on WSL2 and on native Linux (#8096). ...(platform === "jetson" || wslDockerDesktopGpuProofPassed ? { computeConstrained: true } : {}), diff --git a/src/lib/onboard/wsl-docker-desktop-gpu.test.ts b/src/lib/onboard/wsl-docker-desktop-gpu.test.ts index 4ae5344041a..3a2b0098bc2 100644 --- a/src/lib/onboard/wsl-docker-desktop-gpu.test.ts +++ b/src/lib/onboard/wsl-docker-desktop-gpu.test.ts @@ -85,17 +85,58 @@ describe("createArm64WslDockerDesktopGpuProver (#4565)", () => { expect(runProof).not.toHaveBeenCalled(); }); - it("returns null when the host is not Docker Desktop-backed WSL", () => { + it("proves a denylisted GPU name on native Linux ARM64 (#8096)", () => { + // A native Linux ARM64 host reports a genuine GPU as `JMJWOA-Generic-GPU`. + // Gating the proof on Docker Desktop left no way to verify that GPU, so + // onboarding reported "no NVIDIA GPU detected" on a host where + // `docker run --gpus all` runs a CUDA workload. const runProof = vi.fn(() => passingProof); const prover = createArm64WslDockerDesktopGpuProver({ platform: "linux", arch: "arm64", - detectWslDockerDesktopStatus: () => "not-docker-desktop", + env: {}, + release: "6.17.0-1029-nvidia", + procVersion: "Linux version 6.17.0-1029-nvidia", runProof, log: () => undefined, }); - expect(prover(["JMJWOA-Generic-GPU"])).toBeNull(); - expect(runProof).not.toHaveBeenCalled(); + expect(prover(["JMJWOA-Generic-GPU"])).toEqual(passingProof); + expect(runProof).toHaveBeenCalledTimes(1); + }); + + it("returns the failed bounded proof result on native Linux ARM64", () => { + // The Snapdragon nvidia-smi shim reaches the same path; only the CUDA + // workload separates it from real hardware (#3988/#4565). + const failingProof = { passed: false, timedOut: false, exitCode: 1, diagnostic: "" }; + const runProof = vi.fn(() => failingProof); + const prover = createArm64WslDockerDesktopGpuProver({ + platform: "linux", + arch: "arm64", + env: {}, + release: "6.17.0-1029-nvidia", + procVersion: "Linux version 6.17.0-1029-nvidia", + runProof, + log: () => undefined, + }); + expect(prover(["JMJWOA-Generic-GPU"])).toEqual(failingProof); + expect(runProof).toHaveBeenCalledTimes(1); + }); + + it("leaves WSL hosts without Docker Desktop unproven (#8096)", () => { + // Windows-on-ARM passthrough scope is unchanged: only native Linux is added. + for (const status of ["not-docker-desktop", "unknown"] as const) { + const runProof = vi.fn(() => passingProof); + const prover = createArm64WslDockerDesktopGpuProver({ + platform: "linux", + arch: "arm64", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + detectWslDockerDesktopStatus: () => status, + runProof, + log: () => undefined, + }); + expect(prover(["JMJWOA-Generic-GPU"])).toBeNull(); + expect(runProof).not.toHaveBeenCalled(); + } }); it("runs the bounded proof and reports the result on ARM64 Docker Desktop WSL", () => { @@ -114,16 +155,36 @@ describe("createArm64WslDockerDesktopGpuProver (#4565)", () => { expect(argv).toContain("--gpus"); }); - it("uses an arch-correct CUDA sample image (not the amd64-only nbody) on this ARM64 path", () => { + it("escapes terminal controls in a denylisted GPU name before logging", () => { + const logs: string[] = []; + const prover = createArm64WslDockerDesktopGpuProver({ + platform: "linux", + arch: "arm64", + env: {}, + release: "6.17.0-1029-nvidia", + procVersion: "Linux version 6.17.0-1029-nvidia", + runProof: () => passingProof, + log: (message) => logs.push(message), + }); + + expect(prover(["JMJWOA-Generic-\u001b[2J\nforged status"])).toEqual(passingProof); + expect(logs[0]).toContain("JMJWOA-Generic-\\u{001b}[2J\\u{000a}forged status"); + expect(logs.every((message) => !/[\u0000-\u001f\u007f-\u009f]/u.test(message))).toBe(true); + }); + + it("uses the approved immutable multi-architecture CUDA sample image on this ARM64 path", () => { // The proof only runs on ARM64, so the image must ship a real aarch64 CUDA // binary. `cuda-sample:nbody` packs an x86-64 binary in its arm64 tag and // fails with `exec format error` on the N1X target (#4565); the chosen // vectorAdd image ships a genuine aarch64 binary. - expect(WSL_DOCKER_DESKTOP_GPU_PROOF_COMMAND).toContain("cuda-sample:vectoradd"); + expect(WSL_DOCKER_DESKTOP_GPU_PROOF_COMMAND).toBe( + "docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample@sha256:7c7540bdf1f942d4fb6db97069fd6c289471b54ac29e3c7fcdf914cf77af7d41", + ); + expect(WSL_DOCKER_DESKTOP_GPU_PROOF_COMMAND).not.toContain("vectoradd-cuda12.5.0"); expect(WSL_DOCKER_DESKTOP_GPU_PROOF_COMMAND).not.toContain("nbody"); }); - it("propagates a failing proof so detection stays fail-closed", () => { + it("returns the failed bounded proof result on Docker Desktop WSL", () => { const failing = { passed: false, timedOut: false, exitCode: 1, diagnostic: "no CUDA device" }; const prover = createArm64WslDockerDesktopGpuProver({ platform: "linux", diff --git a/src/lib/onboard/wsl-docker-desktop-gpu.ts b/src/lib/onboard/wsl-docker-desktop-gpu.ts index c69415cbf96..930afef8120 100644 --- a/src/lib/onboard/wsl-docker-desktop-gpu.ts +++ b/src/lib/onboard/wsl-docker-desktop-gpu.ts @@ -4,26 +4,32 @@ import fs from "node:fs"; import os from "node:os"; import { dockerInfoFormat as defaultDockerInfoFormat } from "../adapters/docker"; -import type { Arm64WslDockerDesktopGpuProver, DockerGpuProofResult } from "../inference/gpu-trust"; +import { + escapeGpuNameForTerminal, + type Arm64WslDockerDesktopGpuProver, + type DockerGpuProofResult, +} from "../inference/gpu-trust"; const WSL_DOCKER_DESKTOP_DETECTION_TIMEOUT_MS = 30_000; // This prover only ever runs on ARM64 (see `createArm64WslDockerDesktopGpuProver`), // so the proof image MUST ship a real aarch64 CUDA binary. The older // `cuda-sample:nbody` image is unusable here: its arm64 manifest entry actually // contains an x86-64 ELF, so on the N1X Windows-ARM target it fails with -// `exec /cuda-samples/sample: exec format error` (#4565). `vectoradd-cuda12.5.0` -// ships a genuine aarch64 binary and runs a real CUDA kernel (device alloc + -// add + result verification), which is a strong usability proof that still -// fails closed on the Snapdragon nvidia-smi shim (no usable CUDA device, #3988). -// The image's entrypoint runs vectorAdd directly, so no trailing args are needed. +// `exec /cuda-samples/sample: exec format error` (#4565). The immutable +// vectorAdd manifest below contains both linux/amd64 and linux/arm64 images; its +// ARM64 image ships a genuine aarch64 binary and runs a real CUDA kernel (device +// alloc + add + result verification), which is a strong usability proof that +// still fails closed on the Snapdragon nvidia-smi shim (no usable CUDA device, +// #3988). The image's entrypoint runs vectorAdd directly, so no trailing args +// are needed. export const WSL_DOCKER_DESKTOP_GPU_PROOF_COMMAND = - "docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0"; + "docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample@sha256:7c7540bdf1f942d4fb6db97069fd6c289471b54ac29e3c7fcdf914cf77af7d41"; // The proof runs a real CUDA workload and may first pull the CUDA sample image, // so it is bounded generously (3 min) rather than with the 30s detection // timeout. Operators on slow links can override via // NEMOCLAW_WSL_GPU_PROOF_TIMEOUT_MS. The timeout is the safety bound that keeps -// onboarding from hanging if Docker Desktop GPU passthrough stalls. +// onboarding from hanging if Docker GPU passthrough stalls. const WSL_DOCKER_DESKTOP_GPU_PROOF_DEFAULT_TIMEOUT_MS = 180_000; export function wslDockerDesktopGpuProofTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { @@ -184,10 +190,10 @@ function runWslDockerDesktopGpuProof(argv: string[], timeoutMs: number): DockerG } } -// Build the ARM64 WSL Docker Desktop GPU prover consumed by `detectGpu()` for -// denylisted `JMJWOA-Generic-*` names (#4565). Returns `null` for any host that -// is not ARM64 Linux on Docker Desktop-backed WSL, so the #3988/#4424 fail- -// closed default is preserved everywhere else. When the host IS a candidate it +// Build the ARM64 GPU prover consumed by `detectGpu()` for denylisted +// `JMJWOA-Generic-*` names (#4565). Returns `null` unless the host is ARM64 +// Linux that is either native or Docker Desktop-backed WSL (#8096), so the +// #3988/#4424 fail-closed default is preserved everywhere else. When the host IS a candidate it // runs one bounded Docker `--gpus` CUDA workload (the aarch64 vectorAdd sample): // a real N1X GPU passes, while the Snapdragon nvidia-smi shim — which has no // usable CUDA device — cannot, so the placeholder name alone is never trusted. @@ -201,20 +207,27 @@ export function createArm64WslDockerDesktopGpuProver( const platform = deps.platform ?? process.platform; const arch = deps.arch ?? process.arch; if (platform !== "linux" || arch !== "arm64") return null; - if (detectStatus(deps) !== "docker-desktop") return null; - const names = gpuNames.filter(Boolean).join(", ") || "generic ARM64 GPU"; - log( - ` Running bounded Docker Desktop WSL GPU proof for ${names} (may pull a CUDA sample image)...`, - ); + // Docker Desktop-backed WSL is not the only host that reports a denylisted + // `JMJWOA-Generic-*` name for a genuine GPU; native Linux ARM64 hosts do + // too (#8096), and previously had no way to prove one. Only that host class + // is added here: a WSL host that is not Docker Desktop-backed still returns + // `null`, so Windows-on-ARM passthrough scope is unchanged. The bounded CUDA + // proof remains the trust boundary, not the container runtime. The Snapdragon + // nvidia-smi shim exposes no usable CUDA device, so it still fails closed + // wherever the proof runs (#3988/#4565). + if (detectWsl(deps) && detectStatus(deps) !== "docker-desktop") return null; + const names = + gpuNames.filter(Boolean).map(escapeGpuNameForTerminal).join(", ") || "generic ARM64 GPU"; + log(` Running bounded Docker GPU proof for ${names} (may pull a CUDA sample image)...`); log(` ${WSL_DOCKER_DESKTOP_GPU_PROOF_COMMAND}`); const result = runProof( wslDockerDesktopGpuProofArgv(), wslDockerDesktopGpuProofTimeoutMs(deps.env), ); if (result.passed) { - log(" ✓ Docker Desktop WSL GPU proof passed; trusting the reported GPU."); + log(" ✓ Docker GPU proof passed; trusting the reported GPU."); } else if (result.timedOut) { - log(" ✗ Docker Desktop WSL GPU proof timed out; treating GPU as unproven (CPU fallback)."); + log(" ✗ Docker GPU proof timed out; treating GPU as unproven (CPU fallback)."); log( " Rerun with --no-gpu to skip GPU passthrough, or raise NEMOCLAW_WSL_GPU_PROOF_TIMEOUT_MS.", ); @@ -222,9 +235,7 @@ export function createArm64WslDockerDesktopGpuProver( // The proof binary's architecture did not match the host. This is an image // problem, not a GPU problem, so call it out explicitly rather than letting // the host fall back to CPU as if no GPU were present (#4565). - log( - " ✗ Docker Desktop WSL GPU proof could not run: CUDA sample image architecture does not", - ); + log(" ✗ Docker GPU proof could not run: CUDA sample image architecture does not"); log( " match this host (exec format error). This is a proof-image issue, not a missing GPU.", ); @@ -232,7 +243,7 @@ export function createArm64WslDockerDesktopGpuProver( " Rerun with --no-gpu to skip GPU passthrough, or report this so the proof image can be fixed.", ); } else { - log(" ✗ Docker Desktop WSL GPU proof failed; treating GPU as unproven (CPU fallback)."); + log(" ✗ Docker GPU proof failed; treating GPU as unproven (CPU fallback)."); log(" Rerun with --no-gpu to skip GPU passthrough."); } return result;