diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 893593d9e93..1812f62438d 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -792,6 +792,29 @@ reviews: - path: "src/commands/sandbox/agents/**" instructions: *e2e-sessions-agents-cli + - path: "test/e2e/test-jetson-nvmap-gpu.sh" + instructions: &e2e-jetson-nvmap | + This script is the reporter-workflow E2E for #4231: on a Jetson Orin + host it onboards with GPU, inspects the sandbox user's groups and + /dev/nvmap, runs the cuInit(0) CUDA proof inside the sandbox, and + asserts `nemoclaw status` reports `(CUDA verified)`. It validates that + the Jetson Docker GPU recreate grants the sandbox user the host group + owning /dev/nvmap so CUDA can initialize. Invisible to unit tests + because it needs real Tegra device nodes and a live sandbox. + + **E2E test recommendation:** + - `gpu-jetson-nvmap-e2e` — Jetson /dev/nvmap CUDA usability + status + proof (Jetson-gated; needs a Jetson/Tegra GPU runner with + `vars.JETSON_E2E_ENABLED=true`) + + To run selectively: + ``` + gh workflow run nightly-e2e.yaml --ref -f jobs=gpu-jetson-nvmap-e2e + ``` + + - path: "src/lib/onboard/docker-gpu-patch.ts" + instructions: *e2e-jetson-nvmap + - path: ".github/workflows/nightly-e2e.yaml" instructions: | This is the nightly E2E workflow definition. Changes here affect diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 023b0b4d9d2..66615e41e62 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -88,6 +88,8 @@ # launchable-smoke-e2e Community install path (brev-launchable-ci-cpu.sh) on ubuntu-latest. # gpu-e2e Local Ollama inference on an NVKS ephemeral GPU runner. # gpu-double-onboard-e2e Ollama proxy token consistency after re-onboard (#2553). +# gpu-jetson-nvmap-e2e Jetson Orin /dev/nvmap CUDA usability + status proof (#4231). +# Gated behind vars.JETSON_E2E_ENABLED; needs a Jetson runner. # concurrent-gateway-ports-e2e # Two sandboxes coexisting on the same host with distinct # NEMOCLAW_GATEWAY_PORT values; verifies per-instance @@ -143,7 +145,7 @@ on: onboard-resume-e2e, onboard-negative-paths-e2e, runtime-overrides-e2e, credential-sanitization-e2e, telegram-injection-e2e, overlayfs-autofix-e2e, device-auth-health-e2e, launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, - concurrent-gateway-ports-e2e + gpu-jetson-nvmap-e2e, concurrent-gateway-ports-e2e required: false type: string default: "" @@ -2107,6 +2109,67 @@ jobs: path: /tmp/nemoclaw-gpu-double-onboard-test.log if-no-files-found: ignore + # ── Jetson nvmap GPU status E2E (#4231) ────────────────────── + # Reproduces the reporter's exact Jetson Orin workflow: onboard with GPU, + # then prove the sandbox user can open /dev/nvmap (CUDA cuInit(0)=0) and that + # `nemoclaw status` reports proven CUDA usability instead of a misleading + # bare "enabled". Requires a Jetson/Tegra (arm64 L4T) GPU runner, which the + # project does not yet host — so this job is gated behind + # `vars.JETSON_E2E_ENABLED` (unset by default → skipped) and an explicit + # runner label. When a Jetson runner is provisioned, set the variable and + # point `runs-on` at its label. The same fix has deterministic, hardware-free + # regression coverage in src/lib/onboard/docker-gpu-patch.test.ts. + gpu-jetson-nvmap-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.JETSON_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',gpu-jetson-nvmap-e2e,')) + runs-on: ${{ vars.JETSON_E2E_RUNNER_LABEL || 'linux-arm64-gpu-jetson-orin-latest-1' }} + timeout-minutes: 40 + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-jetson-nvmap" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_PROVIDER: "ollama" + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Verify Jetson GPU availability + run: | + echo "=== Tegra release ===" + cat /etc/nv_tegra_release 2>/dev/null || echo "(no /etc/nv_tegra_release)" + echo "" + echo "=== /dev/nvmap ===" + ls -l /dev/nvmap 2>/dev/null || echo "(no /dev/nvmap)" + echo "" + echo "=== Docker ===" + docker info --format '{{.ServerVersion}}' + docker info --format '{{json .Runtimes}}' + + - name: Run Jetson nvmap GPU E2E test (#4231) + run: bash test/e2e/test-jetson-nvmap-gpu.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-jetson-nvmap-install-log + path: /tmp/nemoclaw-jetson-nvmap-e2e-install.log + if-no-files-found: ignore + + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-jetson-nvmap-test-log + path: /tmp/nemoclaw-jetson-nvmap-e2e-test.log + if-no-files-found: ignore + concurrent-gateway-ports-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && @@ -2228,6 +2291,7 @@ jobs: launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, concurrent-gateway-ports-e2e, ] if: ${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} @@ -2344,6 +2408,7 @@ jobs: launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, concurrent-gateway-ports-e2e, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} @@ -2519,6 +2584,7 @@ jobs: launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, concurrent-gateway-ports-e2e, ] if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} diff --git a/src/lib/onboard/docker-gpu-patch.test.ts b/src/lib/onboard/docker-gpu-patch.test.ts index cfc21b552c3..7b0ae8891b0 100644 --- a/src/lib/onboard/docker-gpu-patch.test.ts +++ b/src/lib/onboard/docker-gpu-patch.test.ts @@ -17,6 +17,7 @@ import { collectDockerGpuPatchDiagnostics, type DockerContainerInspect, detectSandboxFallbackDns, + detectTegraDeviceGroupGids, dockerReportsNvidiaCdiDevices, formatDockerInspectNetworkSummary, getDockerGpuPatchNetworkMode, @@ -800,6 +801,123 @@ describe("docker-gpu-patch sandbox DNS fallback (#3579)", () => { }); }); +// Jetson `/dev/nvmap` group-permission propagation (#4231). The reporter's +// Jetson Orin sandbox saw the GPU devices mounted but CUDA failed with +// `NvRmMemInitNvmap ... Permission denied` / `cuInit(0)=999` because the +// sandbox user (uid/gid 998) was not in the `video` group that owns +// `/dev/nvmap` (`crw-rw---- root video`). The Jetson recreate must grant that +// group via `--group-add` so CUDA can initialize. +describe("Jetson /dev/nvmap group propagation (#4231)", () => { + it("returns the owning GID(s) of present Tegra device nodes, skipping missing and root-owned", () => { + const deviceGids: Record = { + "/dev/nvmap": 44, // root video + "/dev/nvhost-ctrl": 44, + "/dev/nvhost-gpu": 0, // root root — skipped (root already has access) + "/dev/nvgpu/igpu0/ctrl": 110, // render + // every other Tegra node is absent on this host + }; + const gids = detectTegraDeviceGroupGids({ + statDeviceGid: (p: string) => (p in deviceGids ? deviceGids[p] : null), + }); + // Deduped, sorted numerically, root (0) and missing nodes excluded. + expect(gids).toEqual(["44", "110"]); + }); + + it("returns no GIDs when no Tegra device nodes are present (non-Jetson host)", () => { + expect(detectTegraDeviceGroupGids({ statDeviceGid: () => null })).toEqual([]); + }); + + it("emits --group-add for extraGroupGids and dedupes against existing GroupAdd", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.GroupAdd = ["44"]; // baseline already carries video + const args = buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { extraGroupGids: ["44", "110"] }, + ); + // `44` is added exactly once (baseline + extra deduped); `110` added. + expect(args.filter((arg, i) => args[i - 1] === "--group-add" && arg === "44").length).toBe(1); + expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); + }); + + it("does not add --group-add when extraGroupGids is absent", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.GroupAdd = []; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); + expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); + }); + + it("plumbs detected Tegra device GIDs into the Jetson recreate as --group-add", () => { + const dockerCapture = vi.fn((args: readonly string[]) => { + if (args[0] === "ps") return "old-container-id\n"; + if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); + if (args[0] === "info") return ""; + return ""; + }); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const detectTegraDeviceGroupGidsStub = vi.fn(() => ["44"]); + + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { + dockerCapture, + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: detectTegraDeviceGroupGidsStub, + }, + ); + + expect(detectTegraDeviceGroupGidsStub).toHaveBeenCalled(); + expect(dockerRunDetached).toHaveBeenCalledWith( + expect.arrayContaining(["--group-add", "44"]), + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("does not add Tegra device GIDs for the generic (non-Jetson) backend", () => { + const dockerCapture = vi.fn((args: readonly string[]) => { + if (args[0] === "ps") return "old-container-id\n"; + if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); + if (args[0] === "info") return ""; + return ""; + }); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const detectTegraDeviceGroupGidsStub = vi.fn(() => ["44"]); + + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, backend: "generic" }, + { + dockerCapture, + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: detectTegraDeviceGroupGidsStub, + }, + ); + + // Generic backend never queries Tegra device groups and never emits the + // extra --group-add (inspectFixture has no baseline GroupAdd). + expect(detectTegraDeviceGroupGidsStub).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalledWith( + expect.arrayContaining(["--group-add", "44"]), + expect.anything(), + ); + }); +}); + // Regression coverage for NemoClaw issue #4316: the Docker GPU patch path // must distinguish "sandbox never became executable" (Error phase / dead // container) from "GPU proof failed inside an executable sandbox", and the diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 7667447b28d..f1f3675036e 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -80,6 +80,14 @@ export type DockerGpuPatchDeps = { homedir?: () => string; now?: () => Date; detectSandboxFallbackDns?: () => string | null; + /** + * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes + * (`/dev/nvmap`, `/dev/nvhost-*`). Used by the Jetson recreate to grant the + * sandbox user matching `--group-add` membership so CUDA can open them + * (#4231). Injectable so the Jetson permission path is testable without + * Tegra hardware. + */ + detectTegraDeviceGroupGids?: () => string[]; /** Injectable directory lister for unit testing CDI spec discovery. */ readDir?: (dirPath: string) => string[] | null; /** Injectable file reader for unit testing CDI spec content checks. */ @@ -137,6 +145,14 @@ export type DockerGpuCloneRunOptions = { openshellEndpoint?: string | null; sandboxFallbackDns?: string | null; openshellSandboxCommand?: readonly string[] | null; + /** + * Extra supplementary group IDs to add to the recreated container via + * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU + * device nodes (`/dev/nvmap`, `/dev/nvhost-*`); granting the sandbox user + * membership lets CUDA's nvmap init open them instead of failing with + * `NvRmMemInitNvmap ... Permission denied` (#4231). + */ + extraGroupGids?: readonly string[] | null; }; export type DockerGpuPatchDiagnostics = { @@ -261,6 +277,7 @@ function depsWithDefaults(deps: DockerGpuPatchDeps): Required< | "homedir" | "now" | "detectSandboxFallbackDns" + | "detectTegraDeviceGroupGids" > > & DockerGpuPatchDeps { @@ -278,10 +295,67 @@ function depsWithDefaults(deps: DockerGpuPatchDeps): Required< homedir: os.homedir, now: () => new Date(), detectSandboxFallbackDns: () => detectSandboxFallbackDns(), + detectTegraDeviceGroupGids: () => detectTegraDeviceGroupGids(), ...deps, }; } +// Jetson/Tegra device nodes that CUDA opens during driver initialization. +// `/dev/nvmap` is the memory manager whose `NvRmMemInitNvmap` failure the +// reporter hit (#4231); the `nvhost-*`/`nvgpu` nodes are the compute/control +// channels. On L4T these are owned by a non-root group (typically `video`, +// mode `crw-rw----`). +const TEGRA_GPU_DEVICE_NODES = [ + "/dev/nvmap", + "/dev/nvhost-ctrl", + "/dev/nvhost-ctrl-gpu", + "/dev/nvhost-gpu", + "/dev/nvhost-as-gpu", + "/dev/nvhost-prof-gpu", + "/dev/nvhost-dbg-gpu", + "/dev/nvhost-tsg-gpu", + "/dev/nvgpu/igpu0/ctrl", + "/dev/nvgpu/igpu0/as", + "/dev/nvgpu/igpu0/prof", +] as const; + +/** + * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes. + * + * The NVIDIA Container Runtime bind-mounts these nodes into the sandbox + * preserving the host's numeric owner/group, but the OpenShell sandbox runs + * the agent as an unprivileged user that is not a member of that group — so + * CUDA's nvmap init fails with `Permission denied` and `cuInit(0)` returns 999 + * even though the devices are present (#4231). Returning the owning GID(s) + * lets the recreate grant the sandbox user matching `--group-add` membership. + * + * Numeric GIDs (not group names) are returned on purpose: the sandbox image's + * group database need not define a `video`/`render` group at the host's GID, + * and `docker run --group-add ` adds the supplementary group by ID + * regardless of whether a matching name exists inside the container. + */ +export function detectTegraDeviceGroupGids( + deps: { statDeviceGid?: (path: string) => number | null } = {}, +): string[] { + const statGid = + deps.statDeviceGid ?? + ((p: string): number | null => { + try { + return fs.statSync(p).gid; + } catch { + return null; + } + }); + const gids = new Set(); + for (const node of TEGRA_GPU_DEVICE_NODES) { + const gid = statGid(node); + // Skip missing nodes and root-owned (gid 0) nodes: `--group-add 0` would + // not help an unprivileged user, and root already has access regardless. + if (gid !== null && gid > 0) gids.add(String(gid)); + } + return [...gids].sort((a, b) => Number(a) - Number(b)); +} + function resultText(result: DockerRunResult | null | undefined): string { if (!result) return ""; return `${String(result.stderr || "")} ${String(result.stdout || "")}`.trim(); @@ -602,7 +676,20 @@ export function buildDockerGpuCloneRunArgs( // survive even when the caller explicitly opts into --network=host via // NEMOCLAW_DOCKER_GPU_PATCH_NETWORK=host (#3562, #3568). for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); - for (const group of stringArray(host.GroupAdd)) args.push("--group-add", group); + const groupAdds = new Set(stringArray(host.GroupAdd)); + for (const group of groupAdds) args.push("--group-add", group); + // Jetson/Tegra: grant the sandbox user membership in the host group(s) that + // own /dev/nvmap and the nvhost device nodes so CUDA's nvmap init can open + // them. Without this the unprivileged agent user hits EACCES on /dev/nvmap + // and cuInit(0) returns 999 even though the GPU devices are mounted (#4231). + // Dedupe against any GroupAdd the baseline container already carried. + for (const gid of options.extraGroupGids ?? []) { + const normalized = String(gid).trim(); + if (normalized && !groupAdds.has(normalized)) { + groupAdds.add(normalized); + args.push("--group-add", normalized); + } + } if (networkMode !== "host") { const dnsServers = stringArray(host.Dns); for (const dns of dnsServers) args.push("--dns", dns); @@ -943,6 +1030,26 @@ export function recreateOpenShellDockerSandboxWithGpu( cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null; const sandboxFallbackDns = d.detectSandboxFallbackDns(); if (sandboxFallbackDns) cloneOptions.sandboxFallbackDns = sandboxFallbackDns; + // On Jetson the Tegra GPU device nodes (`/dev/nvmap`, `/dev/nvhost-*`) are + // owned by a non-root group, but the sandbox user is not a member — so + // CUDA fails with `NvRmMemInitNvmap ... Permission denied` and `cuInit(0)` + // returns 999 even though the devices are mounted (#4231). Grant the + // sandbox user the owning group(s) so CUDA can initialize. + if (options.backend === "jetson") { + const tegraGroupGids = d.detectTegraDeviceGroupGids(); + if (tegraGroupGids.length > 0) { + cloneOptions.extraGroupGids = tegraGroupGids; + console.log( + ` ✓ Granting sandbox user access to Jetson Tegra GPU device nodes via --group-add ${tegraGroupGids.join( + ", ", + )} (so CUDA can open /dev/nvmap)`, + ); + } else { + console.warn( + " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", + ); + } + } const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); const runResult = d.dockerRunDetached(cloneArgs, { ignoreError: true, @@ -1026,6 +1133,13 @@ export function applyDockerGpuPatchOrExit( sandboxName: string; gpuDevice?: string | null; timeoutSecs: number; + // Forwarded to `recreateOpenShellDockerSandboxWithGpu` so the Jetson + // backend selects the NVIDIA runtime mode AND grants the Tegra device-node + // group(s) to the sandbox user (#4231). Without threading this through, the + // `ensureApplied` fallback path would recreate the container without + // /dev/nvmap group access. + backend?: DockerGpuPatchBackend; + openshellSandboxCommand?: readonly string[] | null; }, deps: Pick, ): DockerGpuPatchResult { diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index 786522ba3b1..5436f2ad39d 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -332,6 +332,17 @@ "deletionReady": false, "notes": "Initial completeness row; classify detailed coverage and deletion evidence in the owning migration issue before deleting." }, + { + "legacyScript": "test/e2e/test-jetson-nvmap-gpu.sh", + "domain": "platform", + "ownerIssue": "#4354", + "status": "not-migrated", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Jetson Orin /dev/nvmap CUDA usability + status reporter-workflow E2E for #4231; Jetson-gated. Classify coverage and deletion evidence in the owning migration issue before deleting." + }, { "legacyScript": "test/e2e/test-hermes-discord-e2e.sh", "domain": "messaging", diff --git a/test/e2e/test-jetson-nvmap-gpu.sh b/test/e2e/test-jetson-nvmap-gpu.sh new file mode 100755 index 00000000000..3f5f98f37bd --- /dev/null +++ b/test/e2e/test-jetson-nvmap-gpu.sh @@ -0,0 +1,323 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Jetson nvmap GPU status E2E: reproduces the EXACT reporter workflow from +# issue #4231 on a Jetson Orin host and proves the fix. +# +# Reporter symptom (NemoClaw v0.0.58, Jetson Orin, reopened after PR #4599): +# - sandbox user groups: uid=998(sandbox) gid=998(sandbox) groups=998(sandbox) +# - /dev/nvmap is `crw-rw---- root video` +# - CUDA fails inside the sandbox with `NvRmMemInitNvmap failed with +# Permission denied`, cuInit(0)=999 +# - `nemoclaw status` still reports "Sandbox GPU: enabled" (misleading) +# +# Root cause: the Jetson Docker GPU recreate did not grant the sandbox user +# membership in the host group (`video`) that owns the Tegra device nodes, so +# CUDA could not open /dev/nvmap even though the devices were mounted. +# +# This test runs the reporter's exact workflow end-to-end: +# 1. Onboard with GPU passthrough (Jetson auto-enables sandbox GPU) +# 2. Inspect the sandbox user's supplementary groups (`id`) +# 3. Inspect /dev/nvmap inside the sandbox (`ls -l`) +# 4. Run the authoritative CUDA usability proof (cuInit(0)) inside the sandbox +# 5. Assert `nemoclaw status` reports "(CUDA verified)" — not a bare/misleading +# "enabled", "(CUDA unverified)", or "(last CUDA proof failed)" +# +# Acceptance gate (#4231): the test passes only when CUDA actually initializes +# in the sandbox (cuInit(0)=0) AND status reflects proven CUDA usability. A +# bare "enabled" is treated as a failure. +# +# Prerequisites: +# - NVIDIA Jetson Orin (or other L4T/Tegra) host with /dev/nvmap present +# - NVIDIA Container Runtime configured for Docker (nvidia-ctk runtime configure) +# - Docker +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# - A working inference provider (default: ollama; onboard handles startup) +# +# On a non-Jetson host this test SKIPS cleanly (exit 0) so it is safe to wire +# into pipelines that may schedule it on mixed hardware. +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# bash test/e2e/test-jetson-nvmap-gpu.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-jetson-nvmap}" +TEST_LOG="/tmp/nemoclaw-jetson-nvmap-e2e-test.log" +INSTALL_LOG="/tmp/nemoclaw-jetson-nvmap-e2e-install.log" +export NEMOCLAW_PROVIDER="${NEMOCLAW_PROVIDER:-ollama}" + +exec > >(tee -a "$TEST_LOG") 2>&1 + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Jetson hardware gate +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Jetson hardware gate" + +is_jetson() { + [ -e /dev/nvmap ] && return 0 + [ -f /etc/nv_tegra_release ] && return 0 + if [ -r /proc/device-tree/model ] && grep -qi "jetson\|orin\|tegra" /proc/device-tree/model 2>/dev/null; then + return 0 + fi + return 1 +} + +if ! is_jetson; then + skip "Not a Jetson/Tegra host (/dev/nvmap absent) — reporter workflow requires Jetson hardware" + echo "" + echo " This test reproduces issue #4231 on Jetson Orin. It cannot run on" + echo " non-Jetson hardware. Hermetic regression coverage of the same fix" + echo " (sandbox user → /dev/nvmap group propagation) lives in" + echo " src/lib/onboard/docker-gpu-patch.test.ts." + echo "" + echo " Skipped (exit 0): no Jetson hardware available." + exit 0 +fi +pass "Jetson/Tegra host detected (/dev/nvmap present)" + +HOST_NVMAP_PERMS="$(ls -l /dev/nvmap 2>/dev/null || true)" +HOST_NVMAP_GID="$(stat -c '%g' /dev/nvmap 2>/dev/null || true)" +HOST_NVMAP_GROUP="$(stat -c '%G' /dev/nvmap 2>/dev/null || true)" +info "Host /dev/nvmap: ${HOST_NVMAP_PERMS}" +info "Host /dev/nvmap owning group: ${HOST_NVMAP_GROUP} (gid ${HOST_NVMAP_GID})" + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive onboard" + exit 1 +fi + +# Best-effort cleanup on any exit (prevents dirty state on reused runners) +# shellcheck disable=SC2329 # invoked via trap +cleanup() { + info "Running exit cleanup..." + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + fi + if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + fi + pkill -f "ollama serve" 2>/dev/null || true + pkill -f "ollama-auth-proxy" 2>/dev/null || true +} +trap cleanup EXIT + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +# Jetson sandbox GPU uses the NVIDIA Container Runtime (not CDI). +if docker info --format '{{json .Runtimes}}' 2>/dev/null | grep -q '"nvidia"\|nvidia:'; then + pass "Docker NVIDIA runtime detected" +else + fail "Docker NVIDIA runtime not detected — run: sudo nvidia-ctk runtime configure --runtime=docker" + exit 1 +fi + +# Pre-cleanup +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi + +# Install Ollama binary if the provider needs it (onboard starts it). +if [ "$NEMOCLAW_PROVIDER" = "ollama" ] && ! command -v ollama >/dev/null 2>&1; then + info "Installing Ollama binary..." + curl -fsSL https://ollama.com/install.sh | sh 2>&1 || true + systemctl stop ollama 2>/dev/null || true + pkill -f "ollama serve" 2>/dev/null || true +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Onboard with GPU (reporter workflow) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Onboard with GPU passthrough" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Running install.sh --non-interactive (Jetson auto-enables sandbox GPU)..." +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Pick up PATH changes from the installer. +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + tail -40 "$INSTALL_LOG" + exit 1 +fi + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# 2a: The Jetson recreate must announce that it grants the Tegra device-node +# group(s) to the sandbox user (the fix for #4231). +if grep -Fq "Granting sandbox user access to Jetson Tegra GPU device nodes via --group-add" "$INSTALL_LOG"; then + GROUP_ADD_LINE="$(grep -F "Granting sandbox user access to Jetson Tegra GPU device nodes" "$INSTALL_LOG" | head -1)" + pass "Onboard granted Tegra device-node group via --group-add" + info "${GROUP_ADD_LINE}" +else + fail "Onboard did not grant the Tegra device-node group (--group-add) — #4231 fix missing" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: In-sandbox device + group + CUDA inspection (reporter workflow) +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: In-sandbox /dev/nvmap, groups, and CUDA proof" + +# 3a: sandbox user supplementary groups — must now include the /dev/nvmap GID. +SANDBOX_ID="$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc 'id' 2>&1)" || true +info "sandbox 'id': ${SANDBOX_ID}" +if [ -n "$HOST_NVMAP_GID" ] && echo "$SANDBOX_ID" | grep -Eq "(^|[(,=])${HOST_NVMAP_GID}([(,) ]|$)"; then + pass "Sandbox user is a member of the /dev/nvmap owning group (gid ${HOST_NVMAP_GID})" +else + fail "Sandbox user is NOT in the /dev/nvmap owning group (gid ${HOST_NVMAP_GID}) — CUDA will be denied" +fi + +# 3b: /dev/nvmap present inside the sandbox. +SANDBOX_NVMAP="$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc 'ls -l /dev/nvmap' 2>&1)" || true +info "sandbox /dev/nvmap: ${SANDBOX_NVMAP}" +if echo "$SANDBOX_NVMAP" | grep -q "/dev/nvmap"; then + pass "/dev/nvmap is present inside the sandbox" +else + fail "/dev/nvmap is not present inside the sandbox" +fi + +# 3c: Authoritative CUDA usability proof — cuInit(0) must return 0. This is the +# exact signal the reporter saw fail (cuInit=999, NvRmMemInitNvmap denied). +CUDA_PROBE='python3 -c '\''import ctypes; lib = ctypes.CDLL("libcuda.so.1"); rc = lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(0 if rc == 0 else 1)'\''' +CUDA_OUT="$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$CUDA_PROBE" 2>&1)" || true +info "sandbox cuInit probe: ${CUDA_OUT}" +if echo "$CUDA_OUT" | grep -q "cuInit(0)=0"; then + pass "CUDA initialized inside the sandbox (cuInit(0)=0)" +elif echo "$CUDA_OUT" | grep -qi "NvRmMemInitNvmap\|Permission denied"; then + fail "CUDA failed with the reporter's nvmap permission error: ${CUDA_OUT}" +else + fail "CUDA did not initialize inside the sandbox: ${CUDA_OUT}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: nemoclaw status must report proven CUDA usability +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: nemoclaw status CUDA proof state" + +STATUS_OUT="$(nemoclaw "$SANDBOX_NAME" status 2>&1)" || true +echo "$STATUS_OUT" | grep -F "Sandbox GPU:" || true + +if echo "$STATUS_OUT" | grep -Fq "Sandbox GPU: enabled"; then + pass "Status reports Sandbox GPU: enabled (Jetson auto-enable)" +else + fail "Status does not report Sandbox GPU enabled" +fi + +# The core #4231 assertion: status must carry "(CUDA verified)" — a bare +# "enabled", "(CUDA unverified)", or "(last CUDA proof failed)" is the +# misleading state the reporter hit and must NOT pass. +if echo "$STATUS_OUT" | grep -Fq "CUDA verified"; then + pass "Status reports (CUDA verified) — GPU usability is proven, not misleading" +elif echo "$STATUS_OUT" | grep -Eq "last CUDA proof failed"; then + fail "Status shows CUDA proof FAILED — Jetson nvmap access not granted (#4231 unfixed)" +elif echo "$STATUS_OUT" | grep -Fq "CUDA unverified"; then + fail "Status shows CUDA UNVERIFIED — proof did not confirm usability (#4231 misleading state)" +else + fail "Status reports a bare 'enabled' with no CUDA proof state (#4231 misleading status)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Jetson nvmap GPU E2E Results (#4231):" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" +echo "" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Jetson nvmap GPU E2E PASSED — CUDA usable + status proven (#4231).\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi